(url, timeout=10, retry=False, max_attempts=3)
Fetch a URL, optionally retrying on failure.

    :param url: The URL to fetch (a string).
    :param timeout: The maximum time in seconds that's allowed to pass before
                    the request is aborted (a number, defaults to 10 seconds).
    :param retry: Whether to retry on failure (defaults to :data:`False`).
    :param max_attempts: The maximum number of attempts when retrying is
                         enabled (an integer, defaults to three).
    :returns: The response body (a byte string).
    :raises: Any of the following exceptions can be raised:

             - :exc:`NotFoundError` when the URL returns a 404 status code.
             - :exc:`InvalidResponseError` when the URL returns a status code
               that isn't 200.
             - :exc:`stopit.TimeoutException` when the request takes longer
               than `timeout` seconds (refer to the `stopit documentation
               <https://pypi.python.org/pypi/stopit>`_ for details).
             - Any exception raised by Python's standard library in the last
               attempt (assuming all attempts raise an exception).
Fetch a URL, optionally retrying on failure.
Fetch a URL optionally retrying on failure .
def fetch_url(url, timeout=10, retry=False, max_attempts=3):
    """
    Fetch a URL, optionally retrying on failure.

    :param url: The URL to fetch (a string).
    :param timeout: The maximum time in seconds that's allowed to pass before
                    the request is aborted (a number, defaults to 10 seconds).
    :param retry: Whether to retry on failure (defaults to :data:`False`).
    :param max_attempts: The maximum number of attempts when retrying is
                         enabled (an integer, defaults to three).
    :returns: The response body (a byte string).
    :raises: Any of the following exceptions can be raised:

             - :exc:`NotFoundError` when the URL returns a 404 status code.
             - :exc:`InvalidResponseError` when the URL returns a status code
               that isn't 200.
             - :exc:`stopit.TimeoutException` when the request takes longer
               than `timeout` seconds (refer to the `stopit documentation
               <https://pypi.python.org/pypi/stopit>`_ for details).
             - Any exception raised by Python's standard library in the last
               attempt (assuming all attempts raise an exception).
    """
    timer = Timer()
    logger.debug("Fetching %s ..", url)
    for i in range(1, max_attempts + 1):
        try:
            with SignalTimeout(timeout, swallow_exc=False):
                response = urlopen(url)
                status_code = response.getcode()
                if status_code != 200:
                    exc_type = (NotFoundError if status_code == 404 else InvalidResponseError)
                    raise exc_type("URL returned unexpected status code %s! (%s)" % (status_code, url))
                response_body = response.read()
                logger.debug("Took %s to fetch %s.", timer, url)
                return response_body
        except NotFoundError:
            # We never retry 404 responses but retry timeouts.
            raise
        except Exception as e:
            if retry and i < max_attempts:
                logger.warning("Failed to fetch %s, retrying (%i/%i, error was: %s)", url, i, max_attempts, e)
            else:
                raise
def fetch_url ( url , timeout = 10 , retry = False , max_attempts = 3 ) : timer = Timer ( ) logger . debug ( "Fetching %s .." , url ) for i in range ( 1 , max_attempts + 1 ) : try : with SignalTimeout ( timeout , swallow_exc = False ) : response = urlopen ( url ) status_code = response . getcode ( ) if status_code != 200 : exc_type = ( NotFoundError if status_code == 404 else InvalidResponseError ) raise exc_type ( "URL returned unexpected status code %s! (%s)" % ( status_code , url ) ) response_body = response . read ( ) logger . debug ( "Took %s to fetch %s." , timer , url ) return response_body except NotFoundError : # We never retry 404 responses but retry timeouts. raise except Exception as e : if retry and i < max_attempts : logger . warning ( "Failed to fetch %s, retrying (%i/%i, error was: %s)" , url , i , max_attempts , e ) else : raise