Python requests.packages.urllib3.util.retry.Retry() Examples

def requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(500, 502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def _init_session(session, **kwargs):
    if session is None:
        if kwargs.get('asynchronous'):
            session = FuturesSession(max_workers=kwargs.get('max_workers', 8))
        else:
            session = Session()
        if kwargs.get('proxies'):
            session.proxies = kwargs.get('proxies')
        retries = Retry(
            total=3,
            backoff_factor=1,
            status_forcelist=[429, 500, 502, 503, 504],
            method_whitelist=["HEAD", "GET", "OPTIONS", "POST", "TRACE"])
        session.mount('https://', TimeoutHTTPAdapter(
            max_retries=retries,
            timeout=kwargs.get('timeout', DEFAULT_TIMEOUT)))
        # TODO: Figure out how to utilize this within the validate_response
        # TODO: This will be a much better way of handling bad requests than
        # TODO: what I'm currently doing.
        # session.hooks['response'] = \
        #     [lambda response, *args, **kwargs: response.raise_for_status()]
        session.headers.update({
            "User-Agent": random.choice(USER_AGENT_LIST)
        })
    return session 

def get_docker_client(base_url=None, retry_read=config.DOCKER_MAX_READ_RETRIES,
                      retry_status_forcelist=(500,)):
    client_key = (retry_read, retry_status_forcelist)
    if client_key not in _DOCKER_CLIENTS:
        client = docker.DockerClient(base_url=base_url or config.DOCKER_URL,
                                     timeout=config.DOCKER_TIMEOUT)
        retries = Retry(total=config.DOCKER_MAX_TOTAL_RETRIES,
                        connect=config.DOCKER_MAX_CONNECT_RETRIES,
                        read=retry_read,
                        method_whitelist=False,
                        status_forcelist=retry_status_forcelist,
                        backoff_factor=config.DOCKER_BACKOFF_FACTOR,
                        raise_on_status=False)
        http_adapter = HTTPAdapter(max_retries=retries)
        client.api.mount('http://', http_adapter)
        _DOCKER_CLIENTS[client_key] = client
    return _DOCKER_CLIENTS[client_key] 

def _requests_retry_session(
        self,
        retries=3,
        backoff_factor=0.3,
        method_whitelist=['HEAD', 'GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'],
        status_forcelist=(500, 502, 503, 504, 520, 524),
        session=None,
    ):
        session = session or requests.Session()
        retry = Retry(
            total=retries,
            read=retries,
            connect=retries,
            status=retries,
            method_whitelist=method_whitelist,
            backoff_factor=backoff_factor,
            status_forcelist=status_forcelist,
        )
        adapter = HTTPAdapter(max_retries=retry)
        session.mount('http://', adapter)
        session.mount('https://', adapter)
        return session 

def requests_retry_session(
    retries=5,
    backoff_factor=0.3,
    status_forcelist=(502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(502, 504),
    session=None,
    **kwargs
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def retry(num=5):
    """"retry connection.
    
        define max tries num
        if the backoff_factor is 0.1, then sleep() will sleep for
        [0.1s, 0.2s, 0.4s, ...] between retries.
        It will also force a retry if the status code returned is 500, 502, 503 or 504.    
    
    """
    s = requests.Session()
    retries = Retry(total=num, backoff_factor=0.1,
                    status_forcelist=[500, 502, 503, 504])
    s.mount('http://', HTTPAdapter(max_retries=retries))

    return s

# CONSTANT 

def requests_retry_session(
    retries=5,
    backoff_factor=1,
    status_forcelist=[401, 402, 403, 500, 502, 504],
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount("http://", adapter)
    session.mount("https://", adapter)
    return session 

def send(self, request, *args, **kwargs):
        content_length_is_unknown = (request.body is not None
                                     and 'Content-Length' not in request.headers)
        if content_length_is_unknown:
            # it seems that requests's HTTPAdapter does not perform
            # retries in such a case, even though they were requested
            # (see the source code of HTTPAdapter.send() in conjunction
            # with urllib3.connectionpool.HTTPConnectionPool.urlopen()
            # and urllib3.util.retry.Retry.increment()...) -- so here
            # we raise an exception to prevent such a silent omission
            # [we analyzed this for requests==2.21.0 and urllib3==1.24.1]
            raise ValueError('non-zero `retries` has been specified and, '
                             'at the same time, Content-Length of the request '
                             'could not be determined (suggested solutions: '
                             'specify `data` whose length is discoverable, '
                             'or specify `retries=0`)')

        return super(_HTTPAdapterForRetries, self).send(request, *args, **kwargs) 

def requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(500, 502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(500, 502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def retry_session():
    """Get a Requests session that retries HTTP and HTTPS requests.
    """
    # Adapted from:
    # https://www.peterbe.com/plog/best-practice-with-retries-with-requests
    session = requests.Session()
    retry = Retry(
        total=RETRY_COUNT,
        read=RETRY_COUNT,
        connect=RETRY_COUNT,
        backoff_factor=RETRY_FACTOR,
        status_forcelist=RETRY_STATUSES,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def __retry_session(self, retries=10, backoff_factor=0.3,
                        status_forcelist=(500, 502, 504),
                        session=None):
        """
        Retry the connection using requests if it fails. Use this as a wrapper
        to request from datapoint
        """

        # requests.Session allows finer control, which is needed to use the
        # retrying code
        the_session = session or requests.Session()

        # The Retry object manages the actual retrying
        retry = Retry(total=retries, read=retries, connect=retries,
                      backoff_factor=backoff_factor,
                      status_forcelist=status_forcelist)

        adapter = HTTPAdapter(max_retries=retry)

        the_session.mount('http://', adapter)
        the_session.mount('https://', adapter)

        return the_session 

def retry_session():
    """Get a Requests session that retries HTTP and HTTPS requests.
    """
    # Adapted from:
    # https://www.peterbe.com/plog/best-practice-with-retries-with-requests
    session = requests.Session()
    retry = Retry(
        total=RETRY_COUNT,
        read=RETRY_COUNT,
        connect=RETRY_COUNT,
        backoff_factor=RETRY_FACTOR,
        status_forcelist=RETRY_STATUSES,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def requests_retry_session(retries=3,
                           backoff_factor=0.3,
                           status_forcelist=(500, 502, 504),
                           session=None,
                           ):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def requests_retry_session(url, retries=3, backoff_factor=1,
                           status_forcelist=(429, 500, 502, 503, 504),
                           session=None):
    """
    Configuration for `requests` retries.

    Args:
        url: url to get
        retries: total number of retry attempts
        backoff_factor: amount of time between attempts
        status_forcelist: retry if response is in list
        session: requests session object

    Example:
        req = requests_retry_session().get(<url>)
    """
    session = session or requests.Session()
    retry = Retry(total=retries,
                  backoff_factor=backoff_factor,
                  status_forcelist=status_forcelist)
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session.get(url) 

def requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(500, 502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(500, 502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def session_with_retry(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(500, 502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session


# Take first value of api list generated and attempt a GET to see if upstream server challenge us back for auth. 

def create_requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(500, 502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def decode(key: str, string: str) -> str:
    string = base64.urlsafe_b64decode(string.encode() + b'===')
    string = string.decode('latin')
    encoded_chars = []
    for i in range(len(string)):
        key_c = key[i % len(key)]
        encoded_c = chr((ord(string[i]) - ord(key_c) + 256) % 256)
        encoded_chars.append(encoded_c)
    encoded_string = ''.join(encoded_chars)
    return encoded_string

# e = encode('a key', 'a message')
# d = decode('a key', e)
# print([e])
# print([d])


# https://www.peterbe.com/plog/best-practice-with-retries-with-requests
# Provides a requests Session object with requests' Retry capabilities.
# TODO: may make numbers below configurable 

def initialize_client_with_connection_retry_config(self, retries):
        connection_policy = documents.ConnectionPolicy()
        connection_policy.ConnectionRetryConfiguration = Retry(
                                                            total=retries,
                                                            read=retries,
                                                            connect=retries,
                                                            backoff_factor=0.3,
                                                            status_forcelist=(500, 502, 504)
                                                        )
        start_time = time.time()
        try:
            cosmos_client.CosmosClient("https://localhost:9999",
                                                {'masterKey': CRUDTests.masterKey},
                                                connection_policy)
            self.fail()
        except ConnectionError as e:
            end_time = time.time()
            return end_time - start_time 

def test_client_request_timeout_when_connection_retry_configuration_specified(self):
        connection_policy = documents.ConnectionPolicy()
        # making timeout 0 ms to make sure it will throw
        connection_policy.RequestTimeout = 0
        connection_policy.ConnectionRetryConfiguration = Retry(
                                                            total=3,
                                                            read=3,
                                                            connect=3,
                                                            backoff_factor=0.3,
                                                            status_forcelist=(500, 502, 504)
                                                        )
        with self.assertRaises(Exception):
            # client does a getDatabaseAccount on initialization, which will time out
            cosmos_client.CosmosClient(CRUDTests.host,
                                                {'masterKey': CRUDTests.masterKey},
                                                connection_policy) 

def get_simple_salesforce_connection(project_config, org_config, api_version=None):
    # Retry on long-running metadeploy jobs
    retries = Retry(total=5, status_forcelist=(502, 503, 504), backoff_factor=0.3)
    adapter = HTTPAdapter(max_retries=retries)

    sf = simple_salesforce.Salesforce(
        instance_url=org_config.instance_url,
        session_id=org_config.access_token,
        version=api_version or project_config.project__package__api_version,
    )
    try:
        app = project_config.keychain.get_service("connectedapp")
        client_name = app.client_id
    except (ServiceNotValid, ServiceNotConfigured):
        client_name = "CumulusCI/{}".format(__version__)

    sf.headers.setdefault(CALL_OPTS_HEADER_KEY, "client={}".format(client_name))
    sf.session.mount("http://", adapter)
    sf.session.mount("https://", adapter)

    return sf 

def create_session():
    """
    retry on 502
    """
    session = Session()
    retry = Retry(
        status=5,
        backoff_factor=0.3,
        status_forcelist=(502,),
        # CAUTION: adding 'POST' to this list which is not technically idempotent
        method_whitelist=("POST", "HEAD", "TRACE", "GET", "PUT", "OPTIONS", "DELETE"),
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount("https://", adapter)
    return session 

def get_requests_retry_session(max_retries=4, backoff_factor=1, status_forcelist=None):
    status_forcelist = status_forcelist or [500, 502, 504]
    # Default max retries 4 with sleeping between retries 1s, 2s, 4s, 8s
    session = requests.Session()
    custom_retry = Retry(total=max_retries,
                         backoff_factor=backoff_factor,
                         status_forcelist=status_forcelist)
    custom_adapter = HTTPAdapter(max_retries=custom_retry)
    # Any request through this session that starts with 'http://' or 'https://'
    # will use the custom Transport Adapter created which include retries
    session.mount('http://', custom_adapter)
    session.mount('https://', custom_adapter)
    return session 

def kube_metrics(self):
        headers = { "Authorization": "Bearer {}".format(self.token) }
        query = { 'labelSelector' : self.labelSelector }
        session = requests.Session()
        retry = Retry(total=3, connect=3, backoff_factor=0.1)
        adapter = HTTPAdapter(max_retries=retry)
        session.mount('http://', adapter)
        session.mount('https://', adapter)
        if self.insecure_tls:
            session.verify = False
        elif os.path.exists(self.ca_cert):
            session.verify = self.ca_cert
        if self.namespaces:
            pod_data = None
            for namespace in self.namespaces:
                if pod_data is None:
                    pod_data = session.get(self.set_namespaced_pod_url(namespace), headers=headers, params=query).json()
                else:
                    pod_data['items'] += session.get(self.set_namespaced_pod_url(namespace), headers=headers, params=query).json()['items']
            payload = {
                'nodes': session.get(self.api_nodes_url, headers=headers, params=query).json(),
                'pods':  pod_data
            }
        else:
            payload = {
                'nodes': session.get(self.api_nodes_url, headers=headers, params=query).json(),
                'pods':  session.get(self.api_pods_url,  headers=headers, params=query).json()
            }

        return payload 

def __init__(self, *args, **kwargs):
        super(RetryAdapter, self).__init__(*args, **kwargs)
        self.max_retries = Retry(total=RETRY_COUNT,
                                 backoff_factor=BACKOFF_FACTOR) 

def scrape_page(url):
    """
    Scrape a given url

    :param url: url for page

    :return: response object
    """
    response = requests.Session()
    retries = Retry(total=10, backoff_factor=.1)
    response.mount('http://', HTTPAdapter(max_retries=retries))

    try:
        response = response.get(url, timeout=5)
        response.raise_for_status()
        page = response.text
    except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError):
        page = None
    except requests.exceptions.ReadTimeout:
        # If it times out and it's the schedule print an error message...otherwise just make the page = None
        if "schedule" in url:
            raise Exception("Timeout Error: The NHL API took too long to respond to our request. "
                                "Please Try Again (you may need to try a few times before it works). ")
        else:
            print_error("Timeout Error: The server took too long to respond to our request.")
            page = None

    # Pause for 1 second - make it more if you want
    time.sleep(1)

    return page 

def request(url, method, payload=None):
    retryTotal = 5 if os.getenv("REQ_RETRY_TOTAL") is None else int(os.getenv("REQ_RETRY_TOTAL"))
    retryConnect = 5 if os.getenv("REQ_RETRY_CONNECT") is None else int(
        os.getenv("REQ_RETRY_CONNECT"))
    retryRead = 5 if os.getenv("REQ_RETRY_READ") is None else int(os.getenv("REQ_RETRY_READ"))
    retryBackoffFactor = 0.2 if os.getenv("REQ_RETRY_BACKOFF_FACTOR") is None else float(
        os.getenv("REQ_RETRY_BACKOFF_FACTOR"))
    timeout = 10 if os.getenv("REQ_TIMEOUT") is None else float(os.getenv("REQ_TIMEOUT"))

    username = os.getenv("REQ_USERNAME")
    password = os.getenv("REQ_PASSWORD")
    if username and password:
        auth = (username, password)
    else:
        auth = None

    r = requests.Session()
    retries = Retry(total=retryTotal,
                    connect=retryConnect,
                    read=retryRead,
                    backoff_factor=retryBackoffFactor,
                    status_forcelist=[500, 502, 503, 504])
    r.mount("http://", HTTPAdapter(max_retries=retries))
    r.mount("https://", HTTPAdapter(max_retries=retries))
    if url is None:
        print(f"{timestamp()} No url provided. Doing nothing.")
        return

    # If method is not provided use GET as default
    if method == "GET" or not method:
        res = r.get("%s" % url, auth=auth, timeout=timeout)
    elif method == "POST":
        res = r.post("%s" % url, auth=auth, json=payload, timeout=timeout)
        print(f"{timestamp()} {method} request sent to {url}. "
              f"Response: {res.status_code} {res.reason} {res.text}")
    return res