Python requests.adapters.HTTPAdapter() Examples

def connect(self):

        if self.session is None:

            # Creating a session that can later be re-use to configure custom mechanisms
            self.session = requests.Session()

            # If we specified some retries, we provide a predefined retry-logic
            if self.retries > 0:
                adapter = HTTPAdapter(
                    max_retries=Retry(
                        total=self.retries,
                        backoff_factor=0.1,
                        status_forcelist=[500, 502, 503, 504],
                    )
                )
                for prefix in "http://", "https://":
                    self.session.mount(prefix, adapter)
        else:
            raise TransportAlreadyConnected("Transport is already connected") 

def __init__(self, endpoint, session_id=None, req_session=None, args=None):
        self._endpoint = endpoint.rstrip('/')
        self._session_id = session_id
        self._args = args or dict()
        # dict structure: {tileable_key -> graph_key, tileable_ids}
        # dict value is a tuple object which records graph key and tileable id
        self._executed_tileables = dict()

        self._serial_type = None
        self._pickle_protocol = pickle.HIGHEST_PROTOCOL

        if req_session:
            self._req_session = req_session
        else:
            import requests
            from requests.adapters import HTTPAdapter

            self._req_session = requests.Session()
            self._req_session.mount('http://stackoverflow.com', HTTPAdapter(max_retries=5))

        self._main() 

def requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(500, 502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def get_requests_session(self):
        requests_session = self._session.GetParameter("requests_session")
        if requests_session == None:
            # To make sure we can use the requests session in the threadpool we
            # need to make sure that the connection pool can block. Otherwise it
            # will raise when it runs out of connections and the threads will be
            # terminated.
            requests_session = requests.Session()
            requests_session.mount("https://", adapters.HTTPAdapter(
                pool_connections=10, pool_maxsize=300, max_retries=10,
                pool_block=True))

            requests_session.mount("http://", adapters.HTTPAdapter(
                pool_connections=10, pool_maxsize=300, max_retries=10,
                pool_block=True))

            self._session.SetCache("requests_session", requests_session)

        return requests_session 

def _requests_retry_session(
        self,
        retries=3,
        backoff_factor=0.3,
        method_whitelist=['HEAD', 'GET', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'TRACE'],
        status_forcelist=(500, 502, 503, 504, 520, 524),
        session=None,
    ):
        session = session or requests.Session()
        retry = Retry(
            total=retries,
            read=retries,
            connect=retries,
            status=retries,
            method_whitelist=method_whitelist,
            backoff_factor=backoff_factor,
            status_forcelist=status_forcelist,
        )
        adapter = HTTPAdapter(max_retries=retry)
        session.mount('http://', adapter)
        session.mount('https://', adapter)
        return session 

def __init__(self, loginer=None, use_proxy=False):
        self.loginer = loginer
        self.use_proxy = use_proxy
        if use_proxy:
            self.proxy_pool = ProxyPool()
            if len(self.proxy_pool) == 0:
                self.use_proxy = False
        self._cookies = None

        self._headers = dict()
        self._headers[
            "User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36"
        self._headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
        self._headers["Accept-Encoding"] = "gzip, deflate, sdch"
        self._headers["Accept-Language"] = "zh-CN,zh;q=0.8"
        self._request_retry = HTTPAdapter(max_retries=3)

        cookie_dict = dict()
        self._cookies = cookie_dict 

def requests_retry_session(
    retries=5,
    backoff_factor=1,
    status_forcelist=[401, 402, 403, 500, 502, 504],
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount("http://", adapter)
    session.mount("https://", adapter)
    return session 

def init_poolmanager(self, connections, maxsize, block=DEFAULT_POOLBLOCK):
        """Initializes a urllib3 PoolManager. This method should not be called
        from user code, and is only exposed for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param connections: The number of urllib3 connection pools to cache.
        :param maxsize: The maximum number of connections to save in the pool.
        :param block: Block when no free connections are available.
        """
        # save these values for pickling
        self._pool_connections = connections
        self._pool_maxsize = maxsize
        self._pool_block = block

        self.poolmanager = AsyncPoolManager(num_pools=connections, maxsize=maxsize, block=block)
        self.connections = [] 

def __init__(self, loginer=None, use_proxy=False):
        self.loginer = loginer
        self.use_proxy = use_proxy
        if use_proxy:
            self.proxy_pool = ProxyPool()
            if len(self.proxy_pool) == 0:
                self.use_proxy = False
        self._cookies = None

        self._headers = dict()
        self._headers["User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36"
        self._headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
        self._headers["Accept-Encoding"] = "gzip, deflate, sdch"
        self._headers["Accept-Language"] = "zh-CN,zh;q=0.8"
        self._request_retry = HTTPAdapter(max_retries=3)

        cookie_dict = dict()
        self._cookies = cookie_dict

    # def init_loginer(self, account, password):
    #     self._cookies = self.loginer.login(account, password)
    #
    # def set_cookies(self, cookies):
    #     self._cookies = cookies 

def proxy_manager_for(self, proxy, **proxy_kwargs):
        """Called to initialize the HTTPAdapter when a proxy is used."""
        try:
            proxy_kwargs['ssl_version'] = ssl.PROTOCOL_TLS
        except AttributeError:
            proxy_kwargs['ssl_version'] = ssl.PROTOCOL_SSLv23
        return super(SSLAdapter, self).proxy_manager_for(proxy, **proxy_kwargs) 

def requests_retry_session(
    retries=5,
    backoff_factor=0.3,
    status_forcelist=(502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(502, 504),
    session=None,
    **kwargs
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def retry(
    total_requests=10, backoff_factor=1, statuses=(500, 502, 503, 504, 429)
):
    """Default HTTP session for requests."""
    _session = requests.Session()

    retries = Retry(
        total=total_requests,
        backoff_factor=backoff_factor,
        status_forcelist=list(statuses)
    )

    _session.mount('http://', HTTPAdapter(max_retries=retries))
    _session.mount('https://', HTTPAdapter(max_retries=retries))

    yield _session 

def get_requests_session(self):
        requests_session = self._session.GetParameter("requests_session")
        if requests_session == None:
            # To make sure we can use the requests session in the threadpool we
            # need to make sure that the connection pool can block. Otherwise it
            # will raise when it runs out of connections and the threads will be
            # terminated.
            requests_session = requests.Session()
            requests_session.mount("https://", adapters.HTTPAdapter(
                pool_connections=10, pool_maxsize=300, max_retries=10,
                pool_block=True))

            requests_session.mount("http://", adapters.HTTPAdapter(
                pool_connections=10, pool_maxsize=300, max_retries=10,
                pool_block=True))

            self._session.SetCache("requests_session", requests_session)

        return requests_session 

def post_with_retries(url: str, data: dict, retries: int, backoff: float) -> int:
    """
    Make a POST request with retries.

    >>> post_with_retries('http://httpstat.us/503', {}, retries=1, backoff=0)
    500
    >>> post_with_retries('https://httpstat.us/200', {}, retries=1, backoff=0)
    200
    """
    retry_adapter = HTTPAdapter(max_retries=Retry(
        total=retries,
        backoff_factor=backoff,
        status_forcelist=[500, 502, 503, 504],
        method_whitelist=frozenset(['POST'])
    ))

    with Session() as session:
        session.mount('http://', retry_adapter)
        session.mount('https://', retry_adapter)

        try:
            response = session.post(url, data=data)
        except RetryError:
            return 500

        return response.status_code 

def retry(num=5):
    """"retry connection.
    
        define max tries num
        if the backoff_factor is 0.1, then sleep() will sleep for
        [0.1s, 0.2s, 0.4s, ...] between retries.
        It will also force a retry if the status code returned is 500, 502, 503 or 504.    
    
    """
    s = requests.Session()
    retries = Retry(total=num, backoff_factor=0.1,
                    status_forcelist=[500, 502, 503, 504])
    s.mount('http://', HTTPAdapter(max_retries=retries))

    return s

# CONSTANT 

def requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(500, 502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def open_session(api_key, max_retries=5, user_agent="civis-python"):
    """Create a new Session which can connect with the Civis API"""
    civis_version = civis.__version__
    session = requests.Session()
    session.auth = (api_key, '')
    session_agent = session.headers.get('User-Agent', '')
    ver_string = "{}.{}.{}".format(sys.version_info.major,
                                   sys.version_info.minor,
                                   sys.version_info.micro)
    user_agent = "{}/Python v{} Civis v{} {}".format(
        user_agent, ver_string, civis_version, session_agent)
    session.headers.update({"User-Agent": user_agent.strip()})
    max_retries = AggressiveRetry(max_retries, backoff_factor=.75,
                                  status_forcelist=civis.civis.RETRY_CODES)
    adapter = HTTPAdapter(max_retries=max_retries)
    session.mount("https://", adapter)

    return session 

def check_github_users(usernames, retries=5, backoff_factor=0.3, status_forcelist=(500, 502, 503, 504)):
    """
    Check if provided usernames exist in Github
    :param usernames: list of usernames
    :param retries: number of retries
    :param backoff_factor: backoff to apply between retries
    :param status_forcelist: HTTP status codes that should be retried
    :return: list of usernames that exist in Github
    """
    session = requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('https://', adapter)
    return [
        username for username in usernames
        if session.get('https://github.com/{}.keys'.format(username)).status_code == 200
    ] 

def requests_retry_session(
    retries=3,
    backoff_factor=0.3,
    status_forcelist=(500, 502, 504),
    session=None,
):
    session = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session 

def get_sync_result(result_payload: dict, sleep_time: int) -> dict:
    # создаём сессию
    session = requests.Session()
    # выставляем кол-во попыток подключения к серверу при ошибке
    session.mount("http://", HTTPAdapter(max_retries=5))
    session.mount("https://", HTTPAdapter(max_retries=5))
    session.verify = False

    while True:
        captcha_response = session.post(get_result_url, json=result_payload).json()

        if captcha_response["errorId"] == 0:
            if captcha_response["status"] == "processing":
                time.sleep(sleep_time)
            else:
                captcha_response.update({"taskId": result_payload["taskId"]})
                session.close()
                return captcha_response
        else:
            captcha_response.update({"taskId": result_payload["taskId"]})
            session.close()
            return captcha_response 

def get_retrying_requests_session(client_statuses=HTTP_CLIENT_STATUS_RETRY,
                                  times=HTTP_MAX_RETRIES, delay=HTTP_BACKOFF_FACTOR,
                                  method_whitelist=None, raise_on_status=True):
    if _http_retries_disabled():
        times = 0

    retry = Retry(
        total=int(times),
        backoff_factor=delay,
        status_forcelist=client_statuses,
        method_whitelist=method_whitelist
    )

    # raise_on_status was added later to Retry, adding compatibility to work
    # with newer versions and ignoring this option with older ones
    if hasattr(retry, 'raise_on_status'):
        retry.raise_on_status = raise_on_status

    session = SessionWithTimeout()
    session.mount('http://', HTTPAdapter(max_retries=retry))
    session.mount('https://', HTTPAdapter(max_retries=retry))

    return session 

def retry_session(retries=3, backoff_factor=0.8, status_forcelist=(500, 502, 504)):
    """Add retry to Requests Session.

    https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.retry.Retry
    """
    session = Session()
    retries = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    # mount all https requests
    session.mount('https://', HTTPAdapter(max_retries=retries))
    return session 

def get_docker_client(base_url=None, retry_read=config.DOCKER_MAX_READ_RETRIES,
                      retry_status_forcelist=(500,)):
    client_key = (retry_read, retry_status_forcelist)
    if client_key not in _DOCKER_CLIENTS:
        client = docker.DockerClient(base_url=base_url or config.DOCKER_URL,
                                     timeout=config.DOCKER_TIMEOUT)
        retries = Retry(total=config.DOCKER_MAX_TOTAL_RETRIES,
                        connect=config.DOCKER_MAX_CONNECT_RETRIES,
                        read=retry_read,
                        method_whitelist=False,
                        status_forcelist=retry_status_forcelist,
                        backoff_factor=config.DOCKER_BACKOFF_FACTOR,
                        raise_on_status=False)
        http_adapter = HTTPAdapter(max_retries=retries)
        client.api.mount('http://', http_adapter)
        _DOCKER_CLIENTS[client_key] = client
    return _DOCKER_CLIENTS[client_key] 

def requests(self, session: requests.Session):
        """
        A backoff factor to apply between attempts after the second try (most errors are resolved immediately by a
        second try without a delay). urllib3 will sleep for: {backoff factor} * (2 ^ ({number of total retries} - 1))
        seconds. If the backoff_factor is 0.1, then sleep() will sleep for [0.0s, 0.2s, 0.4s, ...] between retries.
        It will never be longer than Retry.BACKOFF_MAX. By default, backoff is disabled (set to 0).
        :param session:
        :return:
        """
        session_r = session or requests.Session()
        max_retries = Retry(
            total=self.retries,
            read=self.retries,
            connect=self.retries,
            backoff_factor=0.3,
            status_forcelist=self.status_retries,
        )
        adapter = HTTPAdapter(max_retries=max_retries)
        session_r.mount('http://', adapter)
        session_r.mount('https://', adapter)
        return session_r 

def init_poolmanager(self, connections, maxsize, block=False,
            **pool_kwargs):
        # Rewrite of the requests.adapters.HTTPAdapter.init_poolmanager method
        # to use WeakCiphersPoolManager instead of urllib3's PoolManager
        self._pool_connections = connections
        self._pool_maxsize = maxsize
        self._pool_block = block

        self.poolmanager = WeakCiphersPoolManager(num_pools=connections,
                maxsize=maxsize, block=block, strict=True, **pool_kwargs) 

def get_connection(self, url, proxies=None):
        """Returns a urllib3 connection for the given URL. This should not be
        called from user code, and is only exposed for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param url: The URL to connect to.
        :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
        """
        proxies = proxies or {}
        proxy = proxies.get(urlparse(url.lower()).scheme)

        if proxy:
            proxy_headers = self.proxy_headers(proxy)

            if proxy not in self.proxy_manager:
                self.proxy_manager[proxy] = proxy_from_url(
                    proxy,
                    proxy_headers=proxy_headers,
                    num_pools=self._pool_connections,
                    maxsize=self._pool_maxsize,
                    block=self._pool_block
                )

            conn = self.proxy_manager[proxy].connection_from_url(url)
        else:
            # Only scheme should be lower case
            parsed = urlparse(url)
            url = parsed.geturl()
            conn = self.poolmanager.connection_from_url(url)

        self.connections.append(conn)
        return conn 

def get_doc_from_url(target_url, timeout=5, max_retries=5):
    # 用来自定义头部的
    headers = {}
    # 用来传递参数的
    keyvalue = {}

    # 头部的填充
    headers['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14) ' \
                            'AppleWebKit/605.1.15 (KHTML, like Gecko) ' \
                            'Version/12.0 Safari/605.1.15'

    # 下面是参数的填充，参考图10
    keyvalue['m'] = 'QueryData'
    keyvalue['dbcode'] = 'hgnd'
    keyvalue['rowcode'] = 'zb'
    keyvalue['colcode'] = 'sj'
    keyvalue['wds'] = '[]'
    keyvalue['dfwds'] = '[{"wdcode":"zb","valuecode":"A0301"}]'
    keyvalue['k1'] = str(gettime())

    # 建立一个Session
    s = requests.session()
    s.mount('http://', HTTPAdapter(max_retries=max_retries))
    s.mount('https://', HTTPAdapter(max_retries=max_retries))
    # 在Session基础上进行一次请求
    r = s.get(target_url, params=keyvalue, headers=headers, timeout=timeout)

    data = r.content.decode('gbk')

    doc = pq(data)
    return doc 

def _session_factory():
    """Retry on errors from the API-server. Retry-After header will be respected first, which
    we expect to be set for too_many_requests, and for other errors it will back-off exponentially up
    to the 120s maximum"""
    session = requests.Session()
    retry_statuses = [requests.codes.too_many_requests,
                      requests.codes.internal_server_error,
                      requests.codes.bad_gateway,
                      requests.codes.service_unavailable,
                      requests.codes.gateway_timeout]
    retries = Retry(total=10, backoff_factor=1, status_forcelist=retry_statuses, method_whitelist=False)
    session.mount('http://', HTTPAdapter(max_retries=retries))
    session.mount('https://', HTTPAdapter(max_retries=retries))
    return session 

def _get_session_with_retries(self):
        session = requests.Session()
        retries = Retry(total=3, backoff_factor=1)
        session.mount('http://', HTTPAdapter(max_retries=retries))
        session.mount('https://', HTTPAdapter(max_retries=retries))
        return session