Python urllib3.Timeout() Examples

def __http_pool(self):
        """
        Create HTTP connection pool
        :raise HttpRequestError
        :return: urllib3.HTTPConnectionPool
        """

        try:
            pool = HTTPConnectionPool(self.__cfg.host,
                                      port=self.__cfg.port,
                                      maxsize=self.__cfg.threads,
                                      timeout=Timeout(self.__cfg.timeout, read=self.__cfg.timeout),
                                      block=True)
            if self._HTTP_DBG_LEVEL <= self.__debug.level:
                self.__debug.debug_connection_pool('http_pool_start', pool)
            return pool
        except Exception as error:
            raise HttpRequestError(str(error)) 

def __connect(self):
        num_pools = float(self.pool_size_total) / self.pool_size_per_route
        headers = {
            'User-Agent': 'python-pilosa/%s' % VERSION,
        }

        timeout = urllib3.Timeout(connect=self.connect_timeout, read=self.socket_timeout)
        client_options = {
            "num_pools": num_pools,
            "maxsize": self.pool_size_per_route,
            "block": True,
            "headers": headers,
            "timeout": timeout,
            "retries": self.retry_count,
        }
        if not self.tls_skip_verify:
            client_options["cert_reqs"] = "CERT_REQUIRED"
            client_options["ca_certs"] = self.tls_ca_certificate_path

        client = urllib3.PoolManager(**client_options)
        self.__client = client 

def __https_pool(self):
        """
        Create HTTP connection pool
        :raise HttpsRequestError
        :return: urllib3.HTTPConnectionPool
        """

        try:
            pool = HTTPSConnectionPool(
                    host=self.__cfg.host,
                    port=self.__cfg.port,
                    maxsize=self.__cfg.threads,
                    timeout=Timeout(self.__cfg.timeout, read=self.__cfg.timeout),
                    block=True)
            if self._HTTP_DBG_LEVEL <= self.__debug.level:
                self.__debug.debug_connection_pool('https_pool_start', pool)

            return pool
        except Exception as error:
            raise HttpsRequestError(str(error)) 

def _conn_request(self, conn, request_uri, method, body, headers):
        full_uri = self._create_full_uri(conn, request_uri)

        decode = True if method != 'HEAD' else False

        try:
            urllib3_response = self.pool.request(
                method,
                full_uri,
                body=body,
                headers=headers,
                redirect=False,
                retries=urllib3.Retry(total=False, redirect=0),
                timeout=urllib3.Timeout(total=self.timeout),
                decode_content=decode)

            response = _map_response(urllib3_response, decode=decode)
            content = urllib3_response.data

        except Exception as e:
            raise _map_exception(e)

        return response, content 

def GetExternalIP(self, proxy=None): 
        timeout = self.timeout.get()
        
        file = "Data/tordata%s/ip.txt" % (proxy-9050)
        url = 'http://checkip.amazonaws.com'
        interval = int(self.time.get()-1)
        soc = SOCKSProxyManager('socks5://127.0.0.1:%s/' % proxy)
        s = soc.request('GET', url, timeout=urllib3.Timeout(connect=interval, read=interval), retries=False)
        d = s.data.decode('utf-8').split('\n')
        f = open(file, 'w')
        f.write(d[0])
        f.close()
        s.close()
        
        
    #funkce na zjisteni odezvy pro zjistenou IP adresu, nejdrive zavola zjisteni IP a nasledne provede overeni odezvy, a vypise vystup do aplikace 

def download_http(url, local_path, expected_size_in_bytes=None, progress_indicator=None):
    with __http().request("GET", url, preload_content=False, retries=10,
                          timeout=urllib3.Timeout(connect=45, read=240)) as r, open(local_path, "wb") as out_file:
        if r.status > 299:
            raise urllib.error.HTTPError(url, r.status, "", None, None)
        # noinspection PyBroadException
        try:
            size_from_content_header = int(r.getheader("Content-Length"))
            if expected_size_in_bytes is None:
                expected_size_in_bytes = size_from_content_header
        except BaseException:
            size_from_content_header = None

        chunk_size = 2 ** 16
        bytes_read = 0

        for chunk in r.stream(chunk_size):
            out_file.write(chunk)
            bytes_read += len(chunk)
            if progress_indicator and size_from_content_header:
                progress_indicator(bytes_read, size_from_content_header)
        return expected_size_in_bytes 

def __init__(self,
                 verify=True,
                 proxies=None,
                 timeout=None,
                 max_pool_connections=MAX_POOL_CONNECTIONS,
                 socket_options=None,
                 client_cert=None,
    ):
        self._verify = verify
        self._proxy_config = ProxyConfiguration(proxies=proxies)
        self._pool_classes_by_scheme = {
            'http': botocore.awsrequest.AWSHTTPConnectionPool,
            'https': botocore.awsrequest.AWSHTTPSConnectionPool,
        }
        if timeout is None:
            timeout = DEFAULT_TIMEOUT
        if not isinstance(timeout, (int, float)):
            timeout = Timeout(connect=timeout[0], read=timeout[1])

        self._cert_file = None
        self._key_file = None
        if isinstance(client_cert, str):
            self._cert_file = client_cert
        elif isinstance(client_cert, tuple):
            self._cert_file, self._key_file = client_cert

        self._timeout = timeout
        self._max_pool_connections = max_pool_connections
        self._socket_options = socket_options
        if socket_options is None:
            self._socket_options = []
        self._proxy_managers = {}
        self._manager = PoolManager(**self._get_pool_manager_kwargs())
        self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme 

def _build_request_parameters(self, etcd_nodes, timeout=None):
        kwargs = {'headers': self._get_headers(), 'redirect': self.allow_redirect}

        if timeout is not None:
            kwargs.update(retries=0, timeout=timeout)
        else:
            _, per_node_timeout, per_node_retries = self._calculate_timeouts(etcd_nodes)
            connect_timeout = max(1, per_node_timeout/2)
            kwargs.update(timeout=Timeout(connect=connect_timeout, total=per_node_timeout), retries=per_node_retries)
        return kwargs 

def __init__(self, token, endpoint=constants.DEFAULT_STREAM_ENDPOINT,
                 timeout=constants.DEFAULT_TIMEOUT, compress=True,
                 proxy_url=None):
        super(SSETransport, self).__init__(token, endpoint, timeout)
        pool_args = {
            'url': self._endpoint,
            'headers': {
                'Content-Type': 'text/plain',
                'X-SF-Token': self._token,
                'User-Agent': '{} urllib3/{}'.format(version.user_agent,
                                                     urllib3.__version__)
            },
            'timeout': urllib3.Timeout(connect=self._timeout, read=None),
        }

        if urllib3.util.parse_url(self._endpoint).scheme == 'https':
            pool_args.update({
                'cert_reqs': 'CERT_REQUIRED',  # Force certificate check.
                'ca_certs': certifi.where()    # Path to the Certifi bundle.
            })

        if proxy_url:
            proxy_manager = urllib3.poolmanager.proxy_from_url(proxy_url)
            endpoint = pool_args.pop('url')
            self._http = proxy_manager.connection_from_url(
                    endpoint, pool_kwargs=pool_args)
        else:
            self._http = urllib3.connectionpool.connection_from_url(
                    **pool_args) 

def __init__(self,
                 verify=True,
                 proxies=None,
                 timeout=None,
                 max_pool_connections=MAX_POOL_CONNECTIONS,
                 socket_options=None,
                 client_cert=None,
    ):
        self._verify = verify
        self._proxy_config = ProxyConfiguration(proxies=proxies)
        self._pool_classes_by_scheme = {
            'http': botocore.awsrequest.AWSHTTPConnectionPool,
            'https': botocore.awsrequest.AWSHTTPSConnectionPool,
        }
        if timeout is None:
            timeout = DEFAULT_TIMEOUT
        if not isinstance(timeout, (int, float)):
            timeout = Timeout(connect=timeout[0], read=timeout[1])

        self._cert_file = None
        self._key_file = None
        if isinstance(client_cert, str):
            self._cert_file = client_cert
        elif isinstance(client_cert, tuple):
            self._cert_file, self._key_file = client_cert

        self._timeout = timeout
        self._max_pool_connections = max_pool_connections
        self._socket_options = socket_options
        if socket_options is None:
            self._socket_options = []
        self._proxy_managers = {}
        self._manager = PoolManager(**self._get_pool_manager_kwargs())
        self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme 

def __init__(self,
                 verify=True,
                 proxies=None,
                 timeout=None,
                 max_pool_connections=MAX_POOL_CONNECTIONS,
                 socket_options=None,
                 client_cert=None,
    ):
        self._verify = verify
        self._proxy_config = ProxyConfiguration(proxies=proxies)
        self._pool_classes_by_scheme = {
            'http': botocore.awsrequest.AWSHTTPConnectionPool,
            'https': botocore.awsrequest.AWSHTTPSConnectionPool,
        }
        if timeout is None:
            timeout = DEFAULT_TIMEOUT
        if not isinstance(timeout, (int, float)):
            timeout = Timeout(connect=timeout[0], read=timeout[1])

        self._cert_file = None
        self._key_file = None
        if isinstance(client_cert, str):
            self._cert_file = client_cert
        elif isinstance(client_cert, tuple):
            self._cert_file, self._key_file = client_cert

        self._timeout = timeout
        self._max_pool_connections = max_pool_connections
        self._socket_options = socket_options
        if socket_options is None:
            self._socket_options = []
        self._proxy_managers = {}
        self._manager = PoolManager(**self._get_pool_manager_kwargs())
        self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme 

def retrieve_content_as_string(url):
    with __http().request("GET", url, timeout=urllib3.Timeout(connect=45, read=240)) as response:
        return response.read().decode("utf-8") 

def __init__(self,
                 verify=True,
                 proxies=None,
                 timeout=None,
                 max_pool_connections=MAX_POOL_CONNECTIONS,
                 socket_options=None,
                 client_cert=None,
    ):
        self._verify = verify
        self._proxy_config = ProxyConfiguration(proxies=proxies)
        self._pool_classes_by_scheme = {
            'http': botocore.awsrequest.AWSHTTPConnectionPool,
            'https': botocore.awsrequest.AWSHTTPSConnectionPool,
        }
        if timeout is None:
            timeout = DEFAULT_TIMEOUT
        if not isinstance(timeout, (int, float)):
            timeout = Timeout(connect=timeout[0], read=timeout[1])

        self._cert_file = None
        self._key_file = None
        if isinstance(client_cert, str):
            self._cert_file = client_cert
        elif isinstance(client_cert, tuple):
            self._cert_file, self._key_file = client_cert

        self._timeout = timeout
        self._max_pool_connections = max_pool_connections
        self._socket_options = socket_options
        if socket_options is None:
            self._socket_options = []
        self._proxy_managers = {}
        self._manager = PoolManager(**self._get_pool_manager_kwargs())
        self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme 

def test_client_error(self):
        r = None
        with tracer.start_active_span('test'):
            try:
                r = self.http.request('GET', 'http://doesnotexist.asdf:5000/504',
                                      retries=False,
                                      timeout=urllib3.Timeout(connect=0.5, read=0.5))
            except Exception:
                pass

        spans = self.recorder.queued_spans()
        self.assertEqual(2, len(spans))

        urllib3_span = spans[0]
        test_span = spans[1]

        self.assertIsNone(r)

        # Parent relationships
        self.assertEqual(urllib3_span.p, test_span.s)

        # Same traceId
        traceId = test_span.t
        self.assertEqual(traceId, urllib3_span.t)

        self.assertEqual("test", test_span.data["sdk"]["name"])
        self.assertEqual("urllib3", urllib3_span.n)
        self.assertIsNone(urllib3_span.data["http"]["status"])
        self.assertEqual("http://doesnotexist.asdf:5000/504", urllib3_span.data["http"]["url"])
        self.assertEqual("GET", urllib3_span.data["http"]["method"])
        self.assertIsNotNone(urllib3_span.stack)
        self.assertTrue(type(urllib3_span.stack) is list)
        self.assertTrue(len(urllib3_span.stack) > 1)

        # Error logging
        self.assertIsNone(test_span.ec)
        self.assertEqual(1, urllib3_span.ec) 

def __proxy_pool(self):
        """
        Create Proxy connection pool
        :raise ProxyRequestError
        :return: urllib3.HTTPConnectionPool
        """

        try:

            self.__server = self.__cfg.proxy if True is self.__cfg.is_standalone_proxy else self.__get_random_proxy()

            if self.__get_proxy_type(self.__server) == 'socks':

                disable_warnings(InsecureRequestWarning)

                if not hasattr(self, '__pm'):

                    package_module = importlib.import_module('urllib3.contrib.socks')
                    self.__pm = getattr(package_module, 'SOCKSProxyManager')

                pool = self.__pm(self.__server,
                                 num_pools=self.__cfg.threads,
                                 timeout=Timeout(self.__cfg.timeout,
                                 read=self.__cfg.timeout),
                                 block=True)
            else:
                pool = ProxyManager(self.__server,
                                    num_pools=self.__cfg.threads,
                                    timeout=Timeout(self.__cfg.timeout, read=self.__cfg.timeout),
                                    block=True)
            return pool
        except (DependencyWarning, ProxySchemeUnknown, ImportError) as error:
            raise ProxyRequestError(error) 

def __init__(self,
                 verify=True,
                 proxies=None,
                 timeout=None,
                 max_pool_connections=MAX_POOL_CONNECTIONS,
                 socket_options=None,
                 client_cert=None,
    ):
        self._verify = verify
        self._proxy_config = ProxyConfiguration(proxies=proxies)
        self._pool_classes_by_scheme = {
            'http': botocore.awsrequest.AWSHTTPConnectionPool,
            'https': botocore.awsrequest.AWSHTTPSConnectionPool,
        }
        if timeout is None:
            timeout = DEFAULT_TIMEOUT
        if not isinstance(timeout, (int, float)):
            timeout = Timeout(connect=timeout[0], read=timeout[1])

        self._cert_file = None
        self._key_file = None
        if isinstance(client_cert, str):
            self._cert_file = client_cert
        elif isinstance(client_cert, tuple):
            self._cert_file, self._key_file = client_cert

        self._timeout = timeout
        self._max_pool_connections = max_pool_connections
        self._socket_options = socket_options
        if socket_options is None:
            self._socket_options = []
        self._proxy_managers = {}
        self._manager = PoolManager(**self._get_pool_manager_kwargs())
        self._manager.pool_classes_by_scheme = self._pool_classes_by_scheme 

def api_execute(self, path, method, params=None, timeout=None):
        if not path.startswith('/'):
            raise ValueError('Path does not start with /')

        retry = params.pop('retry', None) if isinstance(params, dict) else None
        kwargs = {'fields': params, 'preload_content': False}

        if method in [self._MGET, self._MDELETE]:
            request_executor = self.http.request
        elif method in [self._MPUT, self._MPOST]:
            request_executor = self.http.request_encode_body
            kwargs['encode_multipart'] = False
        else:
            raise etcd.EtcdException('HTTP method {0} not supported'.format(method))

        # Update machines_cache if previous attempt of update has failed
        if self._update_machines_cache:
            self._load_machines_cache()
        elif not self._use_proxies and time.time() - self._machines_cache_updated > self._machines_cache_ttl:
            self._refresh_machines_cache()

        machines_cache = self.machines_cache
        etcd_nodes = len(machines_cache)
        kwargs.update(self._build_request_parameters(etcd_nodes, timeout))

        while True:
            try:
                response = self._do_http_request(retry, machines_cache, request_executor, method, path, **kwargs)
                return self._handle_server_response(response)
            except etcd.EtcdWatchTimedOut:
                raise
            except etcd.EtcdConnectionFailed as ex:
                try:
                    if self._load_machines_cache():
                        machines_cache = self.machines_cache
                        etcd_nodes = len(machines_cache)
                except Exception as e:
                    logger.debug('Failed to update list of etcd nodes: %r', e)
                sleeptime = retry.sleeptime
                remaining_time = retry.stoptime - sleeptime - time.time()
                nodes, timeout, retries = self._calculate_timeouts(etcd_nodes, remaining_time)
                if nodes == 0:
                    self._update_machines_cache = True
                    raise ex
                retry.sleep_func(sleeptime)
                retry.update_delay()
                # We still have some time left. Partially reduce `machines_cache` and retry request
                kwargs.update(timeout=Timeout(connect=max(1, timeout/2), total=timeout), retries=retries)
                machines_cache = machines_cache[:nodes]