Python aiohttp.TCPConnector() Examples

def get_http_client(app):
    """ get http client """
    if "client" in app:
        return app["client"]

    # first time call, create client interface
    # use shared client so that all client requests
    #   will share the same connection pool
    if "loop" not in app:
        raise KeyError("loop not initialized")
    loop = app["loop"]
    max_tcp_connections = int(config.get("max_tcp_connections"))
    log.info(f"Initiating TCPConnector with limit {max_tcp_connections} connections")
    client = ClientSession(loop=loop, connector=TCPConnector(limit_per_host=max_tcp_connections))
    #create the app object
    app['client'] = client
    return client 

def start(self, host: str = "127.0.0.1", port: int = 8080) -> None:
        connector = None
        self.log.debug(f"Starting appservice web server on {host}:{port}")
        if self.server.startswith("https://") and not self.verify_ssl:
            connector = aiohttp.TCPConnector(verify_ssl=False)
        self._http_session = aiohttp.ClientSession(loop=self.loop, connector=connector)
        self._intent = AppServiceAPI(base_url=self.server, bot_mxid=self.bot_mxid, log=self.log,
                                     token=self.as_token, state_store=self.state_store,
                                     real_user_content_key=self.real_user_content_key,
                                     client_session=self._http_session).bot_intent()
        ssl_ctx = None
        if self.tls_cert and self.tls_key:
            ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
            ssl_ctx.load_cert_chain(self.tls_cert, self.tls_key)
        self.runner = web.AppRunner(self.app)
        await self.runner.setup()
        site = web.TCPSite(self.runner, host, port, ssl_context=ssl_ctx)
        await site.start() 

def test_secure_ok(aiohttp_client, aiohttp_server, ssl_ctx):
    async def handler(request):
        return web.Response()

    app = web.Application()
    app.router.add_get('/', handler)
    await _setup(app, Secure())
    srv = await aiohttp_server(app, ssl=ssl_ctx)
    conn = aiohttp.TCPConnector(ssl=False)
    cl = await aiohttp_client(srv, connector=conn)
    resp = await cl.get('/')
    print(resp.request_info.url)
    assert resp.status == 200
    assert resp.headers['X-Frame-Options'] == 'DENY'
    expected = 'max-age=31536000; includeSubDomains'
    assert resp.headers['Strict-Transport-Security'] == expected
    assert resp.headers['X-Content-Type-Options'] == 'nosniff'
    assert resp.headers['X-XSS-Protection'] == '1; mode=block' 

def test_connect_envvar(monkeypatch):
    monkeypatch.setenv("DOCKER_HOST", "unix:///var/run/does-not-exist-docker.sock")
    docker = Docker()
    assert isinstance(docker.connector, aiohttp.connector.UnixConnector)
    assert docker.docker_host == "unix://localhost"
    with pytest.raises(aiodocker.DockerError):
        await docker.containers.list()
    await docker.close()

    monkeypatch.setenv("DOCKER_HOST", "http://localhost:9999")
    docker = Docker()
    assert isinstance(docker.connector, aiohttp.TCPConnector)
    assert docker.docker_host == "http://localhost:9999"
    with pytest.raises(aiodocker.DockerError):
        await docker.containers.list()
    await docker.close() 

def _send_to_external_chat(self, bot, event, config):
        if event.from_bot:
            # don't send my own messages
            return

        event_timestamp = event.timestamp

        conversation_id = event.conv_id
        conversation_text = event.text

        user_full_name = event.user.full_name
        user_id = event.user_id

        url = config["HUBOT_URL"] + conversation_id
        payload = {"from" : str(user_id.chat_id), "message" : conversation_text}
        headers = {'content-type': 'application/json'}

        connector = aiohttp.TCPConnector(verify_ssl=False)
        asyncio.ensure_future(
            aiohttp.request('post', url, data = json.dumps(payload), headers = headers, connector=connector)
        ).add_done_callback(lambda future: future.result()) 

def cloudfare_session(loop):
    sessions = []

    async def go(**kwargs):
        fake = FakeCloudfare(**kwargs)
        info = await fake.start()
        resolver = FakeResolver(info, loop=asyncio.get_event_loop())
        connector = aiohttp.TCPConnector(resolver=resolver,
                                         ssl=False)

        session = aiohttp.ClientSession(connector=connector)
        sessions.append(session)
        return session

    yield go

    for s in sessions:
        loop.run_until_complete(s.close()) 

def test_connect(proxy,operator,mode=None):
    conn = aiohttp.TCPConnector(verify_ssl=False)
    async with ClientSession(connector=conn) as s:
        try:
            async with s.get(url=TEST_URL,proxy=proxy[2],
                             timeout=10,allow_redirects=False) as resp:
                page = await resp.text()
                if (resp.status != 200 or str(resp.url) != TEST_URL):
                    utils.log(('[INFO]#proxy:{ip} has been dropped\n'
                               '      #Reason:Abnormal url or return Code').format(ip=proxy[1]))
                    operator.del_proxy_with_id(config.free_ipproxy_table,proxy[0])
                    operator.del_proxy_with_id(config.httpbin_table,proxy[0])
                elif mode == 'add':
                    operator.insert_valid_proxy(id=proxy[0])
                else:
                    operator.update_valid_proxy(id=proxy[0])
                   
        except Exception as e:
            utils.log(('[INFO]#proxy:{ip} has been dropped\n'
                       '      #Reason:{msg}').format(ip=proxy[1],msg=str(e)))
            operator.del_proxy_with_id(config.free_ipproxy_table,proxy[0])
            operator.del_proxy_with_id(config.httpbin_table,proxy[0])
        finally:
            operator.commit() 

def test_secure_redirect(aiohttp_client, aiohttp_server, ssl_ctx):
    async def handler(request):
        return web.Response()

    app = web.Application()
    app.router.add_get('/', handler)
    secure = Secure()
    await _setup(app, secure)
    http_srv = await aiohttp_server(app)
    https_srv = await aiohttp_server(app, ssl=ssl_ctx)
    secure._redirect_url = https_srv.make_url('/')
    conn = aiohttp.TCPConnector(ssl=False)
    async with aiohttp.ClientSession(connector=conn) as cl:
        url = http_srv.make_url('/')
        resp = await cl.get(url)
        assert resp.status == 200
        assert resp.request_info.url.scheme == 'https' 

def _init_session(self):
        _safe = self.settings.SPIDER_COOKIES_UNSAFE_MODE
        path = self.settings.SPIDER_COOKIES_LOAD_PATH
        _c_cookies = self.settings.SPIDER_COOKIES_CUSTOM
        jar = aiohttp.CookieJar(unsafe=_safe)
        if _c_cookies:
            cookies = _c_cookies
        else:
            cookies = None
        self.conn = aiohttp.TCPConnector(limit=self.settings.CONCURRENCY)
        self.session = aiohttp.ClientSession(connector=self.conn,
                                             cookies=cookies,
                                             cookie_jar=jar)
        if path:
            if os.path.exists(path):
                try:
                    self.session.cookie_jar.load(path)
                    if cookies:
                        self.session.cookie_jar.update_cookies(cookies)
                except:
                    return
                self.logger.debug(f'Loaded [{self.name}] cookie jar.') 

def async_fetch(cls, urls, descs=None, cb=None, datas=None, fds=None):
        if descs is None:
            descs = []
        if datas is None:
            datas = []
        if fds is None:
            fds = []
        conn = aiohttp.TCPConnector(limit_per_host=cls.CONNECTIONS_PER_HOST)
        async with aiohttp.ClientSession(
                connector=conn,
                headers={'User-Agent': cls.USER_AGENT}
        ) as session:
            coros = [
                asyncio.ensure_future(cls._async_fetch_one(session, url, desc, cb, data, fd))
                for url, desc, data, fd in zip_longest(urls, descs, datas, fds)
            ]
            with tqdm(asyncio.as_completed(coros),
                      total=len(coros),
                      desc="Downloading", unit="files") as t:
                result = [await coro for coro in t]
        return result 

def test_connector_factory(es_params, loop):

    class TCPConnector(aiohttp.TCPConnector):
        used = False

        def __init__(self, *args, **kwargs):
            TCPConnector.used = True
            super(TCPConnector, self).__init__(*args, **kwargs)

    tr = Transport(
        endpoints=[{'host': es_params['host']}],
        sniffer_interval=None,
        loop=loop,
        connector_factory=lambda: TCPConnector(loop=loop)
    )
    assert 1 == len(tr._pool.connections)
    assert TCPConnector.used
    tr.close() 

def __awaitable__(self):
        if self._data is None:
            async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=self.api.verify_ssl)) as session:
                wait_time = self._wait_time()
                if wait_time is None and self.api:
                    try:
                        await self._make_async_request(session)
                    except ServiceUnavailableException:
                        await asyncio.sleep(60)
                        self._wait_time()
                        await self._make_async_request(session)
                else:
                    await asyncio.sleep(wait_time)
                    await self._make_async_request(session)

        return self 

def test_no_x_frame(aiohttp_client, aiohttp_server, ssl_ctx):
    async def handler(request):
        return web.Response()

    app = web.Application()
    app.router.add_get('/', handler)
    await _setup(app, Secure(x_frame=None))
    srv = await aiohttp_server(app, ssl=ssl_ctx)
    conn = aiohttp.TCPConnector(ssl=False)
    cl = await aiohttp_client(srv, connector=conn)
    resp = await cl.get('/')
    print(resp.request_info.url)
    assert resp.status == 200
    assert 'X-Frame-Options' not in resp.headers
    expected = 'max-age=31536000; includeSubDomains'
    assert resp.headers['Strict-Transport-Security'] == expected
    assert resp.headers['X-Content-Type-Options'] == 'nosniff'
    assert resp.headers['X-XSS-Protection'] == '1; mode=block' 

def test_no_sts(aiohttp_client, aiohttp_server, ssl_ctx):
    async def handler(request):
        return web.Response()

    app = web.Application()
    app.router.add_get('/', handler)
    await _setup(app, Secure(sts=None))
    srv = await aiohttp_server(app, ssl=ssl_ctx)
    conn = aiohttp.TCPConnector(ssl=False)
    cl = await aiohttp_client(srv, connector=conn)
    resp = await cl.get('/')
    print(resp.request_info.url)
    assert resp.status == 200
    assert resp.headers['X-Frame-Options'] == 'DENY'
    assert 'Strict-Transport-Security' not in resp.headers
    assert resp.headers['X-Content-Type-Options'] == 'nosniff'
    assert resp.headers['X-XSS-Protection'] == '1; mode=block' 

def _async_loop(self, urls):
        """Asynchronous internal method used to request multiple URLs

        Args:
            urls (list): URLs to fetch

        Returns:
            responses (obj): All URL requests' response coroutines

        """
        results = []
        async with aiohttp.ClientSession(
            connector=aiohttp.TCPConnector(ssl=False)
        ) as session:
            for url in urls:
                result = asyncio.ensure_future(self._get_async(url, session))
                results.append(result)
            responses = await asyncio.gather(*results)
        return responses 

def fetch_json(self, url, headers=None, error_dict=None):
        conn = aiohttp.TCPConnector(
            family=socket.AF_INET,
            verify_ssl=False,
        )
        data = dict()
        async with self.session.get(url, headers=headers) as resp:
            if resp.status == 200:
                data = await resp.json()
            elif error_dict and resp.status in error_dict.keys():
                for k, v in error_dict.items():
                    if resp.status == k:
                        raise v()
            else:
                raise UnknownServerError()

        return data 

def test_no_xss(aiohttp_client, aiohttp_server, ssl_ctx):
    async def handler(request):
        return web.Response()

    app = web.Application()
    app.router.add_get('/', handler)
    await _setup(app, Secure(xss=None))
    srv = await aiohttp_server(app, ssl=ssl_ctx)
    conn = aiohttp.TCPConnector(ssl=False)
    cl = await aiohttp_client(srv, connector=conn)
    resp = await cl.get('/')
    print(resp.request_info.url)
    assert resp.status == 200
    assert resp.headers['X-Frame-Options'] == 'DENY'
    expected = 'max-age=31536000; includeSubDomains'
    assert resp.headers['Strict-Transport-Security'] == expected
    assert resp.headers['X-Content-Type-Options'] == 'nosniff'
    assert 'X-XSS-Protection' not in resp.headers 

def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.http_session = ClientSession(
            connector=TCPConnector(resolver=AsyncResolver(), family=socket.AF_INET)
        )
        self._guild_available = asyncio.Event()

        self.loop.create_task(self.send_log("SeasonalBot", "Connected!")) 

def fetch_clans(self, tags):
        """Fetch clan info by tags."""
        conn = aiohttp.TCPConnector(
            family=socket.AF_INET,
            verify_ssl=False,
        )


        data_list = await asyncio.gather(*[self.fetch_clan(tag, self.session) for tag in tags])

        return data_list 

def _main(url, total_requests):
    connector = TCPConnector(limit=None)
    async with ClientSession(connector=connector) as session, TaskPool(limit) as tasks:

        for i in range(total_requests):
            await tasks.put(fetch(url.format(i), session)) 

def main():
    connector = TCPConnector(limit=None)
    async with ClientSession(connector=connector) as session:
        coros = (fetch(url.format(i), session) for i in range(r))
        await print_when_done(coros) 

def main():
    connector = TCPConnector(limit=None)
    async with ClientSession(connector=connector) as session:
        await run(session, int(sys.argv[1])) 

def handle(self, *args, **options):
        report = Report()
        report.start()

        loop = asyncio.new_event_loop()
        asyncio.set_event_loop(loop)
        connector = aiohttp.TCPConnector(limit=settings.AIOHTTP_LIMIT, force_close=True, loop=loop)
        aiohttp_session = aiohttp.ClientSession(loop=loop, connector=connector)
        tasks = []

        crawler = Crawler(self, loop, aiohttp_session, report)

        for directory in MovieDirectory.objects.all():
            crawler.queue_update_tasks(directory, tasks)

        if tasks:
            loop.run_until_complete(asyncio.wait(tasks))

        aiohttp_session.close()
        loop.close()

        # Delete movies with no path. Those entries are made possible since
        # movies can be saved in the resolvers.
        Movie.objects.filter(path="").delete()

        report.display() 

def __init__(self, url, timeout=None, username="", password="",
                 loop=None, future_class=None, connector=None):
        future_class = functools.partial(asyncio.Future, loop=loop)
        super().__init__(url, timeout=timeout, username=username,
                         password=password, loop=loop,
                         future_class=future_class)
        if connector is None:
            connector = aiohttp.TCPConnector(loop=self._loop)
        self._connector = connector 

def create_tcp_connector(*args, **kwargs) -> aiohttp.TCPConnector:
    """
    Creates TCP connector with resonable defaults.
    For details about available parameters refer to
    `aiohttp.TCPConnector <https://docs.aiohttp.org/en/stable/client_reference.html#tcpconnector>`_
    """
    ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
    ssl_context.load_verify_locations(certifi.where())
    kwargs.setdefault("ssl", ssl_context)
    kwargs.setdefault("limit", DEFAULT_LIMIT)
    # due to https://github.com/python/mypy/issues/4001
    return aiohttp.TCPConnector(*args, **kwargs)  # type: ignore 

def main():
    # pylint: disable=global-statement
    global CATEGORIES
    semaphore = asyncio.BoundedSemaphore(args.limit)
    async with aiohttp.ClientSession() as session:
        async with aiohttp.ClientSession(
                connector=aiohttp.TCPConnector(
                    ssl=args.do_not_verify_ssl)) as session:
            CATEGORIES = await get_existing_categories(
                session, args.url, args.username, args.password, semaphore)
            await upload_scripts(session, args.url, args.username,
                                 args.password, semaphore)
            await upload_extension_attributes(session, args.url, args.username,
                                              args.password, semaphore) 

def submit_slurp(self, data):
        try:
            async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False)) as session:
                r = await session.post(
                    'https://{0}:8080/api?auth={1}&chan=snare_test&msg={2}'.format(
                        self.run_args.slurp_host, self.run_args.slurp_auth, data
                    ), json=data, timeout=10.0
                )
                assert r.status == 200
                r.close()
        except Exception as e:
            self.logger.error('Error submitting slurp: %s', e) 

def __init__(
        self,
        token=None,
        timeout=slacker.DEFAULT_TIMEOUT,
        *,
        loop=None
    ):
        if loop is None:
            loop = asyncio.get_event_loop()

        self.loop = loop

        self.session = aiohttp.ClientSession(
            connector=aiohttp.TCPConnector(
                use_dns_cache=False,
                loop=self.loop,
            ),
        )

        self.methods = {
            requests.get: 'GET',
            requests.post: 'POST',
        }

        self.futs = set()

        super().__init__(token=token, timeout=timeout) 

def aiohttp_client_session(*, timeout=None, resolver=None, **kwargs):
    """use aiodns and support number timeout"""
    if timeout is None:
        timeout = 30
    if isinstance(timeout, (int, float)):
        timeout = aiohttp.ClientTimeout(total=timeout)
    if resolver is None:
        resolver = aiohttp.AsyncResolver()
    # Fix: No route to host. https://github.com/saghul/aiodns/issues/22
    family = socket.AF_INET
    connector = aiohttp.TCPConnector(resolver=resolver, family=family)
    return aiohttp.ClientSession(connector=connector, timeout=timeout, **kwargs) 

def startup(self) -> None:
        """Initialize the crawler manager's redis connection and
        http session used to make requests to shepherd
        """
        self.loop = get_event_loop()
        self.redis = await init_redis(
            env('REDIS_URL', default=DEFAULT_REDIS_URL), self.loop
        )
        self.session = ClientSession(
            connector=TCPConnector(
                resolver=AsyncResolver(loop=self.loop), loop=self.loop
            ),
            json_serialize=partial(json.dumps, ensure_ascii=False),
            loop=self.loop,
        )