Python urllib2.parse() Examples

The following are 12 code examples of urllib2.parse(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module urllib2 , or try the search function .
Example #1
Source File: http.py    From thriftpy2 with MIT License 6 votes vote down vote up
def __init__(self, uri, timeout=None, ssl_context_factory=None):
        """Initialize a HTTP Socket.

        @param uri(str)    The http_scheme:://host:port/path to connect to.
        @param timeout   timeout in ms
        """
        parsed = urllib.parse.urlparse(uri)
        self.scheme = parsed.scheme
        assert self.scheme in ('http', 'https')
        if self.scheme == 'http':
            self.port = parsed.port or http_client.HTTP_PORT
        elif self.scheme == 'https':
            self.port = parsed.port or http_client.HTTPS_PORT
        self.host = parsed.hostname
        self.path = parsed.path
        if parsed.query:
            self.path += '?%s' % parsed.query
        self.__wbuf = BytesIO()
        self.__http = None
        self.__custom_headers = None
        self.__timeout = None
        if timeout:
            self.setTimeout(timeout)
        self._ssl_context_factory = ssl_context_factory 
Example #2
Source File: http.py    From thriftpy2 with MIT License 6 votes vote down vote up
def make_client(service, host='localhost', port=9090, path='', scheme='http',
                proto_factory=TBinaryProtocolFactory(),
                trans_factory=TBufferedTransportFactory(),
                ssl_context_factory=None,
                timeout=DEFAULT_HTTP_CLIENT_TIMEOUT_MS, url=''):
    if url:
        parsed_url = urllib.parse.urlparse(url)
        host = parsed_url.hostname or host
        port = parsed_url.port or port
        scheme = parsed_url.scheme or scheme
        path = parsed_url.path or path
    uri = HTTP_URI.format(scheme=scheme, host=host, port=port, path=path)
    http_socket = THttpClient(uri, timeout, ssl_context_factory)
    transport = trans_factory.get_transport(http_socket)
    iprot = proto_factory.get_protocol(transport)
    transport.open()
    return TClient(service, iprot) 
Example #3
Source File: tornado.py    From thriftpy2 with MIT License 6 votes vote down vote up
def make_client(
        service, host='localhost', port=9090, proto_factory=TBinaryProtocolFactory(),
        io_loop=None, ssl_options=None,
        connect_timeout=TTornadoStreamTransport.DEFAULT_CONNECT_TIMEOUT,
        read_timeout=TTornadoStreamTransport.DEFAULT_READ_TIMEOUT,
        url=''):
    if url:
        parsed_url = urllib.parse.urlparse(url)
        host = parsed_url.hostname or host
        port = parsed_url.port or port
    transport = TTornadoStreamTransport(
        host, port, io_loop=io_loop,
        ssl_options=ssl_options, read_timeout=read_timeout)
    iprot = proto_factory.get_protocol(TMemoryBuffer())
    oprot = proto_factory.get_protocol(transport)
    yield transport.open(connect_timeout)
    client = TTornadoClient(service, iprot, oprot)
    raise gen.Return(client) 
Example #4
Source File: gen_rst.py    From sklearn-theano with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def identify_names(code):
    """Builds a codeobj summary by identifying and resovles used names

    >>> code = '''
    ... from a.b import c
    ... import d as e
    ... print(c)
    ... e.HelloWorld().f.g
    ... '''
    >>> for name, o in sorted(identify_names(code).items()):
    ...     print(name, o['name'], o['module'], o['module_short'])
    c c a.b a.b
    e.HelloWorld HelloWorld d d
    """
    finder = NameFinder()
    finder.visit(ast.parse(code))

    example_code_obj = {}
    for name, full_name in finder.get_mapping():
        # name is as written in file (e.g. np.asarray)
        # full_name includes resolved import path (e.g. numpy.asarray)
        module, attribute = full_name.rsplit('.', 1)
        # get shortened module name
        module_short = get_short_module_name(module, attribute)
        cobj = {'name': attribute, 'module': module,
                'module_short': module_short}
        example_code_obj[name] = cobj
    return example_code_obj 
Example #5
Source File: gen_rst.py    From tick with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def identify_names(code):
    """Builds a codeobj summary by identifying and resolves used names

    >>> code = '''
    ... from a.b import c
    ... import d as e
    ... print(c)
    ... e.HelloWorld().f.g
    ... '''
    >>> for name, o in sorted(identify_names(code).items()):
    ...     print(name, o['name'], o['module'], o['module_short'])
    c c a.b a.b
    e.HelloWorld HelloWorld d d
    """
    finder = NameFinder()
    finder.visit(ast.parse(code))

    example_code_obj = {}
    for name, full_name in finder.get_mapping():
        # name is as written in file (e.g. np.asarray)
        # full_name includes resolved import path (e.g. numpy.asarray)
        module, attribute = full_name.rsplit('.', 1)
        # get shortened module name
        module_short = get_short_module_name(module, attribute)
        cobj = {'name': attribute, 'module': module,
                'module_short': module_short}
        example_code_obj[name] = cobj
    return example_code_obj 
Example #6
Source File: docs_resolv.py    From spectrum with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def parse_sphinx_searchindex(searchindex):
    """Parse a Sphinx search index

    Parameters
    ----------
    searchindex : str
        The Sphinx search index (contents of searchindex.js)

    Returns
    -------
    filenames : list of str
        The file names parsed from the search index.
    objects : dict
        The objects parsed from the search index.
    """
    # Make sure searchindex uses UTF-8 encoding
    if hasattr(searchindex, 'decode'):
        searchindex = searchindex.decode('UTF-8')

    # parse objects
    query = 'objects:'
    pos = searchindex.find(query)
    if pos < 0:
        raise ValueError('"objects:" not found in search index')

    sel = _select_block(searchindex[pos:], '{', '}')
    objects = _parse_dict_recursive(sel)

    # parse filenames
    query = 'filenames:'
    pos = searchindex.find(query)
    if pos < 0:
        raise ValueError('"filenames:" not found in search index')
    filenames = searchindex[pos + len(query) + 1:]
    filenames = filenames[:filenames.find(']')]
    filenames = [f.strip('"') for f in filenames.split(',')]

    return filenames, objects 
Example #7
Source File: gen_rst.py    From tensorlib with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def identify_names(code):
    """Builds a codeobj summary by identifying and resovles used names

    >>> code = '''
    ... from a.b import c
    ... import d as e
    ... print(c)
    ... e.HelloWorld().f.g
    ... '''
    >>> for name, o in sorted(identify_names(code).items()):
    ...     print(name, o['name'], o['module'], o['module_short'])
    c c a.b a.b
    e.HelloWorld HelloWorld d d
    """
    finder = NameFinder()
    finder.visit(ast.parse(code))

    example_code_obj = {}
    for name, full_name in finder.get_mapping():
        # name is as written in file (e.g. np.asarray)
        # full_name includes resolved import path (e.g. numpy.asarray)
        module, attribute = full_name.rsplit('.', 1)
        # get shortened module name
        module_short = get_short_module_name(module, attribute)
        cobj = {'name': attribute, 'module': module,
                'module_short': module_short}
        example_code_obj[name] = cobj
    return example_code_obj 
Example #8
Source File: gen_rst.py    From polylearn with BSD 2-Clause "Simplified" License 5 votes vote down vote up
def identify_names(code):
    """Builds a codeobj summary by identifying and resovles used names

    >>> code = '''
    ... from a.b import c
    ... import d as e
    ... print(c)
    ... e.HelloWorld().f.g
    ... '''
    >>> for name, o in sorted(identify_names(code).items()):
    ...     print(name, o['name'], o['module'], o['module_short'])
    c c a.b a.b
    e.HelloWorld HelloWorld d d
    """
    finder = NameFinder()
    finder.visit(ast.parse(code))

    example_code_obj = {}
    for name, full_name in finder.get_mapping():
        # name is as written in file (e.g. np.asarray)
        # full_name includes resolved import path (e.g. numpy.asarray)
        module, attribute = full_name.rsplit('.', 1)
        # get shortened module name
        module_short = get_short_module_name(module, attribute)
        cobj = {'name': attribute, 'module': module,
                'module_short': module_short}
        example_code_obj[name] = cobj
    return example_code_obj 
Example #9
Source File: rpc.py    From thriftpy2 with MIT License 5 votes vote down vote up
def make_client(service, host="localhost", port=9090, unix_socket=None,
                proto_factory=TBinaryProtocolFactory(),
                trans_factory=TBufferedTransportFactory(),
                timeout=3000, cafile=None, ssl_context=None, certfile=None,
                keyfile=None, url="", socket_family=socket.AF_INET):
    if url:
        parsed_url = urllib.parse.urlparse(url)
        host = parsed_url.hostname or host
        port = parsed_url.port or port
    if unix_socket:
        socket = TSocket(unix_socket=unix_socket, socket_timeout=timeout)
        if certfile:
            warnings.warn("SSL only works with host:port, not unix_socket.")
    elif host and port:
        if cafile or ssl_context:
            socket = TSSLSocket(host, port, socket_timeout=timeout,
                                socket_family=socket_family, cafile=cafile,
                                certfile=certfile, keyfile=keyfile,
                                ssl_context=ssl_context)
        else:
            socket = TSocket(host, port, socket_family=socket_family, socket_timeout=timeout)
    else:
        raise ValueError("Either host/port or unix_socket or url must be provided.")

    transport = trans_factory.get_transport(socket)
    protocol = proto_factory.get_protocol(transport)
    transport.open()
    return TClient(service, protocol) 
Example #10
Source File: rpc.py    From thriftpy2 with MIT License 5 votes vote down vote up
def make_client(service, host='localhost', port=9090, unix_socket=None,
                proto_factory=TAsyncBinaryProtocolFactory(),
                trans_factory=TAsyncBufferedTransportFactory(),
                timeout=3000, connect_timeout=None,
                cafile=None, ssl_context=None,
                certfile=None, keyfile=None,
                validate=True, url='',
                socket_timeout=None):
    if socket_timeout is not None:
        warnings.warn(
            "The 'socket_timeout' argument is deprecated. "
            "Please use 'timeout' instead.",
            DeprecationWarning,
        )
        timeout = socket_timeout
    if url:
        parsed_url = urllib.parse.urlparse(url)
        host = parsed_url.hostname or host
        port = parsed_url.port or port
    if unix_socket:
        socket = TAsyncSocket(unix_socket=unix_socket,
                              connect_timeout=connect_timeout,
                              socket_timeout=timeout)
        if certfile:
            warnings.warn("SSL only works with host:port, not unix_socket.")
    elif host and port:
        socket = TAsyncSocket(
            host, port,
            socket_timeout=timeout, connect_timeout=connect_timeout,
            cafile=cafile, ssl_context=ssl_context,
            certfile=certfile, keyfile=keyfile, validate=validate)
    else:
        raise ValueError("Either host/port or unix_socket or url must be provided.")

    transport = trans_factory.get_transport(socket)
    protocol = proto_factory.get_protocol(transport)
    yield from transport.open()
    return TAsyncClient(service, protocol) 
Example #11
Source File: rpc.py    From thriftpy2 with MIT License 4 votes vote down vote up
def client_context(service, host="localhost", port=9090, unix_socket=None,
                   proto_factory=TBinaryProtocolFactory(),
                   trans_factory=TBufferedTransportFactory(),
                   timeout=None, socket_timeout=3000, connect_timeout=3000,
                   cafile=None, ssl_context=None, certfile=None, keyfile=None,
                   url=""):
    if url:
        parsed_url = urllib.parse.urlparse(url)
        host = parsed_url.hostname or host
        port = parsed_url.port or port

    if timeout:
        warnings.warn("`timeout` deprecated, use `socket_timeout` and "
                      "`connect_timeout` instead.")
        socket_timeout = connect_timeout = timeout

    if unix_socket:
        socket = TSocket(unix_socket=unix_socket,
                         connect_timeout=connect_timeout,
                         socket_timeout=socket_timeout)
        if certfile:
            warnings.warn("SSL only works with host:port, not unix_socket.")
    elif host and port:
        if cafile or ssl_context:
            socket = TSSLSocket(host, port,
                                connect_timeout=connect_timeout,
                                socket_timeout=socket_timeout,
                                cafile=cafile,
                                certfile=certfile, keyfile=keyfile,
                                ssl_context=ssl_context)
        else:
            socket = TSocket(host, port,
                             connect_timeout=connect_timeout,
                             socket_timeout=socket_timeout)
    else:
        raise ValueError("Either host/port or unix_socket or url must be provided.")

    try:
        transport = trans_factory.get_transport(socket)
        protocol = proto_factory.get_protocol(transport)
        transport.open()
        yield TClient(service, protocol)

    finally:
        transport.close() 
Example #12
Source File: http.py    From thriftpy2 with MIT License 4 votes vote down vote up
def flush(self):
        # Pull data out of buffer
        # Do this before opening a new connection in case there isn't data
        data = self.__wbuf.getvalue()
        self.__wbuf = BytesIO()
        if not data:  # No data to flush, ignore
            return

        if self.isOpen():
            self.close()
        self.open()

        # HTTP request
        self.__http.putrequest('POST', self.path, skip_host=True)

        # Write headers
        self.__http.putheader('Host', self.host)
        self.__http.putheader('Content-Type', 'application/x-thrift')
        self.__http.putheader('Content-Length', str(len(data)))

        if (not self.__custom_headers or
                'User-Agent' not in self.__custom_headers):
            user_agent = 'Python/THttpClient'
            script = os.path.basename(sys.argv[0])
            if script:
                user_agent = '%s (%s)' % (
                    user_agent, urllib.parse.quote(script))
                self.__http.putheader('User-Agent', user_agent)

        if self.__custom_headers:
            for key, val in self.__custom_headers.items():
                self.__http.putheader(key, val)

        self.__http.endheaders()

        # Write payload
        self.__http.send(data)

        # Get reply to flush the request
        response = self.__http.getresponse()
        self.code, self.message, self.headers = (
            response.status, response.msg, response.getheaders())
        self.response = response