Python urllib.splittype() Examples

The following are code examples for showing how to use urllib.splittype(). They are extracted from open source Python projects. You can vote up the examples you like or vote down the ones you don't like. You can also save this page to your account.

Example 1
Project: kinect-2-libras   Author: inessadl   File: xmlrpclib.py    (Apache License 2.0) View Source Project 6 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,
                 allow_none=0, use_datetime=0):
        # establish a "logical" server connection

        # get the url
        import urllib
        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError, "unsupported XML-RPC protocol"
        self.__host, self.__handler = urllib.splithost(uri)
        if not self.__handler:
            self.__handler = "/RPC2"

        if transport is None:
            if type == "https":
                transport = SafeTransport(use_datetime=use_datetime)
            else:
                transport = Transport(use_datetime=use_datetime)
        self.__transport = transport

        self.__encoding = encoding
        self.__verbose = verbose
        self.__allow_none = allow_none 
Example 2
Project: darkc0de-old-stuff   Author: tuwid   File: proxy.py    (GNU General Public License v3.0) View Source Project 6 votes vote down vote up
def request(self, method, url, body=None, headers={}):
        # Request is called before connect, so can interpret url and get
        # real host/port to be used to make CONNECT request to proxy
        proto, rest = urllib.splittype(url)

        if proto is None:
            raise ValueError, "unknown URL type: %s" % url

        # Get host
        host, rest = urllib.splithost(rest)

        # Try to get port
        host, port = urllib.splitport(host)

        # If port is not defined try to get from proto
        if port is None:
            try:
                port = self._ports[proto]
            except KeyError:
                raise ValueError, "unknown protocol for: %s" % url

        self._real_host = host
        self._real_port = int(port)

        httplib.HTTPConnection.request(self, method, url, body, headers) 
Example 3
Project: touch-pay-client   Author: HackPucBemobi   File: simplejsonrpc.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,version=None):
        self.location = uri             # server location (url)
        self.trace = verbose            # show debug messages
        self.exceptions = True          # raise errors? (JSONRPCError)
        self.timeout = None
        self.json_request = self.json_response = ''
        self.version = version          # '2.0' for jsonrpc2

        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError("unsupported JSON-RPC protocol")
        self.__host, self.__handler = urllib.splithost(uri)

        if transport is None:
            if type == "https":
                transport = JSONSafeTransport()
            else:
                transport = JSONTransport()
        self.__transport = transport
        self.__encoding = encoding
        self.__verbose = verbose 
Example 4
Project: true_review_web2py   Author: lucadealfaro   File: simplejsonrpc.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,version=None):
        self.location = uri             # server location (url)
        self.trace = verbose            # show debug messages
        self.exceptions = True          # raise errors? (JSONRPCError)
        self.timeout = None
        self.json_request = self.json_response = ''
        self.version = version          # '2.0' for jsonrpc2

        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError("unsupported JSON-RPC protocol")
        self.__host, self.__handler = urllib.splithost(uri)

        if transport is None:
            if type == "https":
                transport = JSONSafeTransport()
            else:
                transport = JSONTransport()
        self.__transport = transport
        self.__encoding = encoding
        self.__verbose = verbose 
Example 5
Project: spc   Author: whbrewer   File: simplejsonrpc.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,version=None):
        self.location = uri             # server location (url)
        self.trace = verbose            # show debug messages
        self.exceptions = True          # raise errors? (JSONRPCError)
        self.timeout = None
        self.json_request = self.json_response = ''
        self.version = version          # '2.0' for jsonrpc2

        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError("unsupported JSON-RPC protocol")
        self.__host, self.__handler = urllib.splithost(uri)

        if transport is None:
            if type == "https":
                transport = JSONSafeTransport()
            else:
                transport = JSONTransport()
        self.__transport = transport
        self.__encoding = encoding
        self.__verbose = verbose 
Example 6
Project: Problematica-public   Author: TechMaz   File: simplejsonrpc.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,version=None):
        self.location = uri             # server location (url)
        self.trace = verbose            # show debug messages
        self.exceptions = True          # raise errors? (JSONRPCError)
        self.timeout = None
        self.json_request = self.json_response = ''
        self.version = version          # '2.0' for jsonrpc2

        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError("unsupported JSON-RPC protocol")
        self.__host, self.__handler = urllib.splithost(uri)

        if transport is None:
            if type == "https":
                transport = JSONSafeTransport()
            else:
                transport = JSONTransport()
        self.__transport = transport
        self.__encoding = encoding
        self.__verbose = verbose 
Example 7
Project: specto   Author: mrknow   File: _pydev_xmlrpclib.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,
                 allow_none=0, use_datetime=0):
        # establish a "logical" server connection

        # get the url
        import urllib
        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError("unsupported XML-RPC protocol")
        self.__host, self.__handler = urllib.splithost(uri)
        if not self.__handler:
            self.__handler = "/RPC2"

        if transport is None:
            if type == "https":
                transport = SafeTransport(use_datetime=use_datetime)
            else:
                transport = Transport(use_datetime=use_datetime)
        self.__transport = transport

        self.__encoding = encoding
        self.__verbose = verbose
        self.__allow_none = allow_none 
Example 8
Project: rekall-agent-server   Author: rekall-innovations   File: simplejsonrpc.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,version=None):
        self.location = uri             # server location (url)
        self.trace = verbose            # show debug messages
        self.exceptions = True          # raise errors? (JSONRPCError)
        self.timeout = None
        self.json_request = self.json_response = ''
        self.version = version          # '2.0' for jsonrpc2

        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError("unsupported JSON-RPC protocol")
        self.__host, self.__handler = urllib.splithost(uri)

        if transport is None:
            if type == "https":
                transport = JSONSafeTransport()
            else:
                transport = JSONTransport()
        self.__transport = transport
        self.__encoding = encoding
        self.__verbose = verbose 
Example 9
Project: slugiot-client   Author: slugiot   File: simplejsonrpc.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,version=None):
        self.location = uri             # server location (url)
        self.trace = verbose            # show debug messages
        self.exceptions = True          # raise errors? (JSONRPCError)
        self.timeout = None
        self.json_request = self.json_response = ''
        self.version = version          # '2.0' for jsonrpc2

        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError("unsupported JSON-RPC protocol")
        self.__host, self.__handler = urllib.splithost(uri)

        if transport is None:
            if type == "https":
                transport = JSONSafeTransport()
            else:
                transport = JSONTransport()
        self.__transport = transport
        self.__encoding = encoding
        self.__verbose = verbose 
Example 10
Project: griffith   Author: Strit   File: PluginMovieMovieMeter.py    (license) View Source Project 6 votes vote down vote up
def make_connection(self, host):
        self.realhost = host
        proxies = urllib.getproxies()
        proxyurl = None
        if 'http' in proxies:
            proxyurl = proxies['http']
        elif 'all' in proxies:
            proxyurl = proxies['all']
        if proxyurl:
            urltype, proxyhost = urllib.splittype(proxyurl)
            host, selector = urllib.splithost(proxyhost)
            h = httplib.HTTP(host)
            self.proxy_is_used = True
            return h
        else:
            self.proxy_is_used = False
            return Transport.make_connection(self, host) 
Example 11
Project: watcher   Author: nosmokingbandit   File: rtorrent.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=False,
                 allow_none=False, use_datetime=False):
        type, uri = urllib.splittype(uri)
        if type not in ('scgi'):
            raise IOError('unsupported XML-RPC protocol')
        self.__host, self.__handler = urllib.splithost(uri)
        if not self.__handler:
            self.__handler = '/'
        
        if transport is None:
            transport = SCGITransport(use_datetime=use_datetime)
        self.__transport = transport
        
        self.__encoding = encoding
        self.__verbose = verbose
        self.__allow_none = allow_none 
Example 12
Project: w4py   Author: Cito   File: PickleRPC.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, uri, transport=None, verbose=False, binary=True,
            compressRequest=True, acceptCompressedResponse=True):
        """Establish a "logical" server connection."""
        # get the url
        import urllib
        typ, uri = urllib.splittype(uri)
        if typ not in ('http', 'https'):
            raise IOError('unsupported Pickle-RPC protocol')
        self._host, self._handler = urllib.splithost(uri)
        if not self._handler:
            self._handler = '/PickleRPC'

        if transport is None:
            transport = (SafeTransport if typ == 'https' else Transport)()
        self._transport = transport

        self._verbose = verbose
        self._binary = binary
        self._compressRequest = compressRequest
        self._acceptCompressedResponse = acceptCompressedResponse 
Example 13
Project: StuffShare   Author: StuffShare   File: simplejsonrpc.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,version=None):
        self.location = uri             # server location (url)
        self.trace = verbose            # show debug messages
        self.exceptions = True          # raise errors? (JSONRPCError)
        self.timeout = None
        self.json_request = self.json_response = ''
        self.version = version          # '2.0' for jsonrpc2

        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError("unsupported JSON-RPC protocol")
        self.__host, self.__handler = urllib.splithost(uri)

        if transport is None:
            if type == "https":
                transport = JSONSafeTransport()
            else:
                transport = JSONTransport()
        self.__transport = transport
        self.__encoding = encoding
        self.__verbose = verbose 
Example 14
Project: enigma2-plugins   Author: opendreambox   File: xmlrpclib.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,
                 allow_none=0, use_datetime=0):
        # establish a "logical" server connection

        # get the url
        import urllib
        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError, "unsupported XML-RPC protocol"
        self.__host, self.__handler = urllib.splithost(uri)
        if not self.__handler:
            self.__handler = "/RPC2"

        if transport is None:
            if type == "https":
                transport = SafeTransport(use_datetime=use_datetime)
            else:
                transport = Transport(use_datetime=use_datetime)
        self.__transport = transport

        self.__encoding = encoding
        self.__verbose = verbose
        self.__allow_none = allow_none 
Example 15
Project: kinect-2-libras   Author: inessadl   File: macurl2path.py    (Apache License 2.0) View Source Project 5 votes vote down vote up
def url2pathname(pathname):
    """OS-specific conversion from a relative URL of the 'file' scheme
    to a file system path; not recommended for general use."""
    #
    # XXXX The .. handling should be fixed...
    #
    tp = urllib.splittype(pathname)[0]
    if tp and tp != 'file':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    # Turn starting /// into /, an empty hostname means current host
    if pathname[:3] == '///':
        pathname = pathname[2:]
    elif pathname[:2] == '//':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    components = pathname.split('/')
    # Remove . and embedded ..
    i = 0
    while i < len(components):
        if components[i] == '.':
            del components[i]
        elif components[i] == '..' and i > 0 and \
                                  components[i-1] not in ('', '..'):
            del components[i-1:i+1]
            i = i-1
        elif components[i] == '' and i > 0 and components[i-1] != '':
            del components[i]
        else:
            i = i+1
    if not components[0]:
        # Absolute unix path, don't start with colon
        rv = ':'.join(components[1:])
    else:
        # relative unix path, start with colon. First replace
        # leading .. by empty strings (giving ::file)
        i = 0
        while i < len(components) and components[i] == '..':
            components[i] = ''
            i = i + 1
        rv = ':' + ':'.join(components)
    # and finally unquote slashes and other funny characters
    return urllib.unquote(rv) 
Example 16
Project: mongodb-monitoring   Author: jruaux   File: binding.py    (Apache License 2.0) View Source Project 5 votes vote down vote up
def _spliturl(url):
    scheme, opaque = urllib.splittype(url)
    netloc, path = urllib.splithost(opaque)
    host, port = urllib.splitport(netloc)
    # Strip brackets if its an IPv6 address
    if host.startswith('[') and host.endswith(']'): host = host[1:-1]
    if port is None: port = DEFAULT_PORT
    return scheme, host, port, path

# Given an HTTP request handler, this wrapper objects provides a related
# family of convenience methods built using that handler. 
Example 17
Project: code   Author: ActiveState   File: recipe-224043.py    (MIT License) View Source Project 5 votes vote down vote up
def getpage(self, url_pair):
        # Incoming argument name is a (URL, fragment) pair.
        # The page may have been cached in the name_table variable.
        url, fragment = url_pair
        if self.name_table.has_key(url):
            return self.name_table[url]

        scheme, path = urllib.splittype(url)
        if scheme in ('mailto', 'news', 'javascript', 'telnet'):
            self.note(1, " Not checking %s URL" % scheme)
            return None
        isint = self.inroots(url)

        # Ensure that openpage gets the URL pair to
        # print out its error message and record the error pair
        # correctly.
        if not isint:
            if not self.checkext:
                self.note(1, " Not checking ext link")
                return None
            f = self.openpage(url_pair)
            if f:
                self.safeclose(f)
            return None
        text, nurl = self.readhtml(url_pair)

        if nurl != url:
            self.note(1, " Redirected to %s", nurl)
            url = nurl
        if text:
            return Page(text, url, maxpage=self.maxpage, checker=self)

    # These next three functions take (URL, fragment) pairs as
    # arguments, so that openpage() receives the appropriate tuple to
    # record error messages. 
Example 18
Project: code   Author: ActiveState   File: recipe-224043.py    (MIT License) View Source Project 5 votes vote down vote up
def savefilename(self, url):
        type, rest = urllib.splittype(url)
        host, path = urllib.splithost(rest)
        path = path.lstrip("/")
        user, host = urllib.splituser(host)
        host, port = urllib.splitnport(host)
        host = host.lower()
        if not path or path[-1] == "/":
            path = path + "index.html"
        if os.sep != "/":
            path = os.sep.join(path.split("/"))
            if os.name == "mac":
                path = os.sep + path
        path = os.path.join(host, path)
        return path 
Example 19
Project: Splunk_CBER_App   Author: MHaggis   File: binding.py    (license) View Source Project 5 votes vote down vote up
def _spliturl(url):
    scheme, opaque = urllib.splittype(url)
    netloc, path = urllib.splithost(opaque)
    host, port = urllib.splitport(netloc)
    # Strip brackets if its an IPv6 address
    if host.startswith('[') and host.endswith(']'): host = host[1:-1]
    if port is None: port = DEFAULT_PORT
    return scheme, host, port, path

# Given an HTTP request handler, this wrapper objects provides a related
# family of convenience methods built using that handler. 
Example 20
Project: hostapd-mana   Author: adde88   File: xmlrpclib.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,
                 allow_none=0, use_datetime=0, context=None):
        # establish a "logical" server connection

        if isinstance(uri, unicode):
            uri = uri.encode('ISO-8859-1')

        # get the url
        import urllib
        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError, "unsupported XML-RPC protocol"
        self.__host, self.__handler = urllib.splithost(uri)
        if not self.__handler:
            self.__handler = "/RPC2"

        if transport is None:
            if type == "https":
                transport = SafeTransport(use_datetime=use_datetime, context=context)
            else:
                transport = Transport(use_datetime=use_datetime)
        self.__transport = transport

        self.__encoding = encoding
        self.__verbose = verbose
        self.__allow_none = allow_none 
Example 21
Project: hostapd-mana   Author: adde88   File: macurl2path.py    (license) View Source Project 5 votes vote down vote up
def url2pathname(pathname):
    """OS-specific conversion from a relative URL of the 'file' scheme
    to a file system path; not recommended for general use."""
    #
    # XXXX The .. handling should be fixed...
    #
    tp = urllib.splittype(pathname)[0]
    if tp and tp != 'file':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    # Turn starting /// into /, an empty hostname means current host
    if pathname[:3] == '///':
        pathname = pathname[2:]
    elif pathname[:2] == '//':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    components = pathname.split('/')
    # Remove . and embedded ..
    i = 0
    while i < len(components):
        if components[i] == '.':
            del components[i]
        elif components[i] == '..' and i > 0 and \
                                  components[i-1] not in ('', '..'):
            del components[i-1:i+1]
            i = i-1
        elif components[i] == '' and i > 0 and components[i-1] != '':
            del components[i]
        else:
            i = i+1
    if not components[0]:
        # Absolute unix path, don't start with colon
        rv = ':'.join(components[1:])
    else:
        # relative unix path, start with colon. First replace
        # leading .. by empty strings (giving ::file)
        i = 0
        while i < len(components) and components[i] == '..':
            components[i] = ''
            i = i + 1
        rv = ':' + ':'.join(components)
    # and finally unquote slashes and other funny characters
    return urllib.unquote(rv) 
Example 22
Project: p2pool-bch   Author: amarian12   File: Client.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, url, config = Config):
        proto, uri = urllib.splittype(url)

        # apply some defaults
        if uri[0:2] != '//':
            if proto != None:
                uri = proto + ':' + uri

            uri = '//' + uri
            proto = 'http'

        host, path = urllib.splithost(uri)

        try:
            int(host)
            host = 'localhost:' + host
        except:
            pass

        if not path:
            path = '/'

        if proto not in ('http', 'https', 'httpg'):
            raise IOError, "unsupported SOAP protocol"
        if proto == 'httpg' and not config.GSIclient:
            raise AttributeError, \
                  "GSI client not supported by this Python installation"
        if proto == 'https' and not config.SSLclient:
            raise AttributeError, \
                "SSL client not supported by this Python installation"

        self.user,host = urllib.splituser(host)
        self.proto = proto
        self.host = host
        self.path = path 
Example 23
Project: web_crawler   Author: NearXdu   File: scrapy_qq.py    (license) View Source Project 5 votes vote down vote up
def parse(self, response):
        def getdomain(url):
            proto, rest = urllib.splittype(url)
            host, rest = urllib.splithost(rest)
            return "http://"+host

        sel=scrapy.Selector(response)
        links_in_a_page = sel.xpath('//a[@href]')

        for link_sel in links_in_a_page:
            item=QqurlItem()
            link=str(link_sel.re('href="(.*?)"')[0])
            
            if link:
                if not link.startswith('http'):
                    if link.startswith('javascript'):
                        continue
                    if link.startswith('//support'):
                        continue
                    link=getdomain(response.url)+link


                if  re.match('.*comment.*',link):
					continue


                yield scrapy.Request(link,callback=self.parse)
                if not re.match('.*comment.*',link):
                    if re.match('^http.*qq.com.*\.s?html?$',link):
                        item['link']=link
                        yield item 
Example 24
Project: web_crawler   Author: NearXdu   File: scrapy_sohu.py    (license) View Source Project 5 votes vote down vote up
def parse(self, response):
        def getdomain(url):
            #proto,rest=urllib.splittype(url)
            #host,rest=urllib.splithost(rest)
            return "http:"

        sel =  scrapy.Selector(response)
        links_in_a_page=sel.xpath('//a[@href]')

        for link_sel in links_in_a_page:
            item=SohuItem()
            link=str(link_sel.re('href="(.*?)"')[0])

            if link:
                if not link.startswith('http'):
                    link=getdomain(response.url)+link

                yield scrapy.Request(link,callback=self.parse)

                p1=re.compile(r'.*/a/.*')
                p2=re.compile(r'.*#comment_area$')
                p3=re.compile(r'.*news.sohu.com.*s?html?$')

				

                if (re.match(p3,link) or re.match(p1,link)) and (not re.match(p2,link)):
                    #print ('T: '+link)
                    item['link']=link
                    yield item
                else:
                    pass
                    #print ('F: '+link) 
Example 25
Project: api-gateway-demo-sign-python   Author: aliyun   File: response.py    (license) View Source Project 5 votes vote down vote up
def parse_host(self):
        proto, rest = urllib.splittype(self.get_host())
        host, rest = urllib.splithost(rest)
        host, port = urllib.splitport(host)
        return host 
Example 26
Project: p2pool-unitus   Author: amarian12   File: Client.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, url, config = Config):
        proto, uri = urllib.splittype(url)

        # apply some defaults
        if uri[0:2] != '//':
            if proto != None:
                uri = proto + ':' + uri

            uri = '//' + uri
            proto = 'http'

        host, path = urllib.splithost(uri)

        try:
            int(host)
            host = 'localhost:' + host
        except:
            pass

        if not path:
            path = '/'

        if proto not in ('http', 'https', 'httpg'):
            raise IOError, "unsupported SOAP protocol"
        if proto == 'httpg' and not config.GSIclient:
            raise AttributeError, \
                  "GSI client not supported by this Python installation"
        if proto == 'https' and not config.SSLclient:
            raise AttributeError, \
                "SSL client not supported by this Python installation"

        self.user,host = urllib.splituser(host)
        self.proto = proto
        self.host = host
        self.path = path 
Example 27
Project: super_simple_siem   Author: elucidant   File: binding.py    (license) View Source Project 5 votes vote down vote up
def _spliturl(url):
    scheme, opaque = urllib.splittype(url)
    netloc, path = urllib.splithost(opaque)
    host, port = urllib.splitport(netloc)
    # Strip brackets if its an IPv6 address
    if host.startswith('[') and host.endswith(']'): host = host[1:-1]
    if port is None: port = DEFAULT_PORT
    return scheme, host, port, path

# Given an HTTP request handler, this wrapper objects provides a related
# family of convenience methods built using that handler. 
Example 28
Project: p2pool-dgb-sha256   Author: ilsawa   File: Client.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, url, config = Config):
        proto, uri = urllib.splittype(url)

        # apply some defaults
        if uri[0:2] != '//':
            if proto != None:
                uri = proto + ':' + uri

            uri = '//' + uri
            proto = 'http'

        host, path = urllib.splithost(uri)

        try:
            int(host)
            host = 'localhost:' + host
        except:
            pass

        if not path:
            path = '/'

        if proto not in ('http', 'https', 'httpg'):
            raise IOError, "unsupported SOAP protocol"
        if proto == 'httpg' and not config.GSIclient:
            raise AttributeError, \
                  "GSI client not supported by this Python installation"
        if proto == 'https' and not config.SSLclient:
            raise AttributeError, \
                "SSL client not supported by this Python installation"

        self.user,host = urllib.splituser(host)
        self.proto = proto
        self.host = host
        self.path = path 
Example 29
Project: oil   Author: oilshell   File: test_urllib.py    (license) View Source Project 5 votes vote down vote up
def test_splittype(self):
        splittype = urllib.splittype
        self.assertEqual(splittype('type:opaquestring'), ('type', 'opaquestring'))
        self.assertEqual(splittype('opaquestring'), (None, 'opaquestring'))
        self.assertEqual(splittype(':opaquestring'), (None, ':opaquestring'))
        self.assertEqual(splittype('type:'), ('type', ''))
        self.assertEqual(splittype('type:opaque:string'), ('type', 'opaque:string')) 
Example 30
Project: oil   Author: oilshell   File: xmlrpclib.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,
                 allow_none=0, use_datetime=0, context=None):
        # establish a "logical" server connection

        if unicode and isinstance(uri, unicode):
            uri = uri.encode('ISO-8859-1')

        # get the url
        import urllib
        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError, "unsupported XML-RPC protocol"
        self.__host, self.__handler = urllib.splithost(uri)
        if not self.__handler:
            self.__handler = "/RPC2"

        if transport is None:
            if type == "https":
                transport = SafeTransport(use_datetime=use_datetime, context=context)
            else:
                transport = Transport(use_datetime=use_datetime)
        self.__transport = transport

        self.__encoding = encoding
        self.__verbose = verbose
        self.__allow_none = allow_none 
Example 31
Project: oil   Author: oilshell   File: rourl2path.py    (license) View Source Project 5 votes vote down vote up
def url2pathname(url):
    """OS-specific conversion from a relative URL of the 'file' scheme
    to a file system path; not recommended for general use."""
    tp = urllib.splittype(url)[0]
    if tp and tp <> 'file':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    # Turn starting /// into /, an empty hostname means current host
    if url[:3] == '///':
        url = url[2:]
    elif url[:2] == '//':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    components = string.split(url, '/')
    if not components[0]:
        if '$' in components:
            del components[0]
        else:
            components[0] = '$'
    # Remove . and embedded ..
    i = 0
    while i < len(components):
        if components[i] == '.':
            del components[i]
        elif components[i] == '..' and i > 0 and \
                                  components[i-1] not in ('', '..'):
            del components[i-1:i+1]
            i -= 1
        elif components[i] == '..':
            components[i] = '^'
            i += 1
        elif components[i] == '' and i > 0 and components[i-1] <> '':
            del components[i]
        else:
            i += 1
    components = map(lambda x: urllib.unquote(x).translate(__slash_dot), components)
    return '.'.join(components) 
Example 32
Project: oil   Author: oilshell   File: macurl2path.py    (license) View Source Project 5 votes vote down vote up
def url2pathname(pathname):
    """OS-specific conversion from a relative URL of the 'file' scheme
    to a file system path; not recommended for general use."""
    #
    # XXXX The .. handling should be fixed...
    #
    tp = urllib.splittype(pathname)[0]
    if tp and tp != 'file':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    # Turn starting /// into /, an empty hostname means current host
    if pathname[:3] == '///':
        pathname = pathname[2:]
    elif pathname[:2] == '//':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    components = pathname.split('/')
    # Remove . and embedded ..
    i = 0
    while i < len(components):
        if components[i] == '.':
            del components[i]
        elif components[i] == '..' and i > 0 and \
                                  components[i-1] not in ('', '..'):
            del components[i-1:i+1]
            i = i-1
        elif components[i] == '' and i > 0 and components[i-1] != '':
            del components[i]
        else:
            i = i+1
    if not components[0]:
        # Absolute unix path, don't start with colon
        rv = ':'.join(components[1:])
    else:
        # relative unix path, start with colon. First replace
        # leading .. by empty strings (giving ::file)
        i = 0
        while i < len(components) and components[i] == '..':
            components[i] = ''
            i = i + 1
        rv = ':' + ':'.join(components)
    # and finally unquote slashes and other funny characters
    return urllib.unquote(rv) 
Example 33
Project: oil   Author: oilshell   File: websucker.py    (license) View Source Project 5 votes vote down vote up
def savefilename(self, url):
        type, rest = urllib.splittype(url)
        host, path = urllib.splithost(rest)
        path = path.lstrip("/")
        user, host = urllib.splituser(host)
        host, port = urllib.splitnport(host)
        host = host.lower()
        if not path or path[-1] == "/":
            path = path + "index.html"
        if os.sep != "/":
            path = os.sep.join(path.split("/"))
        path = os.path.join(host, path)
        return path 
Example 34
Project: python2-tracer   Author: extremecoders-re   File: test_urllib.py    (license) View Source Project 5 votes vote down vote up
def test_splittype(self):
        splittype = urllib.splittype
        self.assertEqual(splittype('type:opaquestring'), ('type', 'opaquestring'))
        self.assertEqual(splittype('opaquestring'), (None, 'opaquestring'))
        self.assertEqual(splittype(':opaquestring'), (None, ':opaquestring'))
        self.assertEqual(splittype('type:'), ('type', ''))
        self.assertEqual(splittype('type:opaque:string'), ('type', 'opaque:string')) 
Example 35
Project: python2-tracer   Author: extremecoders-re   File: xmlrpclib.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,
                 allow_none=0, use_datetime=0, context=None):
        # establish a "logical" server connection

        if unicode and isinstance(uri, unicode):
            uri = uri.encode('ISO-8859-1')

        # get the url
        import urllib
        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError, "unsupported XML-RPC protocol"
        self.__host, self.__handler = urllib.splithost(uri)
        if not self.__handler:
            self.__handler = "/RPC2"

        if transport is None:
            if type == "https":
                transport = SafeTransport(use_datetime=use_datetime, context=context)
            else:
                transport = Transport(use_datetime=use_datetime)
        self.__transport = transport

        self.__encoding = encoding
        self.__verbose = verbose
        self.__allow_none = allow_none 
Example 36
Project: python2-tracer   Author: extremecoders-re   File: rourl2path.py    (license) View Source Project 5 votes vote down vote up
def url2pathname(url):
    """OS-specific conversion from a relative URL of the 'file' scheme
    to a file system path; not recommended for general use."""
    tp = urllib.splittype(url)[0]
    if tp and tp <> 'file':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    # Turn starting /// into /, an empty hostname means current host
    if url[:3] == '///':
        url = url[2:]
    elif url[:2] == '//':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    components = string.split(url, '/')
    if not components[0]:
        if '$' in components:
            del components[0]
        else:
            components[0] = '$'
    # Remove . and embedded ..
    i = 0
    while i < len(components):
        if components[i] == '.':
            del components[i]
        elif components[i] == '..' and i > 0 and \
                                  components[i-1] not in ('', '..'):
            del components[i-1:i+1]
            i -= 1
        elif components[i] == '..':
            components[i] = '^'
            i += 1
        elif components[i] == '' and i > 0 and components[i-1] <> '':
            del components[i]
        else:
            i += 1
    components = map(lambda x: urllib.unquote(x).translate(__slash_dot), components)
    return '.'.join(components) 
Example 37
Project: python2-tracer   Author: extremecoders-re   File: macurl2path.py    (license) View Source Project 5 votes vote down vote up
def url2pathname(pathname):
    """OS-specific conversion from a relative URL of the 'file' scheme
    to a file system path; not recommended for general use."""
    #
    # XXXX The .. handling should be fixed...
    #
    tp = urllib.splittype(pathname)[0]
    if tp and tp != 'file':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    # Turn starting /// into /, an empty hostname means current host
    if pathname[:3] == '///':
        pathname = pathname[2:]
    elif pathname[:2] == '//':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    components = pathname.split('/')
    # Remove . and embedded ..
    i = 0
    while i < len(components):
        if components[i] == '.':
            del components[i]
        elif components[i] == '..' and i > 0 and \
                                  components[i-1] not in ('', '..'):
            del components[i-1:i+1]
            i = i-1
        elif components[i] == '' and i > 0 and components[i-1] != '':
            del components[i]
        else:
            i = i+1
    if not components[0]:
        # Absolute unix path, don't start with colon
        rv = ':'.join(components[1:])
    else:
        # relative unix path, start with colon. First replace
        # leading .. by empty strings (giving ::file)
        i = 0
        while i < len(components) and components[i] == '..':
            components[i] = ''
            i = i + 1
        rv = ':' + ':'.join(components)
    # and finally unquote slashes and other funny characters
    return urllib.unquote(rv) 
Example 38
Project: python2-tracer   Author: extremecoders-re   File: websucker.py    (license) View Source Project 5 votes vote down vote up
def savefilename(self, url):
        type, rest = urllib.splittype(url)
        host, path = urllib.splithost(rest)
        path = path.lstrip("/")
        user, host = urllib.splituser(host)
        host, port = urllib.splitnport(host)
        host = host.lower()
        if not path or path[-1] == "/":
            path = path + "index.html"
        if os.sep != "/":
            path = os.sep.join(path.split("/"))
        path = os.path.join(host, path)
        return path 
Example 39
Project: sslstrip-hsts-openwrt   Author: adde88   File: xmlrpclib.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None, verbose=0,
                 allow_none=0, use_datetime=0, context=None):
        # establish a "logical" server connection

        if isinstance(uri, unicode):
            uri = uri.encode('ISO-8859-1')

        # get the url
        import urllib
        type, uri = urllib.splittype(uri)
        if type not in ("http", "https"):
            raise IOError, "unsupported XML-RPC protocol"
        self.__host, self.__handler = urllib.splithost(uri)
        if not self.__handler:
            self.__handler = "/RPC2"

        if transport is None:
            if type == "https":
                transport = SafeTransport(use_datetime=use_datetime, context=context)
            else:
                transport = Transport(use_datetime=use_datetime)
        self.__transport = transport

        self.__encoding = encoding
        self.__verbose = verbose
        self.__allow_none = allow_none 
Example 40
Project: sslstrip-hsts-openwrt   Author: adde88   File: macurl2path.py    (license) View Source Project 5 votes vote down vote up
def url2pathname(pathname):
    """OS-specific conversion from a relative URL of the 'file' scheme
    to a file system path; not recommended for general use."""
    #
    # XXXX The .. handling should be fixed...
    #
    tp = urllib.splittype(pathname)[0]
    if tp and tp != 'file':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    # Turn starting /// into /, an empty hostname means current host
    if pathname[:3] == '///':
        pathname = pathname[2:]
    elif pathname[:2] == '//':
        raise RuntimeError, 'Cannot convert non-local URL to pathname'
    components = pathname.split('/')
    # Remove . and embedded ..
    i = 0
    while i < len(components):
        if components[i] == '.':
            del components[i]
        elif components[i] == '..' and i > 0 and \
                                  components[i-1] not in ('', '..'):
            del components[i-1:i+1]
            i = i-1
        elif components[i] == '' and i > 0 and components[i-1] != '':
            del components[i]
        else:
            i = i+1
    if not components[0]:
        # Absolute unix path, don't start with colon
        rv = ':'.join(components[1:])
    else:
        # relative unix path, start with colon. First replace
        # leading .. by empty strings (giving ::file)
        i = 0
        while i < len(components) and components[i] == '..':
            components[i] = ''
            i = i + 1
        rv = ':' + ':'.join(components)
    # and finally unquote slashes and other funny characters
    return urllib.unquote(rv) 
Example 41
Project: nginx_log_parse   Author: daiguadaidai   File: ng_line_parser.py    (license) View Source Project 5 votes vote down vote up
def reference_url(self, reference_url):
        """????URL
        ?????????, ?:
            ??: http://www.ttmark.com/diannao/2014/11/04/470.html
            ???: www.ttmark.com
        """
        proto, rest = urllib.splittype(reference_url)
        res, rest = urllib.splithost(rest)
        if not res:
            self._reference_url = '-'
        else:
            self._reference_url = res 
Example 42
Project: nginx_log_parse   Author: daiguadaidai   File: ng_line_parser.py    (license) View Source Project 5 votes vote down vote up
def request_url(self, request_url):
        """?????URL
        ???????URL???????, ?:
            ??: /wp-admin/admin-ajax.php?postviews_id=1348
            ???: /wp-admin/admin-ajax.php
        """
        proto, rest = urllib.splittype(request_url)
        url_path, url_param = urllib.splitquery(rest)

        if url_path.startswith('/tag/'):
            url_path = '/tag/'

        self._request_url = url_path 
Example 43
Project: TA-connectivity   Author: seunomosowon   File: binding.py    (license) View Source Project 5 votes vote down vote up
def _spliturl(url):
    scheme, opaque = urllib.splittype(url)
    netloc, path = urllib.splithost(opaque)
    host, port = urllib.splitport(netloc)
    # Strip brackets if its an IPv6 address
    if host.startswith('[') and host.endswith(']'): host = host[1:-1]
    if port is None: port = DEFAULT_PORT
    return scheme, host, port, path

# Given an HTTP request handler, this wrapper objects provides a related
# family of convenience methods built using that handler. 
Example 44
Project: p2pool-ltc   Author: ilsawa   File: Client.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, url, config = Config):
        proto, uri = urllib.splittype(url)

        # apply some defaults
        if uri[0:2] != '//':
            if proto != None:
                uri = proto + ':' + uri

            uri = '//' + uri
            proto = 'http'

        host, path = urllib.splithost(uri)

        try:
            int(host)
            host = 'localhost:' + host
        except:
            pass

        if not path:
            path = '/'

        if proto not in ('http', 'https', 'httpg'):
            raise IOError, "unsupported SOAP protocol"
        if proto == 'httpg' and not config.GSIclient:
            raise AttributeError, \
                  "GSI client not supported by this Python installation"
        if proto == 'https' and not config.SSLclient:
            raise AttributeError, \
                "SSL client not supported by this Python installation"

        self.user,host = urllib.splituser(host)
        self.proto = proto
        self.host = host
        self.path = path 
Example 45
Project: p2pool-bsty   Author: amarian12   File: Client.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, url, config = Config):
        proto, uri = urllib.splittype(url)

        # apply some defaults
        if uri[0:2] != '//':
            if proto != None:
                uri = proto + ':' + uri

            uri = '//' + uri
            proto = 'http'

        host, path = urllib.splithost(uri)

        try:
            int(host)
            host = 'localhost:' + host
        except:
            pass

        if not path:
            path = '/'

        if proto not in ('http', 'https', 'httpg'):
            raise IOError, "unsupported SOAP protocol"
        if proto == 'httpg' and not config.GSIclient:
            raise AttributeError, \
                  "GSI client not supported by this Python installation"
        if proto == 'https' and not config.SSLclient:
            raise AttributeError, \
                "SSL client not supported by this Python installation"

        self.user,host = urllib.splituser(host)
        self.proto = proto
        self.host = host
        self.path = path 
Example 46
Project: deb-python-jsonrpclib   Author: openstack   File: jsonrpc.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, uri, transport=None, encoding=None,
                 verbose=0, version=None):
        import urllib
        if not version:
            version = config.version
        self.__version = version
        schema, uri = urllib.splittype(uri)
        if schema not in ('http', 'https', 'unix'):
            raise IOError('Unsupported JSON-RPC protocol.')
        if schema == 'unix':
            if not USE_UNIX_SOCKETS:
                # Don't like the "generic" Exception...
                raise UnixSocketMissing("Unix sockets not available.")
            self.__host = uri
            self.__handler = '/'
        else:
            self.__host, self.__handler = urllib.splithost(uri)
            if not self.__handler:
                # Not sure if this is in the JSON spec?
                # self.__handler = '/'
                self.__handler == '/'
        if transport is None:
            if schema == 'unix':
                transport = UnixTransport()
            elif schema == 'https':
                transport = SafeTransport()
            else:
                transport = Transport()
        self.__transport = transport
        self.__encoding = encoding
        self.__verbose = verbose 
Example 47
Project: splunk_ta_ps4_f1_2016   Author: jonathanvarley   File: binding.py    (license) View Source Project 5 votes vote down vote up
def _spliturl(url):
    scheme, opaque = urllib.splittype(url)
    netloc, path = urllib.splithost(opaque)
    host, port = urllib.splitport(netloc)
    # Strip brackets if its an IPv6 address
    if host.startswith('[') and host.endswith(']'): host = host[1:-1]
    if port is None: port = DEFAULT_PORT
    return scheme, host, port, path

# Given an HTTP request handler, this wrapper objects provides a related
# family of convenience methods built using that handler. 
Example 48
Project: splunk_ta_ps4_f1_2016   Author: jonathanvarley   File: binding.py    (license) View Source Project 5 votes vote down vote up
def _spliturl(url):
    scheme, opaque = urllib.splittype(url)
    netloc, path = urllib.splithost(opaque)
    host, port = urllib.splitport(netloc)
    # Strip brackets if its an IPv6 address
    if host.startswith('[') and host.endswith(']'): host = host[1:-1]
    if port is None: port = DEFAULT_PORT
    return scheme, host, port, path

# Given an HTTP request handler, this wrapper objects provides a related
# family of convenience methods built using that handler. 
Example 49
Project: py-google-safelist   Author: furio   File: urlutils.py    (license) View Source Project 5 votes vote down vote up
def url_permutations(url):
        """Try all permutations of hostname and path which can be applied
        to blacklisted URLs"""
        def url_host_permutations(host):
            if re.match(r'\d+\.\d+\.\d+\.\d+', host):
                yield host
                return
            parts = host.split('.')
            l = min(len(parts),5)
            if l > 4:
                yield host
            for i in xrange(l-1):
                yield '.'.join(parts[i-l:])
        def url_path_permutations(path):
            if path != '/':
                yield path
            query = None
            if '?' in path:
                path, query =  path.split('?', 1)
            if query is not None:
                yield path
            path_parts = path.split('/')[0:-1]
            curr_path = ''
            for i in xrange(min(4, len(path_parts))):
                curr_path = curr_path + path_parts[i] + '/'
                yield curr_path
        protocol, address_str = urllib.splittype(url)
        host, path = urllib.splithost(address_str)
        user, host = urllib.splituser(host)
        host, port = urllib.splitport(host)
        host = host.strip('/')
        for h in url_host_permutations(host):
            for p in url_path_permutations(path):
                yield '%s%s' % (h, p) 
Example 50
Project: TA-SyncKVStore   Author: georgestarcher   File: binding.py    (license) View Source Project 5 votes vote down vote up
def _spliturl(url):
    scheme, opaque = urllib.splittype(url)
    netloc, path = urllib.splithost(opaque)
    host, port = urllib.splitport(netloc)
    # Strip brackets if its an IPv6 address
    if host.startswith('[') and host.endswith(']'): host = host[1:-1]
    if port is None: port = DEFAULT_PORT
    return scheme, host, port, path

# Given an HTTP request handler, this wrapper objects provides a related
# family of convenience methods built using that handler.