Python urlparse.urlsplit() Examples

The following are code examples for showing how to use urlparse.urlsplit(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: pyblish-win   Author: pyblish   File: urllib2.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def reduce_uri(self, uri, default_port=True):
        """Accept authority or URI and extract only the authority and path."""
        # note HTTP URLs do not have a userinfo component
        parts = urlparse.urlsplit(uri)
        if parts[1]:
            # URI
            scheme = parts[0]
            authority = parts[1]
            path = parts[2] or '/'
        else:
            # host or host:port
            scheme = None
            authority = uri
            path = '/'
        host, port = splitport(authority)
        if default_port and port is None and scheme is not None:
            dport = {"http": 80,
                     "https": 443,
                     }.get(scheme)
            if dport is not None:
                authority = "%s:%d" % (host, dport)
        return authority, path 
Example 2
Project: pyblish-win   Author: pyblish   File: test_urlparse.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_issue14072(self):
        p1 = urlparse.urlsplit('tel:+31-641044153')
        self.assertEqual(p1.scheme, 'tel')
        self.assertEqual(p1.path, '+31-641044153')

        p2 = urlparse.urlsplit('tel:+31641044153')
        self.assertEqual(p2.scheme, 'tel')
        self.assertEqual(p2.path, '+31641044153')

        # Assert for urlparse
        p1 = urlparse.urlparse('tel:+31-641044153')
        self.assertEqual(p1.scheme, 'tel')
        self.assertEqual(p1.path, '+31-641044153')

        p2 = urlparse.urlparse('tel:+31641044153')
        self.assertEqual(p2.scheme, 'tel')
        self.assertEqual(p2.path, '+31641044153') 
Example 3
Project: pyblish-win   Author: pyblish   File: test_urlparse.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_attributes_without_netloc(self):
        # This example is straight from RFC 3261.  It looks like it
        # should allow the username, hostname, and port to be filled
        # in, but doesn't.  Since it's a URI and doesn't use the
        # scheme://netloc syntax, the netloc and related attributes
        # should be left empty.
        uri = "sip:[email protected];maddr=239.255.255.1;ttl=15"
        p = urlparse.urlsplit(uri)
        self.assertEqual(p.netloc, "")
        self.assertEqual(p.username, None)
        self.assertEqual(p.password, None)
        self.assertEqual(p.hostname, None)
        self.assertEqual(p.port, None)
        self.assertEqual(p.geturl(), uri)

        p = urlparse.urlparse(uri)
        self.assertEqual(p.netloc, "")
        self.assertEqual(p.username, None)
        self.assertEqual(p.password, None)
        self.assertEqual(p.hostname, None)
        self.assertEqual(p.port, None)
        self.assertEqual(p.geturl(), uri) 
Example 4
Project: openhatch   Author: campbe13   File: base_extras.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def enhance_next_to_annotate_it_with_newuser_is_true(value, **args):
    # value is a URL -- the value of "next"
    # here, we enhance it to get a query string indicating the user is a new
    # user
    parts = urlparse.urlsplit(value)
    if parts.query:
        new_query = parts.query
        if 'newuser=true' not in new_query:
            new_query += '&newuser=true'
    else:
        new_query = 'newuser=true'
    return urlparse.urlunsplit(
        (parts[0], parts[1], parts[2], new_query, parts.fragment))

# Starting point for the below code
# was http://www.djangosnippets.org/snippets/1656/ 
Example 5
Project: git-externals   Author: develersrl   File: git_externals.py    MIT License 6 votes vote down vote up
def normalize_gitext_url(url):
    # an absolute url is already normalized
    if urlparse(url).netloc != '' or url.startswith('[email protected]'):
        return url

    # relative urls use the root url of the current origin
    remote_name = git('config', 'branch.%s.remote' % current_branch()).strip()
    remote_url = git('config', 'remote.%s.url' % remote_name).strip()

    if remote_url.startswith('[email protected]'):
        prefix = remote_url[:remote_url.index(':')+1]
        remote_url = prefix + url.strip('/')
    else:
        remote_url = urlunsplit(urlsplit(remote_url)._replace(path=url))

    return remote_url 
Example 6
Project: pyomni   Author: taxpon   File: WebdavClient.py    MIT License 6 votes vote down vote up
def _checkUrl(url):
    """
    Checks the given URL for validity.
    
    @param url: URL to check.
    @type  url: C{string}
    
    @raise ValueError: If the URL does not contain valid/usable content.
    """
    
    parts = urlsplit(url, allow_fragments=False)
    if len(parts[0]) == 0 or len(parts[1]) == 0 or len(parts[2]) == 0:
        raise ValueError("Invalid URL: " + repr(url))


# small test:
# asks for WebDAV collection or file (if not passed) 
# and lists the content of the collection or downloads the file; 
# requests user name and password where required by server 
Example 7
Project: vulscan   Author: vulscanteam   File: option.py    MIT License 6 votes vote down vote up
def _setHTTPProxy():
    """
    Check and set the HTTP/SOCKS proxy for all HTTP requests.
    """

    if not conf.proxy:
        return

    infoMsg = "setting the HTTP/SOCKS proxy for all HTTP requests"
    logger.log(CUSTOM_LOGGING.SYSINFO, infoMsg)

    try:
        _ = urlparse.urlsplit(conf.proxy)
    except Exception, ex:
        errMsg = "invalid proxy address '%s' ('%s')" % (conf.proxy, ex)
        raise PocsuiteSyntaxException(errMsg) 
Example 8
Project: devtools-parser   Author: WPO-Foundation   File: devtools_parser.py    Apache License 2.0 6 votes vote down vote up
def get_base_page_info(self, page_data):
        """Find the reverse-ip info for the base page"""
        domain = urlparse.urlsplit(page_data['final_url']).hostname
        try:
            import socket
            addr = socket.gethostbyname(domain)
            host = str(socket.gethostbyaddr(addr)[0])
            page_data['base_page_ip_ptr'] = host
        except Exception:
            pass
        # keep moving up the domain until we can get a NS record
        while domain is not None and 'base_page_dns_soa' not in page_data:
            try:
                import dns.resolver
                dns_servers = dns.resolver.query(domain, "NS")
                dns_server = str(dns_servers[0].target).strip('. ')
                page_data['base_page_dns_ns'] = dns_server
            except Exception:
                pass
            pos = domain.find('.')
            if pos > 0:
                domain = domain[pos + 1:]
            else:
                domain = None 
Example 9
Project: TESTGIT   Author: B-ROY   File: web.py    Apache License 2.0 6 votes vote down vote up
def authenticated(method):
    """Decorate methods with this to require that the user be logged in."""
    @functools.wraps(method)
    def wrapper(self, *args, **kwargs):
        if not self.current_user:
            if self.request.method in ("GET", "HEAD"):
                url = self.get_login_url()
                if "?" not in url:
                    if urlparse.urlsplit(url).scheme:
                        # if login url is absolute, make next absolute too
                        next_url = self.request.full_url()
                    else:
                        next_url = self.request.uri
                    url += "?" + urllib.urlencode(dict(next=next_url))
                self.redirect(url)
                return
            raise HTTPError(403)
        return method(self, *args, **kwargs)
    return wrapper 
Example 10
Project: bugatsinho.github.io   Author: bugatsinho   File: base.py    GNU General Public License v3.0 6 votes vote down vote up
def unshorten(self, uri, type=None):

        domain = urlsplit(uri).netloc

        if not domain:
            return uri, "No domain found in URI!"

        had_google_outbound, uri = self._clear_google_outbound_proxy(uri)

        if re.search(self._adfly_regex, domain, re.IGNORECASE) or type == 'adfly':
            return self._unshorten_adfly(uri)
        if re.search(self._adfocus_regex, domain, re.IGNORECASE) or type == 'adfocus':
            return self._unshorten_adfocus(uri)
        if re.search(self._linkbucks_regex, domain, re.IGNORECASE) or type == 'linkbucks':
            return self._unshorten_linkbucks(uri)
        if re.search(self._lnxlu_regex, domain, re.IGNORECASE) or type == 'lnxlu':
            return self._unshorten_lnxlu(uri)
        if re.search(self._shst_regex, domain, re.IGNORECASE):
            return self._unshorten_shst(uri)
        if re.search(self._hrefli_regex, domain, re.IGNORECASE):
            return self._unshorten_hrefli(uri)
        if re.search(self._anonymz_regex, domain, re.IGNORECASE):
            return self._unshorten_anonymz(uri)

        return uri, 200 
Example 11
Project: pyblish-win   Author: pyblish   File: cookielib.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def request_path(request):
    """Path component of request-URI, as defined by RFC 2965."""
    url = request.get_full_url()
    parts = urlparse.urlsplit(url)
    path = escape_path(parts.path)
    if not path.startswith("/"):
        # fix bad RFC 2396 absoluteURI
        path = "/" + path
    return path 
Example 12
Project: pyblish-win   Author: pyblish   File: test_urlparse.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_http_roundtrips(self):
        # urlparse.urlsplit treats 'http:' as an optimized special case,
        # so we test both 'http:' and 'https:' in all the following.
        # Three cheers for white box knowledge!
        testcases = [
            ('://www.python.org',
             ('www.python.org', '', '', '', ''),
             ('www.python.org', '', '', '')),
            ('://www.python.org#abc',
             ('www.python.org', '', '', '', 'abc'),
             ('www.python.org', '', '', 'abc')),
            ('://www.python.org?q=abc',
             ('www.python.org', '', '', 'q=abc', ''),
             ('www.python.org', '', 'q=abc', '')),
            ('://www.python.org/#abc',
             ('www.python.org', '/', '', '', 'abc'),
             ('www.python.org', '/', '', 'abc')),
            ('://a/b/c/d;p?q#f',
             ('a', '/b/c/d', 'p', 'q', 'f'),
             ('a', '/b/c/d;p', 'q', 'f')),
            ]
        for scheme in ('http', 'https'):
            for url, parsed, split in testcases:
                url = scheme + url
                parsed = (scheme,) + parsed
                split = (scheme,) + split
                self.checkRoundtrips(url, parsed, split) 
Example 13
Project: pyblish-win   Author: pyblish   File: test_urlparse.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_unparse_parse(self):
        for u in ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',]:
            self.assertEqual(urlparse.urlunsplit(urlparse.urlsplit(u)), u)
            self.assertEqual(urlparse.urlunparse(urlparse.urlparse(u)), u) 
Example 14
Project: pyblish-win   Author: pyblish   File: test_urlparse.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_attributes_bad_port(self):
        """Check handling of non-integer ports."""
        p = urlparse.urlsplit("http://www.example.net:foo")
        self.assertEqual(p.netloc, "www.example.net:foo")
        self.assertRaises(ValueError, lambda: p.port)

        p = urlparse.urlparse("http://www.example.net:foo")
        self.assertEqual(p.netloc, "www.example.net:foo")
        self.assertRaises(ValueError, lambda: p.port) 
Example 15
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: testing.py    Apache License 2.0 5 votes vote down vote up
def make_test_environ_builder(app, path='/', base_url=None, *args, **kwargs):
    """Creates a new test builder with some application defaults thrown in."""
    http_host = app.config.get('SERVER_NAME')
    app_root = app.config.get('APPLICATION_ROOT')
    if base_url is None:
        url = url_parse(path)
        base_url = 'http://%s/' % (url.netloc or http_host or 'localhost')
        if app_root:
            base_url += app_root.lstrip('/')
        if url.netloc:
            path = url.path
    return EnvironBuilder(path, base_url, *args, **kwargs) 
Example 16
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: testing.py    Apache License 2.0 5 votes vote down vote up
def make_test_environ_builder(app, path='/', base_url=None, *args, **kwargs):
    """Creates a new test builder with some application defaults thrown in."""
    http_host = app.config.get('SERVER_NAME')
    app_root = app.config.get('APPLICATION_ROOT')
    if base_url is None:
        url = url_parse(path)
        base_url = 'http://%s/' % (url.netloc or http_host or 'localhost')
        if app_root:
            base_url += app_root.lstrip('/')
        if url.netloc:
            path = url.path
    return EnvironBuilder(path, base_url, *args, **kwargs) 
Example 17
Project: flasky   Author: RoseOu   File: testing.py    MIT License 5 votes vote down vote up
def make_test_environ_builder(app, path='/', base_url=None, *args, **kwargs):
    """Creates a new test builder with some application defaults thrown in."""
    http_host = app.config.get('SERVER_NAME')
    app_root = app.config.get('APPLICATION_ROOT')
    if base_url is None:
        url = url_parse(path)
        base_url = 'http://%s/' % (url.netloc or http_host or 'localhost')
        if app_root:
            base_url += app_root.lstrip('/')
        if url.netloc:
            path = url.path
    return EnvironBuilder(path, base_url, *args, **kwargs) 
Example 18
Project: flasky   Author: RoseOu   File: svn_utils.py    MIT License 5 votes vote down vote up
def parse_external_prop(lines):
    """
    Parse the value of a retrieved svn:externals entry.

    possible token setups (with quotng and backscaping in laters versions)
        URL[@#] EXT_FOLDERNAME
        [-r#] URL EXT_FOLDERNAME
        EXT_FOLDERNAME [-r#] URL
    """
    externals = []
    for line in lines.splitlines():
        line = line.lstrip()  # there might be a "\ "
        if not line:
            continue

        if sys.version_info < (3, 0):
            #shlex handles NULLs just fine and shlex in 2.7 tries to encode
            #as ascii automatiically
            line = line.encode('utf-8')
        line = shlex.split(line)
        if sys.version_info < (3, 0):
            line = [x.decode('utf-8') for x in line]

        #EXT_FOLDERNAME is either the first or last depending on where
        #the URL falls
        if urlparse.urlsplit(line[-1])[0]:
            external = line[0]
        else:
            external = line[-1]

        external = decode_as_string(external, encoding="utf-8")
        externals.append(os.path.normpath(external))

    return externals 
Example 19
Project: Splunk_CBER_App   Author: MHaggis   File: script.py    MIT License 5 votes vote down vote up
def service(self):
        """ Returns a Splunk service object for this script invocation.

        The service object is created from the Splunkd URI and session key
        passed to the command invocation on the modular input stream. It is
        available as soon as the :code:`Script.stream_events` method is
        called.

        :return: :class:splunklib.client.Service. A value of None is returned,
        if you call this method before the :code:`Script.stream_events` method
        is called.

        """
        if self._service is not None:
            return self._service

        if self._input_definition is None:
            return None

        splunkd_uri = self._input_definition.metadata["server_uri"]
        session_key = self._input_definition.metadata["session_key"]

        splunkd = urlsplit(splunkd_uri, allow_fragments=False)

        self._service = Service(
            scheme=splunkd.scheme,
            host=splunkd.hostname,
            port=splunkd.port,
            token=session_key,
        )

        return self._service 
Example 20
Project: nightcrawler   Author: luckyr13   File: nightcrawler.py    MIT License 5 votes vote down vote up
def parse_base_url(self, url):
        """
            Create a valid base url

            @param url: Url to clean
            @type url: string
        """
        r1 = urlparse.urlsplit(url)
        r1 = "%s://%s/" % (r1.scheme, r1.netloc)
        return r1 
Example 21
Project: nightcrawler   Author: luckyr13   File: nightcrawler.py    MIT License 5 votes vote down vote up
def clean_url(self, url):
        """
            Clear url parameters and querys

            @param url: Url to clean
            @type url: string
        """
        r1 = urlparse.urlsplit(url)
        r1 = "%s://%s%s" % (r1.scheme, r1.netloc, r1.path)
        return r1 
Example 22
Project: nightcrawler   Author: luckyr13   File: nightcrawler.py    MIT License 5 votes vote down vote up
def map_files(self, url):
        """
            This function first verifies if the url has the valid MIME type
            and then calls fetch_links_from_url() to parse the HTML data

            @param url: Base url from where to start crawling
            @type url: string
        """
        
        rq = None
        mfiles = []
        r1 = urlparse.urlsplit(url)
        if not r1.scheme and not r1.netloc:
            url = self.base_url + url
        
        print "\n+ Crawling: %s" % (url,)
        
        try:
            rq = requests.get(url, timeout=TIMEOUT)
        except:
            raise Exception("Connection error (1):(")

        # Dont proceed if this page dont exists
        if rq.status_code != 200:
            rq.raise_for_status()

        # Check for content-type == text/html
        contype = self.get_contype(rq.headers['content-type'])
        print '    - Content-Type:', contype

        if contype != 'text/html':
            print '\tERROR: Is not a text/html\n'
            return mfiles

        # Parse page and search links
        self.fetch_links_from_url(rq.text, url) 
Example 23
Project: mx   Author: graalvm   File: mx.py    GNU General Public License v2.0 5 votes vote down vote up
def _validate_abolute_url(urlstr, acceptNone=False):
    if urlstr is None:
        return acceptNone
    url = _urllib_parse.urlsplit(urlstr)
    return url.scheme and (url.netloc or url.path) 
Example 24
Project: presto-admin   Author: prestosql   File: prestoclient.py    Apache License 2.0 5 votes vote down vote up
def _get_response_from(self, uri):
        """
        Sends a GET request to the Presto server at the specified next_uri
        and updates the response

        Remove the scheme and host/port from the uri; the connection itself
        has that information.
        """
        parts = list(urlparse.urlsplit(uri))
        parts[0] = None
        parts[1] = None
        location = urlparse.urlunsplit(parts)
        conn = self._get_connection()
        headers = {"X-Presto-User": self.user}
        self._add_auth_headers(headers)
        conn.request("GET", location, headers=headers)
        response = conn.getresponse()

        if response.status != 200:
            conn.close()
            _LOGGER.error("Error making GET request to %s: %s %s" %
                          (uri, response.status, response.reason))
            return False

        answer = response.read()
        conn.close()

        self.response_from_server = json.loads(answer)
        _LOGGER.info("GET request successful for uri: " + uri)
        return True 
Example 25
Project: openhatch   Author: campbe13   File: webcrawler.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def domain(url):
    """Returns the domain part of an URL."""
    return urlparse.urlsplit(url)[1].split(":")[0] 
Example 26
Project: bitpay-brick   Author: javgh   File: bitpayprovider.py    MIT License 5 votes vote down vote up
def get_bitcoin_uri_with_bluetooth_address(self, bluetooth_address):
        (scheme, netloc, path, query, fragment) = \
                urlparse.urlsplit(self.bitcoin_uri)
        query_parts = urlparse.parse_qs(query)
        query_parts['amount'] = float(query_parts['amount'][0])
        query_parts['r'] = 'bt:%s' % bluetooth_address.replace(':', '')
        query = urllib.urlencode(query_parts)
        return urlparse.urlunsplit((scheme, netloc, path, query, fragment)) 
Example 27
Project: misp42splunk   Author: remg427   File: script.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def service(self):
        """ Returns a Splunk service object for this script invocation.

        The service object is created from the Splunkd URI and session key
        passed to the command invocation on the modular input stream. It is
        available as soon as the :code:`Script.stream_events` method is
        called.

        :return: :class:splunklib.client.Service. A value of None is returned,
        if you call this method before the :code:`Script.stream_events` method
        is called.

        """
        if self._service is not None:
            return self._service

        if self._input_definition is None:
            return None

        splunkd_uri = self._input_definition.metadata["server_uri"]
        session_key = self._input_definition.metadata["session_key"]

        splunkd = urlsplit(splunkd_uri, allow_fragments=False)

        self._service = Service(
            scheme=splunkd.scheme,
            host=splunkd.hostname,
            port=splunkd.port,
            token=session_key,
        )

        return self._service 
Example 28
Project: misp42splunk   Author: remg427   File: iri2uri.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def iri2uri(uri):
    """Convert an IRI to a URI. Note that IRIs must be
    passed in a unicode strings. That is, do not utf-8 encode
    the IRI before passing it into the function."""
    if isinstance(uri ,unicode):
        (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri)
        authority = authority.encode('idna')
        # For each character in 'ucschar' or 'iprivate'
        #  1. encode as utf-8
        #  2. then %-encode each octet of that utf-8
        uri = urlparse.urlunsplit((scheme, authority, path, query, fragment))
        uri = "".join([encode(c) for c in uri])
    return uri 
Example 29
Project: misp42splunk   Author: remg427   File: __init__.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def urldefrag(url):
    if "#" in url:
        s, n, p, q, frag = urlsplit(url)
        defrag = urlunsplit((s, n, p, q, ''))
    else:
        defrag = url
        frag = ''
    return defrag, frag 
Example 30
Project: d6tpipe   Author: d6t   File: s3.py    MIT License 5 votes vote down vote up
def _path_to_bucket_and_key(path):
        (scheme, netloc, path, query, fragment) = urlsplit(path,
                                                           allow_fragments=False)
        question_mark_plus_query = '?' + query if query else ''
        path_without_initial_slash = path[1:] + question_mark_plus_query
        return netloc, path_without_initial_slash 
Example 31
Project: quiz-extensions   Author: ucfopen   File: tool_provider.py    MIT License 5 votes vote down vote up
def build_return_url(self):
        '''
        If the Tool Consumer sent a return URL, add any set messages to the
        URL.
        '''
        if not self.launch_presentation_return_url:
            return None

        lti_message_fields = ['lti_errormsg', 'lti_errorlog',
                              'lti_msg', 'lti_log']

        messages = dict([(key, getattr(self, key))
                         for key in lti_message_fields
                         if getattr(self, key, None)])

        # Disassemble original return URL and reassemble with our options added
        original = urlsplit(self.launch_presentation_return_url)

        combined = messages.copy()
        combined.update(dict(parse_qsl(original.query)))

        combined_query = urlencode(combined)

        return urlunsplit((
            original.scheme,
            original.netloc,
            original.path,
            combined_query,
            original.fragment
        )) 
Example 32
Project: TornadoWeb   Author: VxCoder   File: web.py    Apache License 2.0 5 votes vote down vote up
def authenticated(method):
    """Decorate methods with this to require that the user be logged in.

    If the user is not logged in, they will be redirected to the configured
    `login url <RequestHandler.get_login_url>`.

    If you configure a login url with a query parameter, Tornado will
    assume you know what you're doing and use it as-is.  If not, it
    will add a `next` parameter so the login page knows where to send
    you once you're logged in.
    """
    @functools.wraps(method)
    def wrapper(self, *args, **kwargs):
        if not self.current_user:
            if self.request.method in ("GET", "HEAD"):
                url = self.get_login_url()
                if "?" not in url:
                    if urlparse.urlsplit(url).scheme:
                        # if login url is absolute, make next absolute too
                        next_url = self.request.full_url()
                    else:
                        next_url = self.request.uri
                    url += "?" + urlencode(dict(next=next_url))
                self.redirect(url)
                return
            raise HTTPError(403)
        return method(self, *args, **kwargs)
    return wrapper 
Example 33
Project: pyomni   Author: taxpon   File: WebdavClient.py    MIT License 5 votes vote down vote up
def __init__(self, url, connection=None, validateResourceNames=True):
        """
        Creates an instance for the given URL
        User must invoke validate() after construction to check the resource on the server.
        
        @param url: Unique resource location for this storer.
        @type  url: C{string}
        @param connection: this optional parameter contains a Connection object 
            for the host part of the given URL. Passing a connection saves 
            memory by sharing this connection. (defaults to None)
        @type  connection: L{webdav.Connection}
        @raise WebdavError: If validation of resource name path parts fails.
        """

        assert connection == None or isinstance(connection, Connection)
        parts = urlsplit(url, allow_fragments=False)
        self.path = parts[2]
        self.validateResourceNames = validateResourceNames
        
        # validate URL path
        for part in self.path.split('/'):
            if part != '' and not "ino:" in part: # explicitly allowing this character sequence as a part of a path (Tamino 4.4)
                if self.validateResourceNames:
                    try:
                        validateResourceName(part)
                    except WrongNameError:
                        raise WebdavError("Found invalid resource name part.")
                self.name = part
        # was: filter(lambda part: part and validateResourceName(part), self.path.split('/'))
        # but filter is deprecated
        
        self.defaultNamespace = None     # default XML name space of properties
        if connection:
            self.connection = connection
        else:
            conn = parts[1].split(":")
            if len(conn) == 1:
                self.connection = Connection(conn[0], protocol = parts[0])  # host and protocol
            else:
                self.connection = Connection(conn[0], int(conn[1]), protocol = parts[0])  # host and port and protocol
        self.versionHandler = VersionHandler(self.connection, self.path) 
Example 34
Project: analyzer-website   Author: santiagolizardo   File: iri2uri.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def iri2uri(uri):
    """Convert an IRI to a URI. Note that IRIs must be
    passed in a unicode strings. That is, do not utf-8 encode
    the IRI before passing it into the function."""
    if isinstance(uri ,unicode):
        (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri)
        authority = authority.encode('idna')
        # For each character in 'ucschar' or 'iprivate'
        #  1. encode as utf-8
        #  2. then %-encode each octet of that utf-8
        uri = urlparse.urlunsplit((scheme, authority, path, query, fragment))
        uri = "".join([encode(c) for c in uri])
    return uri 
Example 35
Project: Flask_Blog   Author: sugarguo   File: testing.py    GNU General Public License v3.0 5 votes vote down vote up
def make_test_environ_builder(app, path='/', base_url=None, *args, **kwargs):
    """Creates a new test builder with some application defaults thrown in."""
    http_host = app.config.get('SERVER_NAME')
    app_root = app.config.get('APPLICATION_ROOT')
    if base_url is None:
        url = url_parse(path)
        base_url = 'http://%s/' % (url.netloc or http_host or 'localhost')
        if app_root:
            base_url += app_root.lstrip('/')
        if url.netloc:
            path = url.path
    return EnvironBuilder(path, base_url, *args, **kwargs) 
Example 36
Project: Flask_Blog   Author: sugarguo   File: svn_utils.py    GNU General Public License v3.0 5 votes vote down vote up
def parse_external_prop(lines):
    """
    Parse the value of a retrieved svn:externals entry.

    possible token setups (with quotng and backscaping in laters versions)
        URL[@#] EXT_FOLDERNAME
        [-r#] URL EXT_FOLDERNAME
        EXT_FOLDERNAME [-r#] URL
    """
    externals = []
    for line in lines.splitlines():
        line = line.lstrip()  # there might be a "\ "
        if not line:
            continue

        if sys.version_info < (3, 0):
            #shlex handles NULLs just fine and shlex in 2.7 tries to encode
            #as ascii automatiically
            line = line.encode('utf-8')
        line = shlex.split(line)
        if sys.version_info < (3, 0):
            line = [x.decode('utf-8') for x in line]

        #EXT_FOLDERNAME is either the first or last depending on where
        #the URL falls
        if urlparse.urlsplit(line[-1])[0]:
            external = line[0]
        else:
            external = line[-1]

        external = decode_as_string(external, encoding="utf-8")
        externals.append(os.path.normpath(external))

    return externals 
Example 37
Project: wikiwhere   Author: mkrnr   File: ip_location_extraction.py    MIT License 5 votes vote down vote up
def get_ip_locations(self,json_data):
        url_location_dictionary = {}
        
        url_count = 0
        for article in json_data:
            for url in json_data[article]:
                url_count += 1
                if url in url_location_dictionary:
                    continue
                try:
                    ip = socket.gethostbyname(urlsplit(url).netloc)
        
                    response = self.db_reader.country(ip)
            
                    if response.country.iso_code is not None:
                        url_location_dictionary[url] = response.country.iso_code
                except socket.gaierror:
                    print "URL not found: " + url
                    continue
                except geoip2.errors.AddressNotFoundError:
                    print "IP location not found: " + ip
                    continue
                except Exception as exception:
                    print "Continue after " + exception.__class__.__name__ + " for URL: " + url 
                    continue

        return url_location_dictionary 
Example 38
Project: Health-Checker   Author: KriAga   File: clean.py    MIT License 5 votes vote down vote up
def allow_embedded_url(self, el, url):
        if (self.whitelist_tags is not None
            and el.tag not in self.whitelist_tags):
            return False
        scheme, netloc, path, query, fragment = urlsplit(url)
        netloc = netloc.lower().split(':', 1)[0]
        if scheme not in ('http', 'https'):
            return False
        if netloc in self.host_whitelist:
            return True
        return False 
Example 39
Project: oriskami-python   Author: oriskami   File: test_requestor.py    MIT License 5 votes vote down vote up
def __eq__(self, other):
        query = urlparse.urlsplit(other).query or other

        parsed = oriskami.util.parse_qsl(query)
        return self.expected == sorted(parsed) 
Example 40
Project: oriskami-python   Author: oriskami   File: test_requestor.py    MIT License 5 votes vote down vote up
def __init__(self, expected):
        self.exp_parts = urlparse.urlsplit(expected) 
Example 41
Project: oriskami-python   Author: oriskami   File: test_requestor.py    MIT License 5 votes vote down vote up
def __eq__(self, other):
        other_parts = urlparse.urlsplit(other)

        for part in ('scheme', 'netloc', 'path', 'fragment'):
            expected = getattr(self.exp_parts, part)
            actual = getattr(other_parts, part)
            if expected != actual:
                print 'Expected %s "%s" but got "%s"' % (
                    part, expected, actual)
                return False

        q_matcher = QueryMatcher(oriskami.util.parse_qsl(self.exp_parts.query))
        return q_matcher == other 
Example 42
Project: oriskami-python   Author: oriskami   File: test_requestor.py    MIT License 5 votes vote down vote up
def __eq__(self, other):
        query = urlparse.urlsplit(other).query or other

        parsed = ubivar.util.parse_qsl(query)
        return self.expected == sorted(parsed) 
Example 43
Project: oriskami-python   Author: oriskami   File: test_requestor.py    MIT License 5 votes vote down vote up
def __init__(self, expected):
        self.exp_parts = urlparse.urlsplit(expected) 
Example 44
Project: wpt-sync   Author: mozilla   File: wptfyi.py    Mozilla Public License 2.0 5 votes vote down vote up
def __init__(self, initial_url):
        if initial_url:
            parts = urlparse.urlsplit(initial_url)
            self.scheme = parts.scheme
            self.host = parts.netloc
            self.path = parts.path
            self.query = urlparse.parse_qsl(parts.query, keep_blank_values=True)
            self.fragment = parts.fragment
        else:
            self.scheme = ""
            self.host = ""
            self.path = ""
            self.query = []
            self.fragment = "" 
Example 45
Project: wpt-sync   Author: mozilla   File: __init__.py    Mozilla Public License 2.0 5 votes vote down vote up
def parse_test(test_id):
    id_parts = urlparse.urlsplit(test_id)
    dir_name, test_file = id_parts.path.rsplit("/", 1)
    if dir_name[0] == "/":
        dir_name = dir_name[1:]
    test_name = urlparse.urlunsplit((None, None, test_file, id_parts.query,
                                     id_parts.fragment))
    return dir_name, test_name 
Example 46
Project: wpt-sync   Author: mozilla   File: bug.py    Mozilla Public License 2.0 5 votes vote down vote up
def bug_number_from_url(url):
    if url is None:
        return None
    bugs = urlparse.parse_qs(urlparse.urlsplit(url).query).get("id")
    if bugs:
        return bugs[0] 
Example 47
Project: wpt-sync   Author: mozilla   File: bug.py    Mozilla Public License 2.0 5 votes vote down vote up
def id_from_url(self, url, bz_url=None):
        if bz_url is None:
            bz_url = self.bz_url
        if not url.startswith(bz_url):
            return None
        parts = urlparse.urlsplit(url)
        query = urlparse.parse_qs(parts.query)
        if "id" not in query or len(query["id"]) != 1:
            return None
        return query["id"][0] 
Example 48
Project: wpt-sync   Author: mozilla   File: gh.py    Mozilla Public License 2.0 5 votes vote down vote up
def __init__(self, token, url):
        self.gh = github.Github(token)
        self.repo_name = urlparse.urlsplit(url).path.lstrip("/")
        self.pr_cache = {}
        self._repo = None 
Example 49
Project: SuperDPF   Author: jmichelsen   File: super_dpf.py    GNU General Public License v3.0 5 votes vote down vote up
def _parse_feed_url(url):
        if urlsplit(url).path.startswith('/data/feed/'):
            return url
        elif urlsplit(url).netlock == 'picasaweb.google.com':
            response = requests.get(url)
            parsed = BeautifulStoneSoup(
                response.content, selfClosingTags=['meta', 'link', 'base'])

            for link in parsed.findAll('link'):
                if 'rel' in link:
                    if link['rel'] == 'alternate':
                        return link['href'] 
Example 50
Project: platzi-hello-gae   Author: xertica-cloud   File: testing.py    GNU General Public License v2.0 5 votes vote down vote up
def make_test_environ_builder(app, path='/', base_url=None, *args, **kwargs):
    """Creates a new test builder with some application defaults thrown in."""
    http_host = app.config.get('SERVER_NAME')
    app_root = app.config.get('APPLICATION_ROOT')
    if base_url is None:
        url = url_parse(path)
        base_url = 'http://%s/' % (url.netloc or http_host or 'localhost')
        if app_root:
            base_url += app_root.lstrip('/')
        if url.netloc:
            path = url.path
    return EnvironBuilder(path, base_url, *args, **kwargs) 
Example 51
Project: vulscan   Author: vulscanteam   File: funs.py    MIT License 5 votes vote down vote up
def url2ip(url):
    """
    works like turning 'http://baidu.com' => '180.149.132.47'
    """
    iport = urlsplit(url)[1].split(':')
    if len(iport) > 1:
        return gethostbyname(iport[0]), iport[1]
    return gethostbyname(iport[0]) 
Example 52
Project: sd-agent-core-plugins   Author: serverdensity   File: check.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def hostname_for_event(self, clean_server_name, agentConfig):
        """Return a reasonable hostname for a replset membership event to mention."""
        uri = urlsplit(clean_server_name)
        if '@' in uri.netloc:
            hostname = uri.netloc.split('@')[1].split(':')[0]
        else:
            hostname = uri.netloc.split(':')[0]
        if hostname == 'localhost':
            hostname = self.hostname
        return hostname 
Example 53
Project: sd-agent-core-plugins   Author: serverdensity   File: check.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _get_url_base(self, url):
        '''
        Return the base of a URL
        '''
        s = urlsplit(url)
        return urlunsplit([s.scheme, s.netloc, '', '', '']) 
Example 54
Project: sd-agent-core-plugins   Author: serverdensity   File: check.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _get_url_base(self, url):
        '''
        Return the base of a URL
        '''
        s = urlsplit(url)
        return urlunsplit([s.scheme, s.netloc, '', '', '']) 
Example 55
Project: sd-agent-core-plugins   Author: serverdensity   File: check.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _get_url_base(self, url):
        '''
        Return the base of a URL
        '''
        s = urlsplit(url)
        return urlunsplit([s.scheme, s.netloc, '', '', '']) 
Example 56
Project: docker-registry-client   Author: yodle   File: AuthorizationService.py    Apache License 2.0 5 votes vote down vote up
def __init__(self, registry, url="", auth=None, verify=False,
                 api_timeout=None):
        # Registry ip:port
        self.registry = urlsplit(registry).netloc
        # Service url, ip:port
        self.url = url
        # Authentication (user, password) or None. Used by request to do
        # basicauth
        self.auth = auth
        # Timeout for HTTP request
        self.api_timeout = api_timeout

        # Desired scope is the scope needed for the next operation on the
        # registry
        self.desired_scope = ""
        # Scope of the token we have
        self.scope = ""
        # Token used to authenticate
        self.token = ""
        # Boolean to enfore https checks. Used by request
        self.verify = verify

        # If we have no url then token are not required. get_new_token will not
        # be called
        if url:
            split = urlsplit(url)
            # user in url will take precedence over giver username
            if split.username and split.password:
                self.auth = (split.username, split.password)

            self.token_required = True
        else:
            self.token_required = False 
Example 57
Project: yosbot   Author: craisins   File: http.py    The Unlicense 5 votes vote down vote up
def prepare_url(url, queries):
    if queries:
        scheme, netloc, path, query, fragment = urlparse.urlsplit(url)

        query = dict(urlparse.parse_qsl(query))
        query.update(queries)
        query = urllib.urlencode(dict((to_utf8(key), to_utf8(value))
                                      for key, value in query.iteritems()))

        url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))

    return url 
Example 58
Project: tesismometro   Author: joapaspe   File: testing.py    MIT License 5 votes vote down vote up
def make_test_environ_builder(app, path='/', base_url=None, *args, **kwargs):
    """Creates a new test builder with some application defaults thrown in."""
    http_host = app.config.get('SERVER_NAME')
    app_root = app.config.get('APPLICATION_ROOT')
    if base_url is None:
        url = url_parse(path)
        base_url = 'http://%s/' % (url.netloc or http_host or 'localhost')
        if app_root:
            base_url += app_root.lstrip('/')
        if url.netloc:
            path = url.path
    return EnvironBuilder(path, base_url, *args, **kwargs) 
Example 59
Project: Graduation_Design-Distributed_Web_Spider   Author: weiyu666   File: yumdama.py    MIT License 5 votes vote down vote up
def post_url(url, fields, files=[]):
    urlparts = urlparse.urlsplit(url)
    return post_multipart(urlparts[1], urlparts[2], fields, files) 
Example 60
Project: SmartProxyPool   Author: 1again   File: base.py    MIT License 5 votes vote down vote up
def process_line(self, data):
        line = data.split(SP)
        if self.type == HttpParser.types.REQUEST_PARSER:
            self.method = line[0].upper()
            self.url = urlparse.urlsplit(line[1])
            self.version = line[2]
        else:
            self.version = line[0]
            self.code = line[1]
            self.reason = b' '.join(line[2:])
        self.state = HttpParser.states.LINE_RCVD 
Example 61
Project: jbox   Author: jpush   File: testing.py    MIT License 5 votes vote down vote up
def make_test_environ_builder(app, path='/', base_url=None, *args, **kwargs):
    """Creates a new test builder with some application defaults thrown in."""
    http_host = app.config.get('SERVER_NAME')
    app_root = app.config.get('APPLICATION_ROOT')
    if base_url is None:
        url = url_parse(path)
        base_url = 'http://%s/' % (url.netloc or http_host or 'localhost')
        if app_root:
            base_url += app_root.lstrip('/')
        if url.netloc:
            path = url.path
            if url.query:
                path += '?' + url.query
    return EnvironBuilder(path, base_url, *args, **kwargs) 
Example 62
Project: neo4j-social-network   Author: bestvibes   File: testing.py    MIT License 5 votes vote down vote up
def make_test_environ_builder(app, path='/', base_url=None, *args, **kwargs):
    """Creates a new test builder with some application defaults thrown in."""
    http_host = app.config.get('SERVER_NAME')
    app_root = app.config.get('APPLICATION_ROOT')
    if base_url is None:
        url = url_parse(path)
        base_url = 'http://%s/' % (url.netloc or http_host or 'localhost')
        if app_root:
            base_url += app_root.lstrip('/')
        if url.netloc:
            path = url.path
    return EnvironBuilder(path, base_url, *args, **kwargs) 
Example 63
Project: neo4j-social-network   Author: bestvibes   File: testing.py    MIT License 5 votes vote down vote up
def make_test_environ_builder(app, path='/', base_url=None, *args, **kwargs):
    """Creates a new test builder with some application defaults thrown in."""
    http_host = app.config.get('SERVER_NAME')
    app_root = app.config.get('APPLICATION_ROOT')
    if base_url is None:
        url = url_parse(path)
        base_url = 'http://%s/' % (url.netloc or http_host or 'localhost')
        if app_root:
            base_url += app_root.lstrip('/')
        if url.netloc:
            path = url.path
    return EnvironBuilder(path, base_url, *args, **kwargs) 
Example 64
Project: RAT-via-Telegram   Author: mvrozanti   File: proxy.py    MIT License 5 votes vote down vote up
def process_line(self, data):
        line = data.split(SP)
        if self.type == HTTP_REQUEST_PARSER:
            self.method = line[0].upper()
            self.url = urlparse.urlsplit(line[1])
            self.version = line[2]
        else:
            self.version = line[0]
            self.code = line[1]
            self.reason = b' '.join(line[2:])
        self.state = HTTP_PARSER_STATE_LINE_RCVD 
Example 65
Project: httpfs   Author: bgaifullin   File: httpfs.py    GNU General Public License v2.0 5 votes vote down vote up
def readdir(self, path, fh):
        try:
            url = self._url(path)
            logging.debug("readdir %s (%s)", path, url)
            response = self.opener.open(url)
            yield '.'
            yield '..'

            content = list()
            parser = FilesParser(urlparse.urlsplit(url).path, content.append)
            while True:
                data = response.read(1024)
                if not data:
                    break
                parser.feed(data)

                for c in content:
                    yield c

                del content[:]

        except urllib.HTTPError as e:
            logging.error("readdir %r", e)
            raise self._error(e)
        except Exception as e:
            logging.exception("getattr %r", e)
            raise FuseOSError(errno.EAGAIN) 
Example 66
Project: grand_central   Author: amiracle   File: script.py    MIT License 5 votes vote down vote up
def service(self):
        """ Returns a Splunk service object for this script invocation.

        The service object is created from the Splunkd URI and session key
        passed to the command invocation on the modular input stream. It is
        available as soon as the :code:`Script.stream_events` method is
        called.

        :return: :class:splunklib.client.Service. A value of None is returned,
        if you call this method before the :code:`Script.stream_events` method
        is called.

        """
        if self._service is not None:
            return self._service

        if self._input_definition is None:
            return None

        splunkd_uri = self._input_definition.metadata["server_uri"]
        session_key = self._input_definition.metadata["session_key"]

        splunkd = urlsplit(splunkd_uri, allow_fragments=False)

        self._service = Service(
            scheme=splunkd.scheme,
            host=splunkd.hostname,
            port=splunkd.port,
            token=session_key,
        )

        return self._service 
Example 67
Project: SplunkForPCAP   Author: DanielSchwartz1   File: script.py    MIT License 5 votes vote down vote up
def service(self):
        """ Returns a Splunk service object for this script invocation.

        The service object is created from the Splunkd URI and session key
        passed to the command invocation on the modular input stream. It is
        available as soon as the :code:`Script.stream_events` method is
        called.

        :return: :class:splunklib.client.Service. A value of None is returned,
        if you call this method before the :code:`Script.stream_events` method
        is called.

        """
        if self._service is not None:
            return self._service

        if self._input_definition is None:
            return None

        splunkd_uri = self._input_definition.metadata["server_uri"]
        session_key = self._input_definition.metadata["session_key"]

        splunkd = urlsplit(splunkd_uri, allow_fragments=False)

        self._service = Service(
            scheme=splunkd.scheme,
            host=splunkd.hostname,
            port=splunkd.port,
            token=session_key,
        )

        return self._service 
Example 68
Project: TAF   Author: couchbaselabs   File: iri2uri.py    Apache License 2.0 5 votes vote down vote up
def iri2uri(uri):
    """Convert an IRI to a URI. Note that IRIs must be
    passed in a unicode strings. That is, do not utf-8 encode
    the IRI before passing it into the function."""
    if isinstance(uri, unicode):
        (scheme, authority, path, query, fragment) = urlparse.urlsplit(uri)
        authority = authority.encode('idna')
        # For each character in 'ucschar' or 'iprivate'
        #  1. encode as utf-8
        #  2. then %-encode each octet of that utf-8
        uri = urlparse.urlunsplit((scheme, authority, path, query, fragment))
        uri = "".join([encode(c) for c in uri])
    return uri 
Example 69
Project: AneMo   Author: jspargo   File: testing.py    GNU General Public License v2.0 5 votes vote down vote up
def make_test_environ_builder(app, path='/', base_url=None, *args, **kwargs):
    """Creates a new test builder with some application defaults thrown in."""
    http_host = app.config.get('SERVER_NAME')
    app_root = app.config.get('APPLICATION_ROOT')
    if base_url is None:
        url = url_parse(path)
        base_url = 'http://%s/' % (url.netloc or http_host or 'localhost')
        if app_root:
            base_url += app_root.lstrip('/')
        if url.netloc:
            path = url.path
    return EnvironBuilder(path, base_url, *args, **kwargs) 
Example 70
Project: bugatsinho.github.io   Author: bugatsinho   File: base.py    GNU General Public License v3.0 5 votes vote down vote up
def unwrap_30x(self, uri, timeout=10):

        domain = urlsplit(uri).netloc
        self._timeout = timeout

        loop_counter = 0
        try:

            if loop_counter > 5:
                raise ValueError("Infinitely looping redirect from URL: '%s'" % (uri,))

            # headers stop t.co from working so omit headers if this is a t.co link
            if domain == 't.co':
                r = requests.get(uri, timeout=self._timeout)
                return r.url, r.status_code
            # p.ost.im uses meta http refresh to redirect.
            if domain == 'p.ost.im':
                r = requests.get(uri, headers=HTTP_HEADER, timeout=self._timeout)
                uri = re.findall(r'.*url\=(.*?)\"\.*', r.text)[0]
                return uri, r.status_code
            else:

                while True:
                    try:
                        r = requests.head(uri, headers=HTTP_HEADER, timeout=self._timeout)
                    except (requests.exceptions.InvalidSchema, requests.exceptions.InvalidURL):
                        return uri, -1

                    retries = 0
                    if 'location' in r.headers and retries < self._maxretries:
                        r = requests.head(r.headers['location'])
                        uri = r.url
                        loop_counter += 1
                        retries = retries + 1
                    else:
                        return r.url, r.status_code


        except Exception as e:
            return uri, str(e) 
Example 71
Project: bugatsinho.github.io   Author: bugatsinho   File: base.py    GNU General Public License v3.0 5 votes vote down vote up
def _clear_google_outbound_proxy(self, url):
        '''
        So google proxies all their outbound links through a redirect so they can detect outbound links.
        This call strips them out if they are present.

        This is useful for doing things like parsing google search results, or if you're scraping google
        docs, where google inserts hit-counters on all outbound links.
        '''

        # This is kind of hacky, because we need to check both the netloc AND
        # part of the path. We could use urllib.parse.urlsplit, but it's
        # easier and just as effective to use string checks.
        if url.startswith("http://www.google.com/url?") or \
                url.startswith("https://www.google.com/url?"):

            qs = urlparse(url).query
            query = parse_qs(qs)

            if "q" in query:  # Google doc outbound links (maybe blogspot, too)
                return True, query["q"].pop()
            elif "url" in query:  # Outbound links from google searches
                return True, query["url"].pop()
            else:
                raise ValueError("Google outbound proxy URL without a target url ('%s')?" % url)

        return False, url 
Example 72
Project: pyblish-win   Author: pyblish   File: test_urlparse.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def checkRoundtrips(self, url, parsed, split):
        result = urlparse.urlparse(url)
        self.assertEqual(result, parsed)
        t = (result.scheme, result.netloc, result.path,
             result.params, result.query, result.fragment)
        self.assertEqual(t, parsed)
        # put it back together and it should be the same
        result2 = urlparse.urlunparse(result)
        self.assertEqual(result2, url)
        self.assertEqual(result2, result.geturl())

        # the result of geturl() is a fixpoint; we can always parse it
        # again to get the same result:
        result3 = urlparse.urlparse(result.geturl())
        self.assertEqual(result3.geturl(), result.geturl())
        self.assertEqual(result3,          result)
        self.assertEqual(result3.scheme,   result.scheme)
        self.assertEqual(result3.netloc,   result.netloc)
        self.assertEqual(result3.path,     result.path)
        self.assertEqual(result3.params,   result.params)
        self.assertEqual(result3.query,    result.query)
        self.assertEqual(result3.fragment, result.fragment)
        self.assertEqual(result3.username, result.username)
        self.assertEqual(result3.password, result.password)
        self.assertEqual(result3.hostname, result.hostname)
        self.assertEqual(result3.port,     result.port)

        # check the roundtrip using urlsplit() as well
        result = urlparse.urlsplit(url)
        self.assertEqual(result, split)
        t = (result.scheme, result.netloc, result.path,
             result.query, result.fragment)
        self.assertEqual(t, split)
        result2 = urlparse.urlunsplit(result)
        self.assertEqual(result2, url)
        self.assertEqual(result2, result.geturl())

        # check the fixpoint property of re-parsing the result of geturl()
        result3 = urlparse.urlsplit(result.geturl())
        self.assertEqual(result3.geturl(), result.geturl())
        self.assertEqual(result3,          result)
        self.assertEqual(result3.scheme,   result.scheme)
        self.assertEqual(result3.netloc,   result.netloc)
        self.assertEqual(result3.path,     result.path)
        self.assertEqual(result3.query,    result.query)
        self.assertEqual(result3.fragment, result.fragment)
        self.assertEqual(result3.username, result.username)
        self.assertEqual(result3.password, result.password)
        self.assertEqual(result3.hostname, result.hostname)
        self.assertEqual(result3.port,     result.port) 
Example 73
Project: Ansible-Example-AB2018   Author: umit-ozturk   File: _iri.py    MIT License 4 votes vote down vote up
def uri_to_iri(value):
    """
    Converts an ASCII URI byte string into a unicode IRI

    :param value:
        An ASCII-encoded byte string of the URI

    :return:
        A unicode string of the IRI
    """

    if not isinstance(value, byte_cls):
        raise TypeError(unwrap(
            '''
            value must be a byte string, not %s
            ''',
            type_name(value)
        ))

    parsed = urlsplit(value)

    scheme = parsed.scheme
    if scheme is not None:
        scheme = scheme.decode('ascii')

    username = _urlunquote(parsed.username, remap=[':', '@'])
    password = _urlunquote(parsed.password, remap=[':', '@'])
    hostname = parsed.hostname
    if hostname:
        hostname = hostname.decode('idna')
    port = parsed.port
    if port and not isinstance(port, int_types):
        port = port.decode('ascii')

    netloc = ''
    if username is not None:
        netloc += username
        if password:
            netloc += ':' + password
        netloc += '@'
    if hostname is not None:
        netloc += hostname
    if port is not None:
        netloc += ':' + str_cls(port)

    path = _urlunquote(parsed.path, remap=['/'], preserve=True)
    query = _urlunquote(parsed.query, remap=['&', '='], preserve=True)
    fragment = _urlunquote(parsed.fragment)

    return urlunsplit((scheme, netloc, path, query, fragment)) 
Example 74
Project: Splunk_CBER_App   Author: MHaggis   File: main.py    MIT License 4 votes vote down vote up
def get_flask_server():
    flask_server = Flask('liveresponse')

    @flask_server.route('/', methods=['GET', 'POST'])
    def index():
        with live_response_lock:
            return render_template('index.html', sensors=live_response_results.keys())

    @flask_server.route('/sensor_result/<sensor_id>')
    def sensor_result(sensor_id):
        sensor_id = int(sensor_id)
        with live_response_lock:
            results = live_response_results.get(sensor_id, {})
            if 'results' in results:
                return render_template('sensor_result.html', results=results['results'], sensor_id=sensor_id)
            else:
                return render_template('no_such_result.html')

    @flask_server.route('/browser_history/<sensor_id>')
    def browser_history(sensor_id):
        sensor_id = int(sensor_id)
        with live_response_lock:
            results = live_response_results.get(sensor_id, {})
            if 'results' in results and 'browser_history' in results['results']:
                with NamedTemporaryFile(delete=False) as tf:
                    tf.write(results['results']['browser_history'])
                    tf.close()
                    db = sqlite3.connect(tf.name)
                    db.row_factory = sqlite3.Row
                    cur = db.cursor()
                    cur.execute("SELECT url, title, datetime(last_visit_time / 1000000 + (strftime('%s', '1601-01-01')), 'unixepoch') as last_visit_time FROM urls ORDER BY last_visit_time DESC")
                    urls = [dict(u) for u in cur.fetchall()]

                    for url in urls:
                        (scheme, netloc, _, _, _) = urlparse.urlsplit(url["url"])
                        url["favicon"] = urlparse.urlunsplit((scheme, netloc, "favicon.ico", "", ""))

                    return render_template('browser_history.html', urls=urls)

            return render_template('no_such_result.html')

    t = threading.Thread(target=flask_server.run, args=['127.0.0.1', 7982])
    t.daemon = True
    return t 
Example 75
Project: Splunk_CBER_App   Author: MHaggis   File: search_command.py    MIT License 4 votes vote down vote up
def service(self):
        """ Returns a Splunk service object for this command invocation or None.

        The service object is created from the Splunkd URI and authentication token passed to the command invocation in
        the search results info file. This data is not passed to a command invocation by default. You must request it by
        specifying this pair of configuration settings in commands.conf:

           .. code-block:: python
               enableheader = true
               requires_srinfo = true

        The :code:`enableheader` setting is :code:`true` by default. Hence, you need not set it. The
        :code:`requires_srinfo` setting is false by default. Hence, you must set it.

        :return: :class:`splunklib.client.Service`, if :code:`enableheader` and :code:`requires_srinfo` are both
        :code:`true`. Otherwise, if either :code:`enableheader` or :code:`requires_srinfo` are :code:`false`, a value
        of :code:`None` is returned.

        """
        if self._service is not None:
            return self._service

        metadata = self._metadata

        if metadata is None:
            return None

        try:
            searchinfo = self._metadata.searchinfo
        except AttributeError:
            return None

        splunkd_uri = searchinfo.splunkd_uri

        if splunkd_uri is None:
            return None

        uri = urlsplit(splunkd_uri, allow_fragments=False)

        self._service = Service(
            scheme=uri.scheme, host=uri.hostname, port=uri.port, app=searchinfo.app, token=searchinfo.session_key)

        return self._service

    # endregion

    # region Methods 
Example 76
Project: misp42splunk   Author: remg427   File: search_command.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def service(self):
        """ Returns a Splunk service object for this command invocation or None.

        The service object is created from the Splunkd URI and authentication token passed to the command invocation in
        the search results info file. This data is not passed to a command invocation by default. You must request it by
        specifying this pair of configuration settings in commands.conf:

           .. code-block:: python
               enableheader = true
               requires_srinfo = true

        The :code:`enableheader` setting is :code:`true` by default. Hence, you need not set it. The
        :code:`requires_srinfo` setting is false by default. Hence, you must set it.

        :return: :class:`splunklib.client.Service`, if :code:`enableheader` and :code:`requires_srinfo` are both
        :code:`true`. Otherwise, if either :code:`enableheader` or :code:`requires_srinfo` are :code:`false`, a value
        of :code:`None` is returned.

        """
        if self._service is not None:
            return self._service

        metadata = self._metadata

        if metadata is None:
            return None

        try:
            searchinfo = self._metadata.searchinfo
        except AttributeError:
            return None

        splunkd_uri = searchinfo.splunkd_uri

        if splunkd_uri is None:
            return None

        uri = urlsplit(splunkd_uri, allow_fragments=False)

        self._service = Service(
            scheme=uri.scheme, host=uri.hostname, port=uri.port, app=searchinfo.app, token=searchinfo.session_key)

        return self._service

    # endregion

    # region Methods 
Example 77
Project: TornadoWeb   Author: VxCoder   File: simple_httpclient.py    Apache License 2.0 4 votes vote down vote up
def __init__(self, io_loop, client, request, release_callback,
                 final_callback, max_buffer_size, tcp_client,
                 max_header_size, max_body_size):
        self.start_time = io_loop.time()
        self.io_loop = io_loop
        self.client = client
        self.request = request
        self.release_callback = release_callback
        self.final_callback = final_callback
        self.max_buffer_size = max_buffer_size
        self.tcp_client = tcp_client
        self.max_header_size = max_header_size
        self.max_body_size = max_body_size
        self.code = None
        self.headers = None
        self.chunks = []
        self._decompressor = None
        # Timeout handle returned by IOLoop.add_timeout
        self._timeout = None
        self._sockaddr = None
        with stack_context.ExceptionStackContext(self._handle_exception):
            self.parsed = urlparse.urlsplit(_unicode(self.request.url))
            if self.parsed.scheme not in ("http", "https"):
                raise ValueError("Unsupported url scheme: %s" %
                                 self.request.url)
            # urlsplit results have hostname and port results, but they
            # didn't support ipv6 literals until python 2.7.
            netloc = self.parsed.netloc
            if "@" in netloc:
                userpass, _, netloc = netloc.rpartition("@")
            host, port = httputil.split_host_and_port(netloc)
            if port is None:
                port = 443 if self.parsed.scheme == "https" else 80
            if re.match(r'^\[.*\]$', host):
                # raw ipv6 addresses in urls are enclosed in brackets
                host = host[1:-1]
            self.parsed_hostname = host  # save final host for _on_connect

            if request.allow_ipv6 is False:
                af = socket.AF_INET
            else:
                af = socket.AF_UNSPEC

            ssl_options = self._get_ssl_options(self.parsed.scheme)

            timeout = min(self.request.connect_timeout, self.request.request_timeout)
            if timeout:
                self._timeout = self.io_loop.add_timeout(
                    self.start_time + timeout,
                    stack_context.wrap(functools.partial(self._on_timeout, "while connecting")))
            self.tcp_client.connect(host, port, af=af,
                                    ssl_options=ssl_options,
                                    max_buffer_size=self.max_buffer_size,
                                    callback=self._on_connect) 
Example 78
Project: SuperDPF   Author: jmichelsen   File: super_dpf.py    GNU General Public License v3.0 4 votes vote down vote up
def sync(self, verify_local):
        log.info('Syncing {}'.format(self.service_name))
        full_path = '{}/{}'.format(PATHS['photos'], self.subdir)
        duplicates = list()
        cloud_filelist = list()

        response = requests.get(self.settings.get('feed_url'))
        tags = BeautifulStoneSoup(
            response.content).findAll('media:content')
        for tag in tags:
            if tag.get('url', False):
                filename = os.path.join(
                    full_path, os.path.basename(urlsplit(tag['url'])[2]))
                cloud_filelist.append(filename)
                if not os.path.isfile(filename):
                    log.info(
                        'Getting img at {}'.format(tag['url']))
                    response = requests.get(tag['url'], stream=True)
                    try:
                        img = Image.open(StringIO(response.content))
                        img.save('{}'.format(filename), optimize=True,
                                 progressive=True, quality=100,
                                 subsampling=0)
                    except IOError as e:
                        log.error(e)
                else:
                    duplicates.append(tag)

        if duplicates:
            log.info(
                'Skipped {} duplicate images from {}.'.format(
                    len(duplicates), self.service_name))

        if verify_local:
            local_filelist = \
                [os.path.join(full_path, f) for f in os.listdir(full_path)]
            self.verify_local(cloud_filelist, local_filelist)
            log.info('Local {} files verified successfully.'.format(
                self.service_name))

        if not cloud_filelist:
            log.info('{} has nothing to sync.'.format(self.service_name))

        log.info('Done syncing {}'.format(self.service_name)) 
Example 79
Project: grand_central   Author: amiracle   File: search_command.py    MIT License 4 votes vote down vote up
def service(self):
        """ Returns a Splunk service object for this command invocation or None.

        The service object is created from the Splunkd URI and authentication token passed to the command invocation in
        the search results info file. This data is not passed to a command invocation by default. You must request it by
        specifying this pair of configuration settings in commands.conf:

           .. code-block:: python
               enableheader = true
               requires_srinfo = true

        The :code:`enableheader` setting is :code:`true` by default. Hence, you need not set it. The
        :code:`requires_srinfo` setting is false by default. Hence, you must set it.

        :return: :class:`splunklib.client.Service`, if :code:`enableheader` and :code:`requires_srinfo` are both
        :code:`true`. Otherwise, if either :code:`enableheader` or :code:`requires_srinfo` are :code:`false`, a value
        of :code:`None` is returned.

        """
        if self._service is not None:
            return self._service

        metadata = self._metadata

        if metadata is None:
            return None

        try:
            searchinfo = self._metadata.searchinfo
        except AttributeError:
            return None

        splunkd_uri = searchinfo.splunkd_uri

        if splunkd_uri is None:
            return None

        uri = urlsplit(splunkd_uri, allow_fragments=False)

        self._service = Service(
            scheme=uri.scheme, host=uri.hostname, port=uri.port, app=searchinfo.app, token=searchinfo.session_key)

        return self._service

    # endregion

    # region Methods 
Example 80
Project: SplunkForPCAP   Author: DanielSchwartz1   File: search_command.py    MIT License 4 votes vote down vote up
def service(self):
        """ Returns a Splunk service object for this command invocation or None.

        The service object is created from the Splunkd URI and authentication token passed to the command invocation in
        the search results info file. This data is not passed to a command invocation by default. You must request it by
        specifying this pair of configuration settings in commands.conf:

           .. code-block:: python
               enableheader = true
               requires_srinfo = true

        The :code:`enableheader` setting is :code:`true` by default. Hence, you need not set it. The
        :code:`requires_srinfo` setting is false by default. Hence, you must set it.

        :return: :class:`splunklib.client.Service`, if :code:`enableheader` and :code:`requires_srinfo` are both
        :code:`true`. Otherwise, if either :code:`enableheader` or :code:`requires_srinfo` are :code:`false`, a value
        of :code:`None` is returned.

        """
        if self._service is not None:
            return self._service

        metadata = self._metadata

        if metadata is None:
            return None

        try:
            searchinfo = self._metadata.searchinfo
        except AttributeError:
            return None

        splunkd_uri = searchinfo.splunkd_uri

        if splunkd_uri is None:
            return None

        uri = urlsplit(splunkd_uri, allow_fragments=False)

        self._service = Service(
            scheme=uri.scheme, host=uri.hostname, port=uri.port, app=searchinfo.app, token=searchinfo.session_key)

        return self._service

    # endregion

    # region Methods