Python urlparse.ParseResult() Examples

The following are code examples for showing how to use urlparse.ParseResult(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: feedthefox   Author: mozilla   File: helpers.py    Mozilla Public License 2.0 7 votes vote down vote up
def urlparams(url_, hash=None, **query):
    """Add a fragment and/or query paramaters to a URL.

    New query params will be appended to exising parameters, except duplicate
    names, which will be replaced.
    """
    url = urlparse.urlparse(url_)
    fragment = hash if hash is not None else url.fragment

    # Use dict(parse_qsl) so we don't get lists of values.
    query_dict = dict(urlparse.parse_qsl(url.query))
    query_dict.update(query)

    query_string = urlencode(
        [(k, v) for k, v in query_dict.items() if v is not None])
    new = urlparse.ParseResult(url.scheme, url.netloc, url.path, url.params,
                               query_string, fragment)
    return new.geturl() 
Example 2
Project: circlecli   Author: particledecay   File: circleapi.py    MIT License 6 votes vote down vote up
def _build_url(self, endpoint, params={}):
        """Return the full URL for the desired endpoint.

        Args:
            endpoint (str): the API endpoint after base URL
            params (dict): any params to include in the request

        Returns:
            (str) the full URL of the request
        """
        new_params = {'circle-token': self._token}
        new_params.update(params)

        parsed_url = urlparse(self._base_url)
        new_parse = ParseResult(scheme=parsed_url.scheme, netloc=parsed_url.netloc,
                                path='/'.join((parsed_url.path, endpoint)),
                                params='', query=urlencode(new_params),
                                fragment='')

        return urlunparse(new_parse) 
Example 3
Project: stormtrooper   Author: CompileInc   File: org_tld_plugin.py    MIT License 6 votes vote down vote up
def normalize_website(cls, w):
        from django.core.validators import EMPTY_VALUES
        from urlparse import urlparse, urlunparse, ParseResult
        w = w.decode('utf-8')
        if w in EMPTY_VALUES:
            return None
        w = w.lower().strip()
        if not w.startswith('http://') and not w.startswith('https://'):
            w = 'http://' + w.lstrip('/')
        try:
            parsed = urlparse(w)
        except ValueError as e:
            return None
        else:
            new_parsed = ParseResult(scheme='http',
                                     netloc=cls.get_website_tld(w),
                                     path=parsed.path.rstrip('/'),
                                     params='',
                                     query=parsed.query,
                                     fragment='')
            return urlunparse(new_parsed) 
Example 4
Project: encompass   Author: mazaclub   File: main_window.py    GNU General Public License v3.0 6 votes vote down vote up
def update_receive_qr(self):
        import urlparse, urllib
        addr = str(self.receive_address_e.text())
        amount = None
        message = None
        url = None
        amount = self.receive_amount_e.get_amount()
        message = unicode(self.receive_message_e.text()).encode('utf8')
        self.save_request_button.setEnabled((amount is not None) or (message != ""))
        uri_scheme = chainkey.chainparams.get_active_chain().coin_name.lower()
        if addr:
            query = []
            if amount:
                query.append('amount=%s'%format_satoshis(amount))
            if message:
                query.append('message=%s'%urllib.quote(message))
            p = urlparse.ParseResult(scheme=uri_scheme, netloc='', path=addr, params='', query='&'.join(query), fragment='')
            url = urlparse.urlunparse(p)
        else:
            url = ""
        self.receive_qr.setData(url)
        if self.qr_window:
            self.qr_window.set_content(addr, amount, message, url) 
Example 5
Project: oxford-cli   Author: irjudson   File: oxford.py    MIT License 6 votes vote down vote up
def analyze_image(ctx, features, image_path):
    """Analyze an image with Oxford."""
    vision_analysis_url = ctx.obj['oxford_url'] + '/analyses'
    params = {
    }
    headers = {
    'Ocp-Apim-Subscription-Key' : ctx.obj['apikeys']['vision']
    }
    if type(image_path) is file:
        headers['Content-type'] = 'application/octet-stream'
        payload = image_path.read()
    if type(image_path) is urlparse.ParseResult:
        headers['Content-type'] = 'application/json'
        payload = json.dumps({ 'Url' : image_path.geturl() })
    try:
        resp = requests.post(vision_analysis_url, params=params, data=payload, headers=headers)
        if resp.status_code == 200:
            print json.dumps(resp.json(), sort_keys=True, indent=2, separators=(',', ': '))
        else:
            print resp.json()['message']
    except Exception as e:
        print e

# use vision api to make a thumbnail 
Example 6
Project: requests-crtauth   Author: spotify   File: __init__.py    Apache License 2.0 6 votes vote down vote up
def _auth_url(url):
    """Returns the authentication URL based on the URL originally requested.

    Args:
        url: String, the original request.url

    Returns:
        String, the authentication URL.
    """
    parsed_url = urlparse.urlparse(url)
    parsed_auth_url = urlparse.ParseResult(parsed_url.scheme,
                                           parsed_url.netloc,
                                           '/_auth',
                                           parsed_url.params,
                                           parsed_url.query,
                                           parsed_url.fragment)
    return parsed_auth_url.geturl() 
Example 7
Project: crispys_webkit   Author: crispycret   File: urls.py    MIT License 6 votes vote down vote up
def print_overview(self, url=None):
		if isinstance(url, urlparse.ParseResult):
			url.host = url.netloc
			url.url = url.geturl()
		else:
			url = self
	
		print '\nOverview:\n'
		print '\tUrl: %s\n' % url.url
		print '\tScheme: %s' % url.scheme
		print '\tHost: %s' % url.host
		print '\tPath: %s' % url.path	
		print '\tQuery: %s' % url.query
		print '\tFragment: %s\n' % url.fragment

	################################################################ 
Example 8
Project: harpoon   Author: evilhero   File: common.py    GNU General Public License v3.0 6 votes vote down vote up
def join_uri(base, uri, construct=True):
    p_uri = urlparse.urlparse(uri)

    # Return if there is nothing to join
    if not p_uri.path:
        return base

    scheme, netloc, path, params, query, fragment = urlparse.urlparse(base)

    # Switch to 'uri' parts
    _, _, _, params, query, fragment = p_uri

    path = join_path(path, p_uri.path)

    result = urlparse.ParseResult(scheme, netloc, path, params, query, fragment)

    if not construct:
        return result

    # Construct from parts
    return urlparse.urlunparse(result) 
Example 9
Project: encompass   Author: martexcoin   File: main_window.py    GNU General Public License v3.0 6 votes vote down vote up
def update_receive_qr(self):
        import urlparse, urllib
        addr = str(self.receive_address_e.text())
        amount = None
        message = None
        url = None
        amount = self.receive_amount_e.get_amount()
        message = unicode(self.receive_message_e.text()).encode('utf8')
        self.save_request_button.setEnabled((amount is not None) or (message != ""))
        uri_scheme = self.active_chain.coin_name.lower()
        if addr:
            query = []
            if amount:
                query.append('amount=%s'%format_satoshis(amount))
            if message:
                query.append('message=%s'%urllib.quote(message))
            p = urlparse.ParseResult(scheme=uri_scheme, netloc='', path=addr, params='', query='&'.join(query), fragment='')
            url = urlparse.urlunparse(p)
        else:
            url = ""
        self.receive_qr.setData(url)
        if self.qr_window:
            self.qr_window.set_content(addr, amount, message, url) 
Example 10
Project: openhatch   Author: campbe13   File: url.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def parse_url(url, encoding=None):
    """Return urlparsed url from the given argument (which could be an already
    parsed url)
    """
    return url if isinstance(url, urlparse.ParseResult) else \
        urlparse.urlparse(unicode_to_str(url, encoding)) 
Example 11
Project: python-indieweb   Author: bear   File: test_login.py    MIT License 5 votes vote down vote up
def runTest(self):
        me            = 'http://127.0.0.1:9999'
        redirect_uri  = 'http://localhost:9999/success'
        client_id     = 'tester'
        r = ninka.indieauth.discoverAuthEndpoints(me)

        assert 'authorization_endpoint' in r

        authURL = None
        for url in r['authorization_endpoint']:
            authURL = url
            break

        assert authURL is not None
        assert authURL.scheme == 'https'
        assert authURL.netloc == 'indieauth.com'
        assert authURL.path   == '/auth'

        url = ParseResult(authURL.scheme, 
                          authURL.netloc,
                          authURL.path,
                          authURL.params,
                          urllib.urlencode({ 'me':            me,
                                             'redirect_uri':  redirect_uri,
                                             'client_id':     client_id,
                                             'scope':         'post',
                                             'response_type': 'id'
                                           }),
                          authURL.fragment).geturl()

        r = requests.get(url, verify=True)

        assert r.status_code == 200 
Example 12
Project: python-indieweb   Author: bear   File: test_micropub.py    MIT License 5 votes vote down vote up
def runTest(self):
        me = 'http://127.0.0.1:9999'
        r = ninka.discoverMicropubEndpoints(me)

        assert 'micropub' in r

        micropubURL = None
        for url in r['micropub']:
            micropubURL = url
            break

        assert micropubURL is not None
        assert micropubURL.scheme == 'http'
        assert micropubURL.netloc == '127.0.0.1:9999'
        assert micropubURL.path   == '/micropub'

        # url = ParseResult(micropubURL.scheme, 
        #                   micropubURL.netloc,
        #                   micropubURL.path,
        #                   micropubURL.params,
        #                   urllib.urlencode({ 'me':            me,
        #                                      'redirect_uri':  redirect_uri,
        #                                      'client_id':     client_id,
        #                                      'scope':         'post',
        #                                      'response_type': 'id'
        #                                    }),
        #                   micropubURL.fragment).geturl()

        # r = requests.get(url, verify=True)

        # assert r.status_code == 200


# POST /micropub HTTP/1.1
# Host: bear.im
# Accept: */*
# Authorization: Bearer 159d4823-de7d-4717-9e4b-401da86a413b
# Content-Length: 30
# Content-Type: application/x-www-form-urlencoded

# h=entry&content=test&slug=test 
Example 13
Project: openode   Author: JvGinkel   File: file_utils.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def store_file(file_object, file_name_prefix=''):
    """Creates an instance of django's file storage
    object based on the file-like object,
    returns the storage object, file name, file url
    """
    file_name = str(
                    time.time()
                ).replace(
                    '.',
                    str(random.randint(0, 100000))
                ) + os.path.splitext(file_object.name)[1].lower()
    file_name = file_name_prefix + file_name

    file_storage = get_storage_class()()
    # use default storage to store file
    file_storage.save(file_name, file_object)

    file_url = file_storage.url(file_name)
    parsed_url = urlparse.urlparse(file_url)
    file_url = urlparse.urlunparse(
        urlparse.ParseResult(
            parsed_url.scheme,
            parsed_url.netloc,
            parsed_url.path,
            '', '', ''
        )
    )

    return file_storage, file_name, file_url 
Example 14
Project: openode   Author: JvGinkel   File: url_utils.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def strip_path(url):
    """srips path, params and hash fragments of the url"""
    purl = urlparse.urlparse(url)
    return urlparse.urlunparse(
        urlparse.ParseResult(
            purl.scheme,
            purl.netloc,
            '', '', '', ''
        )
    ) 
Example 15
Project: daenerys   Author: dongweiming   File: app.py    Apache License 2.0 5 votes vote down vote up
def validate_url(self, url):
        url_path = urllib.quote(url.path, safe=b"/%")
        url_query = urllib.quote(url.query, safe=b"?=&")

        url = ParseResult(url.scheme, url.netloc, url_path,
                          url.params, url_query, url.fragment)

        has_hostname = url.hostname is not None and len(url.hostname) > 0
        has_http_scheme = url.scheme in ("http", "https")
        has_path = not len(url.path) or url.path.startswith("/")

        if not (has_hostname and has_http_scheme and has_path):
            raise NotSupported("invalid url: %s" % repr(url))

        return url 
Example 16
Project: oxford-cli   Author: irjudson   File: oxford.py    MIT License 5 votes vote down vote up
def detect(ctx, analyzesfacelandmarks, analyzesage, analyzesgender, analyzesheadpose, image_path):
    """Detect faces in an image provided on the command line."""
    detection_path = '/detections'
    params = {
    'analyzesFaceLandmarks' : str(analyzesfacelandmarks).lower(),
    'analyzesAge' : str(analyzesage).lower(),
    'analyzesGender' : str(analyzesgender).lower(),
    'analyzesHeadPose' : str(analyzesheadpose).lower()
    }
    headers = {
    'Ocp-Apim-Subscription-Key' : ctx.obj['apikeys']['face']
    }
    face_detect_url = ctx.obj['oxford_url'] + detection_path
    if type(image_path) is file:
        headers['Content-type'] = 'application/octet-stream'
        payload = image_path.read()
    if type(image_path) is urlparse.ParseResult:
        headers['Content-type'] = 'application/json'
        payload = json.dumps({ 'url' : image_path.geturl() })
    try:
        resp = requests.post(face_detect_url, params=params, data=payload, headers=headers)
        if resp.status_code == 200:
            print json.dumps(resp.json(), sort_keys=True, indent=2, separators=(',', ': '))
        else:
            print resp.json()['message']
    except Exception as e:
        print e
                
# find similar faces 
Example 17
Project: oxford-cli   Author: irjudson   File: oxford.py    MIT License 5 votes vote down vote up
def thumbnail(ctx, width, height, smartcrop, thumbnail, image_path):
    """Analyze an image with Oxford."""
    thumbnail_analysis_url = ctx.obj['oxford_url'] + '/thumbnails'
    params = {
        'width' : width,
        'height' : height,
        'smartCropping' : smartcrop
    }
    headers = {
    'Ocp-Apim-Subscription-Key' : ctx.obj['apikeys']['vision']
    }
    if type(image_path) is file:
        headers['Content-type'] = 'application/octet-stream'
        payload = image_path.read()
    if type(image_path) is urlparse.ParseResult:
        headers['Content-type'] = 'application/json'
        payload = json.dumps({ 'Url' : image_path.geturl() })
    try:
        resp = requests.post(thumbnail_analysis_url, params=params, data=payload, headers=headers, stream=True)
        if resp.status_code == 200:
            with open(thumbnail, 'wb') as out_file:
                shutil.copyfileobj(resp.raw, out_file)
        else:
            print resp.json()['message']
    except Exception as e:
        print e
   
# use vision api to recognize text in an image 
Example 18
Project: oxford-cli   Author: irjudson   File: oxford.py    MIT License 5 votes vote down vote up
def ocr(ctx, language, detect_orientation, image_path):
    """Analyze an image with Oxford."""
    thumbnail_analysis_url = ctx.obj['oxford_url'] + '/ocr'
    params = {
        'language' : language,
        'detectOrientation' : detect_orientation
    }
    headers = {
    'Ocp-Apim-Subscription-Key' : ctx.obj['apikeys']['vision']
    }
    if type(image_path) is file:
        headers['Content-type'] = 'application/octet-stream'
        payload = image_path.read()
    if type(image_path) is urlparse.ParseResult:
        headers['Content-type'] = 'application/json'
        payload = json.dumps({ 'Url' : image_path.geturl() })
    try:
        resp = requests.post(thumbnail_analysis_url, params=params, data=payload, headers=headers, stream=True)
        if resp.status_code == 200:
            print json.dumps(resp.json(), sort_keys=True, indent=2, separators=(',', ': '))
        else:
            print resp.json()['message']
    except Exception as e:
        print e

#
# Wiring up subcommands
#

# Face 
Example 19
Project: fastscore-cli   Author: modelop   File: sessioncookie.py    Apache License 2.0 5 votes vote down vote up
def add_url_params(url, params):
    # Unquoting URL first so we don't loose existing args
    url = unquote(url)
    # Extracting url info
    parsed_url = urlparse(url)
    # Extracting URL arguments from parsed URL
    get_args = parsed_url.query
    # Converting URL arguments to dict
    parsed_get_args = dict(parse_qsl(get_args))
    # Merging URL arguments dict with new params
    parsed_get_args.update(params)

    # Bool and Dict values should be converted to json-friendly values
    parsed_get_args.update(
        {k: json.dumps(v) for k, v in parsed_get_args.items()
         if isinstance(v, (bool, dict))}
    )

    # Converting URL argument to proper query string
    encoded_get_args = urlencode(parsed_get_args, doseq=True)
    # Creating new parsed result object based on provided with new
    # URL arguments. Same thing happens inside of urlparse.
    new_url = ParseResult(
        parsed_url.scheme, parsed_url.netloc, parsed_url.path,
        parsed_url.params, encoded_get_args, parsed_url.fragment
    ).geturl()

    return new_url 
Example 20
Project: open-wechat-scorer   Author: lingochamp   File: url_util.py    MIT License 5 votes vote down vote up
def add_url_params(url, params):
    """ Add GET params to provided URL being aware of existing.

    :param url: string of target URL
    :param params: dict containing requested params to be added
    :return: string with updated URL

    >> url = 'http://stackoverflow.com/test?answers=true'
    >> new_params = {'answers': False, 'data': ['some','values']}
    >> add_url_params(url, new_params)
    'http://stackoverflow.com/test?data=some&data=values&answers=false'
    """
    # Unquoting URL first so we don't lose existing args
    url = unquote(url)
    # Extracting url info
    parsed_url = urlparse(url)
    # Extracting URL arguments from parsed URL
    get_args = parsed_url.query
    # Converting URL arguments to dict
    parsed_get_args = dict(parse_qsl(get_args))
    # Merging URL arguments dict with new params
    parsed_get_args.update(params)

    # Bool and Dict values should be converted to json-friendly values
    # you may throw this part away if you don't like it :)
    parsed_get_args.update(
        {k: dumps(v) for k, v in parsed_get_args.items()
         if isinstance(v, (bool, dict))}
    )

    # Converting URL argument to proper query string
    encoded_get_args = urlencode(parsed_get_args, doseq=True)
    # Creating new parsed result object based on provided with new
    # URL arguments. Same thing happens inside of urlparse.
    new_url = ParseResult(
        parsed_url.scheme, parsed_url.netloc, parsed_url.path,
        parsed_url.params, encoded_get_args, parsed_url.fragment
    ).geturl()

    return new_url 
Example 21
Project: openag_python   Author: OpenAgricultureFoundation   File: utils.py    GNU General Public License v3.0 5 votes vote down vote up
def replicate_per_farm_dbs(cloud_url=None, local_url=None, farm_name=None):
    """
    Sete up replication of the per-farm databases from the local server to the
    cloud server.

    :param str cloud_url: Used to override the cloud url from the global
    configuration in case the calling function is in the process of
    initializing the cloud server
    :param str local_url: Used to override the local url from the global
    configuration in case the calling function is in the process of
    initializing the local server
    :param str farm_name: Used to override the farm name from the global
    configuratino in case the calling function is in the process of
    initializing the farm
    """
    cloud_url = cloud_url or config["cloud_server"]["url"]
    local_url = local_url or config["local_server"]["url"]
    farm_name = farm_name or config["cloud_server"]["farm_name"]
    username = config["cloud_server"]["username"]
    password = config["cloud_server"]["password"]

    # Add credentials to the cloud url
    parsed_cloud_url = urlparse(cloud_url)
    if not parsed_cloud_url.username:
        new_netloc = "{}:{}@{}".format(
            username, password, parsed_cloud_url.netloc
        )
    cloud_url = ParseResult(
        parsed_cloud_url.scheme, new_netloc, parsed_cloud_url.path,
        parsed_cloud_url.params, parsed_cloud_url.query,
        parsed_cloud_url.fragment
    ).geturl()

    server = Server(local_url)
    for db_name in per_farm_dbs:
        remote_db_name = "{}/{}/{}".format(username, farm_name, db_name)
        server.replicate(
            db_name, db_name, urljoin(cloud_url, remote_db_name),
            continuous=True
        ) 
Example 22
Project: rust   Author: limoxi   File: resource_client.py    Apache License 2.0 5 votes vote down vote up
def url_add_params(url, **params):
	""" 在网址中加入新参数 """
	pr = urlparse.urlparse(url)
	query = dict(urlparse.parse_qsl(pr.query))
	query.update(params)
	prlist = list(pr)
	prlist[4] = urllib.urlencode(query)
	return urlparse.ParseResult(*prlist).geturl() 
Example 23
Project: webproxycache   Author: RedHatEMEA   File: cache.py    GNU General Public License v3.0 5 votes vote down vote up
def _handle(self, f, netloc):
        req = Request(f, netloc)
        with certlock:
            print >>sys.stderr, "          " + " ".join([req.verb, urlparse.urlunparse(req.url), req.http])

        if req.verb == "CONNECT" and netloc is None:
            req.write("HTTP/1.1 200 Connection established\r\n\r\n")
            req.flush()

            serverctx = mk_serverctx(http_netloc(req.url)[0])
            self.request = serverctx.wrap_socket(self.request, server_side=True)

            return self.handle(http_netloc(req.url))

        if req.verb == "GET" and http_netloc(req.url) == ("rubygems.org", 443) and req.path().startswith("/api/v1/dependencies?"):
            url = list(req.url)
            qs = urlparse.parse_qs(url[4])
            qs["gems"] = ",".join(sorted(qs["gems"][0].split(",")))
            url[4] = urllib.urlencode(qs)
            req.url = urlparse.ParseResult(*url)

        if req.verb == "GET" and http_netloc(req.url) == ("cacert", 80):
            return LocalResponse(req, "certs/ca.crt").serve()

        try:
            return CachedResponse(req).serve()
        except NotInCacheException:
            return UncachedResponse(req).serve() 
Example 24
Project: python-phoenixdb   Author: MobileGuru1013   File: client.py    Apache License 2.0 5 votes vote down vote up
def parse_url(url):
    url = urlparse.urlparse(url)
    if not url.scheme and not url.netloc and url.path:
        netloc = url.path
        if ':' not in netloc:
            netloc = '{}:8765'.format(netloc)
        return urlparse.ParseResult('http', netloc, '/', '', '', '')
    return url


# Defined in phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java 
Example 25
Project: crispys_webkit   Author: crispycret   File: urls.py    MIT License 5 votes vote down vote up
def _delegate_url_pieces(self):
		""" Delegate the url parts from ParseResult, to self """
		self.scheme, self.host, self.path, self.params,\
		self.query, self.fragment = urlparse.urlparse(self.url) 
Example 26
Project: harpoon   Author: evilhero   File: common.py    GNU General Public License v3.0 5 votes vote down vote up
def update_uri(uri, construct=True, **kwargs):
    if isinstance(uri, urlparse.ParseResult):
        uri = dict(uri._asdict())

    if type(uri) is not dict:
        raise ValueError("Unknown URI type")

    uri.update(kwargs)

    result = urlparse.ParseResult(**uri)

    if not construct:
        return result

    return urlparse.urlunparse(result) 
Example 27
Project: daywatch   Author: tryolabs   File: middleware.py    MIT License 5 votes vote down vote up
def should_follow(self, response, spider):
        parsed = urlparse(response.url)
        url = ParseResult(
            parsed.scheme,
            parsed.netloc,
            parsed.path,
            parsed.params,
            None,
            None
        )
        url = url.geturl()
        return url not in spider.disallow_urls 
Example 28
Project: node-gn   Author: Shouqun   File: gerrit_client.py    MIT License 5 votes vote down vote up
def reparse_url(parsed_url, query_params):
  return urlparse.ParseResult(
      scheme=parsed_url.scheme,
      netloc=parsed_url.netloc,
      path=parsed_url.path,
      params=parsed_url.params,
      fragment=parsed_url.fragment,
      query=urllib.urlencode(query_params, doseq=True)) 
Example 29
Project: opds-reader   Author: steinarb   File: model.py    GNU General Public License v3.0 5 votes vote down vote up
def downloadMetadataUsingCalibreRestApi(self, opdsUrl):
        # The "updated" values on the book metadata, in the OPDS returned
        # by calibre, are unrelated to the books they are returned with:
        # the "updated" value is the same value for all books metadata,
        # and this value is the last modified date of the entire calibre
        # database.
        #
        # It is therefore necessary to use the calibre REST API to get
        # a meaningful timestamp for the books

        # Get the base of the web server, from the OPDS URL
        parsedOpdsUrl = urlparse.urlparse(opdsUrl)

        # GET the search URL twice: the first time is to get the total number
        # of books in the other calibre.  The second GET gets arguments
        # to retrieve all book ids in the other calibre.
        parsedCalibreRestSearchUrl = urlparse.ParseResult(parsedOpdsUrl.scheme, parsedOpdsUrl.netloc, '/ajax/search', '', '', '')
        calibreRestSearchUrl = parsedCalibreRestSearchUrl.geturl()
        calibreRestSearchResponse = urllib2.urlopen(calibreRestSearchUrl)
        calibreRestSearchJsonResponse = json.load(calibreRestSearchResponse)
        getAllIdsArgument = 'num=' + str(calibreRestSearchJsonResponse['total_num']) + '&offset=0'
        parsedCalibreRestSearchUrl = urlparse.ParseResult(parsedOpdsUrl.scheme, parsedOpdsUrl.netloc, '/ajax/search', '', getAllIdsArgument, '').geturl()
        calibreRestSearchResponse = urllib2.urlopen(parsedCalibreRestSearchUrl)
        calibreRestSearchJsonResponse = json.load(calibreRestSearchResponse)
        bookIds = map(str, calibreRestSearchJsonResponse['book_ids'])

        # Get the metadata for all books by adding the list of
        # all IDs as a GET argument
        bookIdsGetArgument = 'ids=' + ','.join(bookIds)
        parsedCalibreRestBooksUrl = urlparse.ParseResult(parsedOpdsUrl.scheme, parsedOpdsUrl.netloc, '/ajax/books', '', bookIdsGetArgument, '')
        calibreRestBooksResponse = urllib2.urlopen(parsedCalibreRestBooksUrl.geturl())
        booksDictionary = json.load(calibreRestBooksResponse)
        self.updateTimestampInMetadata(bookIds, booksDictionary) 
Example 30
Project: tensorboard   Author: tensorflow   File: import_google_fonts.py    Apache License 2.0 5 votes vote down vote up
def open_url(url):
    ru = urlparse.urlparse(url)
    pu = urlparse.ParseResult("", "", ru.path, ru.params, ru.query, ru.fragment)
    if ru.scheme == "https":
        c = httplib.HTTPSConnection(ru.netloc)
    else:
        c = httplib.HTTPConnection(ru.netloc)
    c.putrequest("GET", pu.geturl())
    c.putheader("User-Agent", FLAGS.user_agent)
    c.endheaders()
    return c.getresponse() 
Example 31
Project: fast_urlparse   Author: Parsely   File: test_urlparse.py    MIT License 5 votes vote down vote up
def test_result_pairs(self):
        # Check encoding and decoding between result pairs
        result_types = [
          urlparse.DefragResult,
          urlparse.SplitResult,
          urlparse.ParseResult,
        ]
        for result_type in result_types:
            self._check_result_type(result_type) 
Example 32
Project: kaku   Author: bear   File: twitter_feed.py    MIT License 5 votes vote down vote up
def validateDomain(domain=None):
    if domain is None:
        logger.error('A domain must be specified.')
        return None
    url = urlparse(domain)
    if len(url.scheme) == 0 or (url.netloc) == 0:
        logger.error('The domain must be specified with a scheme or location.')
        return None
    return ParseResult(url.scheme, url.netloc, url.path, '', '', '').geturl() 
Example 33
Project: kaku   Author: bear   File: kaku_post.py    MIT License 5 votes vote down vote up
def getAccessToken(domainUrl, accessEndpoint):
    url = urlparse(accessEndpoint)
    if len(url.scheme) == 0 or (url.netloc) == 0:
        access = ParseResult(domainUrl.scheme, domainUrl.netloc, accessEndpoint, '', '', '').geturl()
    else:
        access = ParseResult(url.scheme, url.netloc, url.path, '', '', '').geturl()

    logger.info('A web browser will be opened to the %s url so you can retrieve the authentication token which will be needed next.' % access)
    webbrowser.open(accessEndpoint)
    authToken = input('\nEnter your authentication token (remember to use "s around it)? ')

    return authToken 
Example 34
Project: hubcheck   Author: dskard   File: basepageobject.py    MIT License 5 votes vote down vote up
def _populate_urlparse_tuple(self,scheme='',netloc='',path='',
                                 params='',query='',fragment=''):

        return ParseResult(scheme,netloc,path,params,query,fragment) 
Example 35
Project: python-phoenixdb   Author: lalinsky   File: client.py    Apache License 2.0 5 votes vote down vote up
def parse_url(url):
    url = urlparse.urlparse(url)
    if not url.scheme and not url.netloc and url.path:
        netloc = url.path
        if ':' not in netloc:
            netloc = '{}:8765'.format(netloc)
        return urlparse.ParseResult('http', netloc, '/', '', '', '')
    return url


# Defined in phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java 
Example 36
Project: django-offlinecdn   Author: gabegaster   File: offlinecdn.py    MIT License 5 votes vote down vote up
def reformat_url(self, url_value):
        """strip the scheme and domain from the url object and append the path
        to OFFLINECDN_STATIC_URL.
        """
        url = urlparse.urlparse(url_value)
        if url.scheme or url.netloc:
            urlparts = list(url)
            urlparts[0] = ""
            url = urlparse.ParseResult(*urlparts)
        return url.geturl() 
Example 37
Project: django-offlinecdn   Author: gabegaster   File: offlinecdn.py    MIT License 5 votes vote down vote up
def reformat_url(self, url_value):
        """strip the scheme and domain from the url object and append the path
        to OFFLINECDN_STATIC_URL.
        """
        url = urlparse.urlparse(url_value)
        if url.scheme or url.netloc:
            urlparts = list(url)
            urlparts[0] = ""
            url = urlparse.ParseResult(*urlparts)
        return url.geturl() 
Example 38
Project: python-indieweb   Author: bear   File: indieweb.py    MIT License 4 votes vote down vote up
def handleLogin():
    app.logger.info('handleLogin [%s]' % request.method)

    form = LoginForm(me='', client_id=cfg['client_id'], 
                     redirect_uri='%s/success' % cfg['baseurl'], 
                     from_uri=request.args.get('from_uri'))

    if form.validate_on_submit():
        app.logger.info('me [%s]' % form.me.data)

        me            = baseDomain(form.me.data)
        authEndpoints = ninka.indieauth.discoverAuthEndpoints(me)

        if 'authorization_endpoint' in authEndpoints:
            authURL = None
            for url in authEndpoints['authorization_endpoint']:
                authURL = url
                break

            if authURL is not None:
                url = ParseResult(authURL.scheme, 
                                  authURL.netloc,
                                  authURL.path,
                                  authURL.params,
                                  urllib.urlencode({ 'me':            me,
                                                     'redirect_uri':  form.redirect_uri.data,
                                                     'client_id':     form.client_id.data,
                                                     'scope':         'post',
                                                     'response_type': 'id'
                                                   }),
                                  authURL.fragment).geturl()
                if db is not None:
                    key = 'login-%s' % me
                    data = db.hgetall(key)
                    if data: # clear any existing auth data
                        db.delete('token-%s' % data['token'])
                        db.hdel(key, 'token')
                    db.hset(key, 'from_uri',     form.from_uri.data)
                    db.hset(key, 'redirect_uri', form.redirect_uri.data)
                    db.hset(key, 'client_id',    form.client_id.data)
                    db.hset(key, 'scope',        'post')
                    db.expire(key, cfg['auth_timeout']) # expire in N minutes unless successful
                return redirect(url)
        else:
            return 'insert fancy no auth endpoint found error message here', 403

    templateData['title'] = 'Sign In'
    templateData['form']  = form
    return render_template('login.jinja', **templateData) 
Example 39
Project: python-indieweb   Author: bear   File: test_tokens.py    MIT License 4 votes vote down vote up
def runTest(self):
        me = 'http://127.0.0.1:9999'
        r = ninka.discoverTokenEndpoints(me)

        assert 'token_endpoint' in r

        tokenURL = None
        for url in r['token_endpoint']:
            tokenURL = url
            break

        assert tokenURL is not None
        assert tokenURL.scheme == 'http'
        assert tokenURL.netloc == '127.0.0.1:9999'
        assert tokenURL.path   == '/token'

        url = ParseResult(tokenURL.scheme, 
                          tokenURL.netloc,
                          tokenURL.path,
                          tokenURL.params,
                          urllib.urlencode({ 'me':    me,
                                             'scope': 'post'
                                           }),
                          tokenURL.fragment).geturl()

        r = requests.get(url)

        # token GET with no access_token present
        assert r.status_code == 400

        # token POST

        # token GET with valid access_token


# POST /micropub HTTP/1.1
# Host: bear.im
# Accept: */*
# Authorization: Bearer 159d4823-de7d-4717-9e4b-401da86a413b
# Content-Length: 30
# Content-Type: application/x-www-form-urlencoded

# h=entry&content=test&slug=test 
Example 40
Project: swarm   Author: a7vinx   File: sitemap.py    GNU General Public License v3.0 4 votes vote down vote up
def _parse_url(self,dst,src):
        """
        Check wether target url 'dst' is in the same domain(include port) with url 'src', and 
        convert url into complete url without params.

        Returns:
            String of complete url with query params if it has. if target url is not in the 
            same domain, return '';
        """
        LOG.debug('detecting url: '+dst)
        s_parsed=urlparse.urlparse(src)
        s_scheme=s_parsed.scheme
        s_netloc=s_parsed.netloc
        s_cur_dir=s_parsed.path
        if s_cur_dir[-1]!='/':
            s_cur_dir='/'.join(s_cur_dir.split('/')[:-1])
        else:
            s_cur_dir=s_cur_dir[:-1]

        d_parsed=urlparse.urlparse(dst)
        d_scheme=d_parsed.scheme
        if d_parsed.netloc.find(':')==-1 and d_parsed.netloc!='':
            if d_scheme=='http':
                d_netloc=d_parsed.netloc+':80'
            elif d_scheme=='https':
                d_netloc=d_parsed.netloc+':443'
            elif d_scheme=='':
                d_netloc=d_parsed.netloc+':80' if s_scheme=='http' else d_parsed.netloc+':443'
            else:
                d_netloc=d_parsed.netloc
        else:
            d_netloc=d_parsed.netloc
        # add '/' as prefix if the path does not starts with '/'
        if d_parsed.path!='':
            d_path='/'+d_parsed.path if d_parsed.path[0]!='/' else d_parsed.path
        else:
            d_path='/'
        d_query=d_parsed.query
        
        # if it is a relative url
        if d_netloc=='':
            return urlparse.ParseResult(s_scheme,s_netloc,s_cur_dir+d_path,'',d_query,'').geturl()
        elif d_netloc==s_netloc and (d_scheme==s_scheme or d_scheme==''):
            return urlparse.ParseResult(s_scheme,s_netloc,d_path,'',d_query,'').geturl()
        else:
            return '' 
Example 41
Project: kaku   Author: bear   File: auth.py    MIT License 4 votes vote down vote up
def handleLogin():
    current_app.logger.info('handleLogin [%s]' % request.method)

    me          = None
    redirectURI = '%s/success' % current_app.config['BASEURL']
    fromURI     = request.args.get('from_uri')

    current_app.logger.info('redirectURI [%s] fromURI [%s]' % (redirectURI, fromURI))
    form = LoginForm(me='',
                     client_id=current_app.config['CLIENT_ID'],
                     redirect_uri=redirectURI,
                     from_uri=fromURI)

    if form.validate_on_submit():
        current_app.logger.info('me [%s]' % form.me.data)

        me            = 'https://%s/' % baseDomain(form.me.data, includeScheme=False)
        scope         = ''
        authEndpoints = ninka.indieauth.discoverAuthEndpoints(me)

        if 'authorization_endpoint' in authEndpoints:
            authURL = None
            for url in authEndpoints['authorization_endpoint']:
                authURL = url
                break
            if authURL is not None:
                if me == current_app.config['BASEURL']:
                    scope = 'post update delete'
                url = ParseResult(authURL.scheme,
                                  authURL.netloc,
                                  authURL.path,
                                  authURL.params,
                                  urllib.urlencode({ 'me':            me,
                                                     'redirect_uri':  form.redirect_uri.data,
                                                     'client_id':     form.client_id.data,
                                                     'scope':         scope,
                                                     'response_type': 'id'
                                                   }),
                                  authURL.fragment).geturl()
                if current_app.dbRedis is not None:
                    key  = 'login-%s' % me
                    data = current_app.dbRedis.hgetall(key)
                    if data and 'token' in data:  # clear any existing auth data
                        current_app.dbRedis.delete('token-%s' % data['token'])
                        current_app.dbRedis.hdel(key, 'token')
                    current_app.dbRedis.hset(key, 'auth_url',     ParseResult(authURL.scheme, authURL.netloc, authURL.path, '', '', '').geturl())
                    current_app.dbRedis.hset(key, 'from_uri',     form.from_uri.data)
                    current_app.dbRedis.hset(key, 'redirect_uri', form.redirect_uri.data)
                    current_app.dbRedis.hset(key, 'client_id',    form.client_id.data)
                    current_app.dbRedis.hset(key, 'scope',        scope)
                    current_app.dbRedis.expire(key, current_app.config['AUTH_TIMEOUT'])  # expire in N minutes unless successful
                current_app.logger.info('redirecting to [%s]' % url)
                return redirect(url)
        else:
            return 'insert fancy no auth endpoint found error message here', 403

    templateContext = {}
    templateContext['title'] = 'Sign In'
    templateContext['form']  = form
    return render_template('login.jinja', **templateContext)