Python urllib.parse.ParseResult() Examples

def urlparams(url_, hash=None, **query):
    """Add a fragment and/or query paramaters to a URL.

    New query params will be appended to exising parameters, except duplicate
    names, which will be replaced.
    """
    url = urlparse.urlparse(url_)
    fragment = hash if hash is not None else url.fragment

    # Use dict(parse_qsl) so we don't get lists of values.
    query_dict = dict(urlparse.parse_qsl(url.query))
    query_dict.update(query)

    query_string = urlencode(
        [(k, v) for k, v in query_dict.items() if v is not None])
    new = urlparse.ParseResult(url.scheme, url.netloc, url.path, url.params,
                               query_string, fragment)
    return new.geturl() 

def __init__(self, urlobj: ParseResult):
        self.kind = 'v3io'
        host = urlobj.hostname or environ.get('V3IO_API')
        if urlobj.port:
            host += ':{}'.format(urlobj.port)
        self.url = 'http://{}{}'.format(host, urlobj.path)

        token = environ.get('V3IO_ACCESS_KEY')

        username = urlobj.username or environ.get('V3IO_USERNAME')
        password = urlobj.password or environ.get('V3IO_PASSWORD')

        self.headers = None
        self.auth = None
        if (not urlobj.username and urlobj.password) or token:
            token = urlobj.password or token
            self.headers = {'X-v3io-session-key': token}
        elif username and password:
            self.headers = basic_auth_header(username, password)

        self.path = urlobj.path
        self.workdir = urlobj.fragment 

def s3_exists(url: ParseResult) -> bool:
    """Return is an S3 resource exists.

    Parameters
    ----------
    url: ParseResult
        The parsed URL.

    Returns
    -------
    bool
        True if it exists. False otherwise.

    """
    s3 = boto3.resource('s3')
    try:
        bucket = s3.Bucket(url.netloc)
        path = url.path[1:]  # Not consider starting '/'
        objs = list(bucket.objects.filter(Prefix=path))
        return len(objs) > 0
    except s3.meta.client.exceptions.NoSuchBucket:
        return False 

def s3_remote_file(url: ParseResult) -> bool:
    """Check if an existing S3 hosted artifact is a file or a folder.

    Parameters
    ----------
    url: ParseResult
        The parsed URL.

    Returns
    -------
    bool
        True if it's a file, False if it's a folder.

    """
    s3 = boto3.resource('s3')
    bucket = s3.Bucket(url.netloc)
    path = url.path[1:]  # Not consider starting '/'
    objs = list(bucket.objects.filter(Prefix=path))
    if len(objs) == 1 and objs[0].key == path:
        return True

    return False 

def build_request(self, path=None, headers=None, endpoint=None):
        headers = {} if headers is None else headers
        if endpoint is None:
            endpoint = self._endpoint
        if path is None:
            url = endpoint
        else:
            p = urlparse.urlparse(endpoint)
            # should not use `os.path.join` since it returns path string like "/foo\\bar"
            request_path = path if p.path == "/" else "/".join([p.path, path])
            url = urlparse.urlunparse(
                urlparse.ParseResult(
                    p.scheme, p.netloc, request_path, p.params, p.query, p.fragment
                )
            )
        # use default headers first
        _headers = dict(self._headers)
        # add default headers
        _headers["authorization"] = "TD1 %s" % (self._apikey,)
        _headers["date"] = email.utils.formatdate(time.time())
        _headers["user-agent"] = self._user_agent
        # override given headers
        _headers.update({key.lower(): value for (key, value) in headers.items()})
        return (url, _headers) 

def geturl(self, include_params=True):
        params = self.params
        query = self.query
        fragment = self.fragment

        if not include_params:
            params = ""
            query = ""
            fragment = ""

        r = ParseResult(scheme=self.scheme,
                        netloc=self.netloc,
                        path=self.path,
                        params=params,
                        query=query,
                        fragment=fragment)
        return r.geturl() 

def geturl(self, include_params=True):
        params = self.params
        query = self.query
        fragment = self.fragment

        if not include_params:
            params = ""
            query = ""
            fragment = ""

        r = ParseResult(scheme=self.scheme,
                        netloc=self.netloc,
                        path=self.path,
                        params=params,
                        query=query,
                        fragment=fragment)
        return r.geturl() 

def urlparams(url_, hash=None, **query):
    """Add a fragment and/or query paramaters to a URL.

    New query params will be appended to exising parameters, except duplicate
    names, which will be replaced.
    """
    url = urlparse.urlparse(url_)
    fragment = hash if hash is not None else url.fragment

    # Use dict(parse_qsl) so we don't get lists of values.
    query_dict = dict(urlparse.parse_qsl(url.query))
    query_dict.update(query)

    query_string = urlencode(
        [(k, v) for k, v in query_dict.items() if v is not None])
    new = urlparse.ParseResult(url.scheme, url.netloc, url.path, url.params,
                               query_string, fragment)
    return new.geturl() 

def _uri_sort_query(cls, uri_parsed: UrlParseResult) -> UrlParseResult:
        if uri_parsed.query == '':
            return uri_parsed
        query_dict = parse_qs(uri_parsed.query, keep_blank_values=True)
        if 'variables' not in query_dict:
            return uri_parsed
        variables = query_dict['variables'][0]
        variables_dict = json_loads(variables)
        variables_dict_sorted = OrderedDict((k, variables_dict[k]) for k in sorted(variables_dict))
        query_dict['variables'][0] = json_dumps(variables_dict_sorted)
        query_sorted = urlencode(query_dict, doseq=True)
        return super()._uri_sort_query(UrlParseResult(
            uri_parsed.scheme,
            uri_parsed.netloc,
            uri_parsed.path,
            uri_parsed.params,
            query_sorted,
            uri_parsed.fragment
        )) 

def fetch_api_description(
    url: typing.Union[str, ParseResult, SplitResult], insecure: bool = False
):
    """Fetch the API description from the remote MAAS instance."""
    url_describe = urljoin(_ensure_url_string(url), "describe/")
    connector = aiohttp.TCPConnector(verify_ssl=(not insecure))
    session = aiohttp.ClientSession(connector=connector)
    async with session, session.get(url_describe) as response:
        if response.status != HTTPStatus.OK:
            raise RemoteError("{0} -> {1.status} {1.reason}".format(url, response))
        elif response.content_type != "application/json":
            raise RemoteError(
                "Expected application/json, got: %s" % response.content_type
            )
        else:
            return await response.json() 

def __init__(self, urlobj: ParseResult):
        self.kind = 'http'
        host = urlobj.hostname
        if urlobj.port:
            host += ':{}'.format(urlobj.port)
        self.url = '{}://{}{}'.format(urlobj.scheme, host, urlobj.path)
        if urlobj.username or urlobj.password:
            self.auth = (urlobj.username, urlobj.password)
            self.nuclio_header = basic_auth_header(urlobj.username,
                                                   urlobj.password)
        else:
            self.auth = None
            self.nuclio_header = None

        self.path = urlobj.path
        self.workdir = urlobj.fragment 

def get_logo_url(base_url, logo_file):
    base_url = parse.urlparse(base_url)
    netloc = base_url.netloc

    if base_url.netloc.startswith('localhost'):
        netloc = 'notify.tools'
    elif base_url.netloc.startswith('www'):
        # strip "www."
        netloc = base_url.netloc[4:]

    logo_url = parse.ParseResult(
        scheme=base_url.scheme,
        netloc='static-logos.' + netloc,
        path=logo_file,
        params=base_url.params,
        query=base_url.query,
        fragment=base_url.fragment
    )
    return parse.urlunparse(logo_url) 

def __init__(self, urlobj: ParseResult):
        self.kind = 'git'
        host = urlobj.hostname or 'github.com'
        if urlobj.port:
            host += ':{}'.format(urlobj.port)
        self.path = 'https://{}{}'.format(host, urlobj.path)

        self.headers = {'Authorization': ''}
        token = urlobj.username or environ.get('GIT_ACCESS_TOKEN')
        if token:
            self.headers = {'Authorization': 'token '.format(token)}

        # format: git://[token@]github.com/org/repo#master[:<workdir>]
        self.branch = 'master'
        self.workdir = None
        if urlobj.fragment:
            parts = urlobj.fragment.split(':')
            if parts[0]:
                self.branch = parts[0]
            if len(parts) > 1:
                self.workdir = parts[1] 

def dispatch(self, request, *args, **kwargs):
        if not self.test_func_tou_required(request.user):
            messages.add_message(
                request,
                messages.INFO,
                "You need to agree to the terms of use before you can do that.",
            )

            # Remember where they were trying to go, so we can redirect them
            # back after they agree. There's logic in TermsView to pick up on
            # this parameter.
            next_path = request.path
            next_param = urlencode({REDIRECT_FIELD_NAME: next_path})
            path = reverse_lazy("terms")
            new_url = ParseResult(
                scheme="",
                netloc="",
                path=str(path),
                params="",
                query=str(next_param),
                fragment="",
            ).geturl()
            return HttpResponseRedirect(new_url)

        return super(ToURequired, self).dispatch(request, *args, **kwargs) 

def __init__(self, system_ids, sources, proxy=None):
        """Create a new importer.

        :param system_ids: A sequence of rack controller system_id's.
        :param sources: A sequence of endpoints; see `ImportBootImages`.
        :param proxy: The HTTP/HTTPS proxy to use, or `None`
        :type proxy: :class:`urlparse.ParseResult` or string
        """
        super().__init__()
        self.system_ids = tuple(flatten(system_ids))
        if isinstance(sources, Sequence):
            self.sources = sources
        else:
            raise TypeError("expected sequence, got: %r" % (sources,))
        if proxy is None or isinstance(proxy, ParseResult):
            self.proxy = proxy
        else:
            self.proxy = urlparse(proxy) 

def get_direct_url(url, headers):
    """Gets the zip direct download link from the project download page"""
    direct_download_url = href_from_link_text(url,
                                              headers,
                                              'Problems Downloading')
    parsed_download_url = urlparse(direct_download_url)
    if parsed_download_url.scheme not in ['http', 'https']:
        # url is relative, and is missing the scheme and netloc
        parsed_parent_url = urlparse(url)
        parsed_download_url = ParseResult(parsed_parent_url.scheme,
                                          parsed_parent_url.netloc,
                                          parsed_download_url.path,
                                          parsed_download_url.params,
                                          parsed_download_url.query,
                                          parsed_download_url.fragment)
        direct_download_url = parsed_download_url.geturl()
    direct_download_url = href_from_link_text(direct_download_url,
                                              headers,
                                              'direct link')
    return direct_download_url 

def get_payable_from_BIP21URI(uri: str, proto: str = "bitcoin") -> Tuple[str, Decimal]:
    """ Computes a 'payable' tuple from a given BIP21 encoded URI.

    :param uri: The BIP21 URI to decode
    :param proto: The expected protocol/scheme (case insensitive)
    :returns: A payable (address, amount) corresponding to the given URI
    :raise: Raises s ValueError if there is no address given or if the
        protocol/scheme doesn't match what is expected
    """
    obj = parse.urlparse(uri)  # type: parse.ParseResult
    if not obj.path or obj.scheme.upper() != proto.upper():
        raise ValueError("Malformed URI")
    if not obj.query:
        return obj.path, None
    query = parse.parse_qs(obj.query)  # type: Dict
    return obj.path, Decimal(query["amount"][0]) 

def dispatch(self, request, *args, **kwargs):
        if not self.test_func_email_required(request.user):
            messages.add_message(
                request,
                messages.INFO,
                "You need to have an email address on file before you can do that.",
            )

            # Remember where they were trying to go, so we can redirect them
            # back after they agree. There's logic in TermsView to pick up on
            # this parameter.
            next_path = request.path
            next_param = urlencode({REDIRECT_FIELD_NAME: next_path})
            path = reverse_lazy("users:email_change")
            new_url = ParseResult(
                scheme="",
                netloc="",
                path=str(path),
                params="",
                query=str(next_param),
                fragment="",
            ).geturl()
            return HttpResponseRedirect(new_url)

        return super(EmailRequired, self).dispatch(request, *args, **kwargs) 

def url_to_string(url):
    """Convert url from ``list`` or ``ParseResult`` to string."""
    if isinstance(url, list):
        return ParseResult(
            scheme=url[0],
            netloc=url[1],
            path=url[2],
            params=None,
            query=None,
            fragment=None,
        ).geturl()

    if isinstance(url, ParseResult):
        return url.geturl()

    if isinstance(url, str):
        return url

    raise ValueError('url value not recognized') 

def __init__(self, urlobj: ParseResult):
        self.kind = 's3'
        self.bucket = urlobj.hostname
        self.key = urlobj.path[1:]
        region = None
        if urlobj.username or urlobj.password:
            self.s3 = boto3.resource('s3', region_name=region,
                                     aws_access_key_id=urlobj.username,
                                     aws_secret_access_key=urlobj.password)
        else:
            self.s3 = boto3.resource('s3', region_name=region) 

def __init__(self, urlobj: ParseResult):
        self.urlobj = urlobj
        self.kind = '' 

def _ensure_url_string(url):
    """Convert `url` to a string URL if it isn't one already."""
    if isinstance(url, str):
        return url
    elif isinstance(url, (ParseResult, SplitResult)):
        return url.geturl()
    else:
        raise TypeError("Could not convert %r to a string URL." % (url,)) 

def host_url(self) -> str:
        return urlunparse(ParseResult(self.scheme, self.host, "", "", "", "")) 

def __init__(self, url, *args, **kwargs):
        if not isinstance(url, ParseResult):
            url = urlparse(url)  # Ensure Path
        super().__init__(url, *args, **kwargs) 

def safeUrl(url:str)->str:
    from urllib import parse
    pr = parse.urlparse(url)
    pr2 = parse.ParseResult(scheme=pr.scheme, netloc=pr.netloc,
                            path=parse.quote(pr.path,'/%'), params=pr.params,
                            query=pr.query, fragment=pr.fragment)
    return parse.urlunparse(pr2) 

def parse_url(url):
    url = urlparse.urlparse(url)
    if not url.scheme and not url.netloc and url.path:
        netloc = url.path
        if ':' not in netloc:
            netloc = '{}:8765'.format(netloc)
        return urlparse.ParseResult('http', netloc, '/', '', '', '')
    return url


# Defined in phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java 

def _parse_db_name(con_params: ParseResult) -> str:
        if not con_params.path:
            raise ValueError('database must be specified in MySQL path')

        path = con_params.path.strip('/')
        if len(path.split('/')) != 1:
            raise ValueError('invalid database path')

        return path 

def flask_request_to_requests_response(flask_request: LocalProxy) -> RequestsResponse:
    urllib_url = urlparse(flask_request.url)

    return requests_request(
        headers=flask_request.headers,
        method=flask_request.method,
        url=ParseResult(
            scheme=urllib_url.scheme
        )
    ) 

def dict_request_to_requests_response(dict_request: dict) -> RequestsResponse:
    return requests_request(
        headers=dict_request['headers'],
        method=dict_request['method'],
        url=ParseResult(
            scheme=dict_request['url']['scheme'],
            netloc=dict_request['url']['netloc'],
            path=dict_request['url']['path'],
            params=dict_request['url']['params'],
            query=urlencode(dict_request['url']['query'], quote_via=lambda x, *a: x),
            fragment=dict_request['url']['fragment']
        ).geturl(),
        json=dict_request['body']
    ) 

def _fetch_catalog(collection_data, esmcol_path):
    """Get the catalog file content, and load it into a pandas dataframe"""

    if 'catalog_file' in collection_data:
        if _is_valid_url(esmcol_path):
            catalog_path = collection_data['catalog_file']
            if not _is_valid_url(catalog_path):
                split_url = urlparse(esmcol_path)
                path = (Path(split_url.path).parent / collection_data['catalog_file']).as_posix()
                components = ParseResult(
                    scheme=split_url.scheme,
                    netloc=split_url.netloc,
                    path=path,
                    params=split_url.params,
                    query=split_url.query,
                    fragment=split_url.fragment,
                )
                catalog = urlunparse(components)
                if not _is_valid_url(catalog):
                    raise FileNotFoundError(f'Unable to find: {catalog}')
                return pd.read_csv(catalog), catalog
            return pd.read_csv(catalog_path), catalog_path

        catalog_path = Path(collection_data['catalog_file'])
        # If the catalog_path does not exist,
        # try constructing a path using the relative path
        if not catalog_path.exists():
            esmcol_path = Path(esmcol_path).absolute()
            catalog = esmcol_path.parent / collection_data['catalog_file']
            if not catalog.exists():
                raise FileNotFoundError(f'Unable to find: {catalog}')
            return pd.read_csv(catalog), catalog

        return pd.read_csv(catalog_path), catalog_path

    return pd.DataFrame(collection_data['catalog_dict']), None