Python urllib.parse.unquote_plus() Examples

def _get_co_name_from_path(self, context):
        """
        The CO name is URL encoded and obtained from the request path
        for a request coming into one of the standard binding endpoints.
        For example the HTTP-Redirect binding request path will have the
        format

        {base}/{backend}/{co_name}/sso/redirect

        :type context: satosa.context.Context
        :rtype: str

        :param context:

        """
        url_encoded_co_name = context.path.split("/")[1]
        co_name = unquote_plus(url_encoded_co_name)

        return co_name 

def parse_userinfo(userinfo):
    """Validates the format of user information in a MongoDB URI.
    Reserved characters like ':', '/', '+' and '@' must be escaped
    following RFC 2396.

    Returns a 2-tuple containing the unescaped username followed
    by the unescaped password.

    :Paramaters:
        - `userinfo`: A string of the form <username>:<password>

    .. versionchanged:: 2.2
       Now uses `urllib.unquote_plus` so `+` characters must be escaped.
    """
    if '@' in userinfo or userinfo.count(':') > 1:
        raise InvalidURI("':' or '@' characters in a username or password "
                         "must be escaped according to RFC 2396.")
    user, _, passwd = _partition(userinfo, ":")
    # No password is expected with GSSAPI authentication.
    if not user:
        raise InvalidURI("The empty string is not valid username.")
    user = unquote_plus(user)
    passwd = unquote_plus(passwd)

    return user, passwd 

def get_transforms_by_type(request, ptype, user):
    if user['auth'] not in ['access_token', 'apitoken']:
        abort(status_code=403, message="Cannot access via Cookies. Use CLI or access via JS in browser")
    payload_type = unquote_plus(ptype)
    try:
        query = await db_model.payloadtype_query()
        payloadtype = await db_objects.get(query, ptype=payload_type)
    except Exception as e:
        print(e)
        return json({'status': 'error', 'error': 'failed to find payload type'})
    try:
        query = await db_model.transform_query()
        transforms = await db_objects.execute(query.where(Transform.payload_type == payloadtype).order_by(
            Transform.t_type, Transform.order
        ))
    except Exception as e:
        print(e)
        return json({'status': 'error', 'error': 'failed to get the transforms'})
    return json({'status': 'success', 'transforms': [t.to_json() for t in transforms]}) 

def remove_operator(request, name, user):
    if user['auth'] not in ['access_token', 'apitoken']:
        abort(status_code=403, message="Cannot access via Cookies. Use CLI or access via JS in browser")
    name = unquote_plus(name)
    if name != user['username'] and not user['admin']:
        return json({'status': 'error', 'error': 'cannot delete anybody but yourself unless you\'re admin'})
    try:
        query = await db_model.operator_query()
        op = await db_objects.get(query, username=name)
    except Exception as e:
        print(e)
        return json({'status': 'error', 'error': 'failed to find operator'})
    try:
        op.deleted = True
        await db_objects.update(op)
        success = {'status': 'success'}
        return json({**success, **op.to_json()})
    except Exception as e:
        print(e)
        return json({'status': 'error', 'error': 'failed to mark operator as deleted'}) 

def POST(self):
        """
        Get client key status.
        /client/status
        """
        # LDAP authentication
        is_auth, message = tools.ldap_authentification(SERVER_OPTS)
        if not is_auth:
            return tools.response_render(message, http_code='401 Unauthorized')

        payload, message = tools.data2map()
        if message:
            return tools.response_render(message, http_code='400 Bad Request')

        if 'realname' in payload:
            realname = unquote_plus(payload['realname'])
        else:
            return tools.response_render(
                'Error: No realname option given.',
                http_code='400 Bad Request')

        return tools.response_render(
            TOOLS.list_keys(realname=realname),
            content_type='application/json') 

def get_file_list_for_c2profiles(request, info, user):
    if user['auth'] not in ['access_token', 'apitoken']:
        abort(status_code=403, message="Cannot access via Cookies. Use CLI or access via JS in browser")
    name = unquote_plus(info)
    try:
        query = await db_model.c2profile_query()
        profile = await db_objects.get(query, name=name)
    except Exception as e:
        print(e)
        return json({'status': 'error', 'error': 'failed to find C2 Profile'})
    try:
        path = "./app/c2_profiles/{}/".format(profile.name)
        files = []
        for (dirpath, dirnames, filenames) in os.walk(path):
            if dirpath != path:
                files.append({"folder": dirpath.replace(path, ""), "dirnames": dirnames, "filenames": filenames})
        return json({'status': 'success', 'files': files})
    except Exception as e:
        return json({'status': 'error', 'error': 'failed getting files: ' + str(e)})


# Get c2 profile files listing for the user 

def _parse_options(opts, delim):
    """Helper method for split_options which creates the options dict.
    Also handles the creation of a list for the URI tag_sets/
    readpreferencetags portion."""
    options = {}
    for opt in opts.split(delim):
        key, val = opt.split("=")
        if key.lower() == 'readpreferencetags':
            options.setdefault('readpreferencetags', []).append(val)
        else:
            # str(option) to ensure that a unicode URI results in plain 'str'
            # option names. 'normalized' is then suitable to be passed as
            # kwargs in all Python versions.
            if str(key) in options:
                warnings.warn("Duplicate URI option %s" % (str(key),))
            options[str(key)] = unquote_plus(val)

    # Special case for deprecated options
    if "wtimeout" in options:
        if "wtimeoutMS" in options:
            options.pop("wtimeout")
        warnings.warn("Option wtimeout is deprecated, use 'wtimeoutMS'"
                      " instead")

    return options 

def format_url_show(url:str, urlenc='utf-8'):
    # return str
    indent = 4
    url = ps.unquote_plus(url, encoding=urlenc)
    pls = re.findall('\?[^&]*|&[^&]*',url)
    pms = [None]
    for i in pls:
        url = url.replace(i,'',1) # fix
        if len(i) > 50 and ',' in i:
            _pms = []
            for j in i.split(','):
                j = ' '*indent + j + ','
                _pms.append(j)
            _pms[-1] = _pms[-1][:-1]
            pms += _pms
        else:
            pms.append(i)
    pms[0] = url
    return '\n'.join(pms) 

def _parse_options(opts, delim):
    """Helper method for split_options which creates the options dict.
    Also handles the creation of a list for the URI tag_sets/
    readpreferencetags portion."""
    options = {}
    for opt in opts.split(delim):
        key, val = opt.split("=")
        if key.lower() == 'readpreferencetags':
            options.setdefault('readpreferencetags', []).append(val)
        else:
            # str(option) to ensure that a unicode URI results in plain 'str'
            # option names. 'normalized' is then suitable to be passed as
            # kwargs in all Python versions.
            if str(key) in options:
                warnings.warn("Duplicate URI option %s" % (str(key),))
            options[str(key)] = unquote_plus(val)

    # Special case for deprecated options
    if "wtimeout" in options:
        if "wtimeoutMS" in options:
            options.pop("wtimeout")
        warnings.warn("Option wtimeout is deprecated, use 'wtimeoutMS'"
                      " instead")

    return options 

def _delete_objects(bucket: str, keys: List[str], client_s3: boto3.client, attempt: int = 1) -> None:
    _logger.debug("len(keys): %s", len(keys))
    batch: List[Dict[str, str]] = [{"Key": key} for key in keys]
    res = client_s3.delete_objects(Bucket=bucket, Delete={"Objects": batch})
    deleted: List[Dict[str, Any]] = res.get("Deleted", [])
    for obj in deleted:
        _logger.debug("s3://%s/%s has been deleted.", bucket, obj.get("Key"))
    errors: List[Dict[str, Any]] = res.get("Errors", [])
    internal_errors: List[str] = []
    for error in errors:
        if error["Code"] != "InternalError":
            raise exceptions.ServiceApiError(errors)
        internal_errors.append(_unquote_plus(error["Key"]))
    if len(internal_errors) > 0:
        if attempt > 5:  # Maximum of 5 attempts (Total of 15 seconds)
            raise exceptions.ServiceApiError(errors)
        time.sleep(attempt)  # Incremental delay (linear)
        _delete_objects(bucket=bucket, keys=internal_errors, client_s3=client_s3, attempt=(attempt + 1)) 

def parse_userinfo(userinfo):
    """Validates the format of user information in a MongoDB URI.
    Reserved characters like ':', '/', '+' and '@' must be escaped
    following RFC 2396.

    Returns a 2-tuple containing the unescaped username followed
    by the unescaped password.

    :Paramaters:
        - `userinfo`: A string of the form <username>:<password>

    .. versionchanged:: 2.2
       Now uses `urllib.unquote_plus` so `+` characters must be escaped.
    """
    if '@' in userinfo or userinfo.count(':') > 1:
        raise InvalidURI("':' or '@' characters in a username or password "
                         "must be escaped according to RFC 2396.")
    user, _, passwd = _partition(userinfo, ":")
    # No password is expected with GSSAPI authentication.
    if not user:
        raise InvalidURI("The empty string is not valid username.")
    user = unquote_plus(user)
    passwd = unquote_plus(passwd)

    return user, passwd 

def parse_userinfo(userinfo):
    """Validates the format of user information in a MongoDB URI.
    Reserved characters like ':', '/', '+' and '@' must be escaped
    following RFC 2396.

    Returns a 2-tuple containing the unescaped username followed
    by the unescaped password.

    :Paramaters:
        - `userinfo`: A string of the form <username>:<password>

    .. versionchanged:: 2.2
       Now uses `urllib.unquote_plus` so `+` characters must be escaped.
    """
    if '@' in userinfo or userinfo.count(':') > 1:
        raise InvalidURI("':' or '@' characters in a username or password "
                         "must be escaped according to RFC 2396.")
    user, _, passwd = _partition(userinfo, ":")
    # No password is expected with GSSAPI authentication.
    if not user:
        raise InvalidURI("The empty string is not valid username.")
    user = unquote_plus(user)
    passwd = unquote_plus(passwd)

    return user, passwd 

def _parse_options(opts, delim):
    """Helper method for split_options which creates the options dict.
    Also handles the creation of a list for the URI tag_sets/
    readpreferencetags portion."""
    options = {}
    for opt in opts.split(delim):
        key, val = opt.split("=")
        if key.lower() == 'readpreferencetags':
            options.setdefault('readpreferencetags', []).append(val)
        else:
            # str(option) to ensure that a unicode URI results in plain 'str'
            # option names. 'normalized' is then suitable to be passed as
            # kwargs in all Python versions.
            if str(key) in options:
                warnings.warn("Duplicate URI option %s" % (str(key),))
            options[str(key)] = unquote_plus(val)

    # Special case for deprecated options
    if "wtimeout" in options:
        if "wtimeoutMS" in options:
            options.pop("wtimeout")
        warnings.warn("Option wtimeout is deprecated, use 'wtimeoutMS'"
                      " instead")

    return options 

def url_unescape(value, encoding='utf-8', plus=True):
        """Decodes the given value from a URL.

        The argument may be either a byte or unicode string.

        If encoding is None, the result will be a byte string.  Otherwise,
        the result is a unicode string in the specified encoding.

        If ``plus`` is true (the default), plus signs will be interpreted
        as spaces (literal plus signs must be represented as "%2B").  This
        is appropriate for query strings and form-encoded values but not
        for the path component of a URL.  Note that this default is the
        reverse of Python's urllib module.

        .. versionadded:: 3.1
           The ``plus`` argument
        """
        unquote = (urllib_parse.unquote_plus if plus else urllib_parse.unquote)
        if encoding is None:
            return unquote(utf8(value))
        else:
            return unicode_type(unquote(utf8(value)), encoding) 

def authorization_required(func):
    @wraps(func)
    @coroutine
    def wrap(self, *args, **kwargs):
        auth_header = self.request.headers.get('Authorization')
        if not auth_header:
            self.set_header('WWW-Authenticate', 'Basic realm="pypi"')
            self.set_status(401)
            raise Return(self.finish("Authorization required"))

        auth_type, data = auth_header.split()
        if auth_type.lower() != 'basic':
            raise Return(self.send_error(400))

        username, password = map(lambda x: unquote_plus(x.decode("utf-8")), base64.b64decode(b(data)).split(b(":")))
        try:
            self.current_user = yield check_password(username, password)
        except LookupError:
            raise HTTPError(403)

        result = yield maybe_future(func(self, *args, **kwargs))
        raise Return(result)

    return wrap 

def process_key_event(event, context):
    processor = EncryptExtantKeys(config)
    for record in event.get('Records', []):
        bucket = record['s3']['bucket']['name']
        key = {'Key': unquote_plus(record['s3']['object']['key']),
               'Size': record['s3']['object']['size']}
        version = record['s3']['object'].get('versionId')
        if version is not None:
            key['VersionId'] = version
            # lambda event is always latest version, but IsLatest
            # is not in record
            key['IsLatest'] = True
            method = processor.process_version
        else:
            method = processor.process_key
        try:
            result = retry(method, s3, key, bucket)
        except ClientError as e:
            # Ensure we know which key caused an issue
            print("error %s:%s code:%s" % (
                bucket, key['Key'], e.response['Error']))
            raise
        if not result:
            return
        print("remediated %s:%s" % (bucket, key['Key'])) 

def load(self, issuer):
        if not self._data_uri.startswith('data:application/pkcs8;kid='):
            raise PrivateKeyRetrieverException('Unrecognised data uri format.')
        splitted = self._data_uri.split(';')
        key_identifier = KeyIdentifier(unquote_plus(
            splitted[1][len('kid='):]))
        key_data = base64.b64decode(splitted[-1].split(',')[-1])
        key = serialization.load_der_private_key(
            key_data,
            password=None,
            backend=cryptography.hazmat.backends.default_backend())
        private_key_pem = key.private_bytes(
            encoding=serialization.Encoding.PEM,
            format=serialization.PrivateFormat.TraditionalOpenSSL,
            encryption_algorithm=serialization.NoEncryption()
        )
        return key_identifier, private_key_pem.decode('utf-8') 

def url_unescape(value, encoding='utf-8', plus=True):
        """Decodes the given value from a URL.

        The argument may be either a byte or unicode string.

        If encoding is None, the result will be a byte string.  Otherwise,
        the result is a unicode string in the specified encoding.

        If ``plus`` is true (the default), plus signs will be interpreted
        as spaces (literal plus signs must be represented as "%2B").  This
        is appropriate for query strings and form-encoded values but not
        for the path component of a URL.  Note that this default is the
        reverse of Python's urllib module.

        .. versionadded:: 3.1
           The ``plus`` argument
        """
        unquote = (urllib_parse.unquote_plus if plus else urllib_parse.unquote)
        if encoding is None:
            return unquote(utf8(value))
        else:
            return unicode_type(unquote(utf8(value)), encoding) 

def url_decode(self):
        """Converts URI/URL percent-encoded characters back to their raw values.
        
        Returns:
            Chepy: The Chepy object.

        Examples:
            >>> Chepy("https://google.com/%3Flol%3Dsome+data%26a%3D1").url_decode().o
            "https://google.com/?lol=some data&a=1"
        """
        self.state = _urllib_unquote_plus(self._convert_to_str())
        return self 

def get_container_file_list_for_c2profiles(request, info, user):
    if user['auth'] not in ['access_token', 'apitoken']:
        abort(status_code=403, message="Cannot access via Cookies. Use CLI or access via JS in browser")
    name = unquote_plus(info)
    try:
        query = await db_model.c2profile_query()
        profile = await db_objects.get(query, name=name)
    except Exception as e:
        print(e)
        return json({'status': 'error', 'error': 'failed to find C2 Profile'})
    try:
        status = await send_c2_rabbitmq_message(profile.name, "listfiles", "", user['username'])
        return json(status)
    except Exception as e:
        return json({'status': 'error', 'error': 'failed getting files: ' + str(e)}) 

def log_request_data(http_response, logger):
    """Send a debug log message with basic information
    of the HTTP request that was sent for the given HTTP response.

    :param requests.models.Response http_response: HTTP response object
    """
    sent_request = {
        'method': http_response.request.method,
        'url': http_response.request.url,
        'headers': http_response.request.headers,
        'decoded_body': unquote_plus(repr(http_response.request.body)),
    }
    logger.debug("Request sent: %s", sent_request)
    return 0 

def _remove_javascript_link(self, link):
        # links like "j a v a s c r i p t:" might be interpreted in IE
        new = _substitute_whitespace('', unquote_plus(link))
        if _is_javascript_scheme(new):
            # FIXME: should this be None to delete?
            return ''
        return link 

def stop_c2profile(request, info, user):
    if user['auth'] not in ['access_token', 'apitoken']:
        abort(status_code=403, message="Cannot access via Cookies. Use CLI or access via JS in browser")
    name = unquote_plus(info)
    return await stop_c2profile_func(name, user['username']) 

def unquote_str(value, encoding='utf-8'):
        # In python2, unquote() gives us a string back that has the urldecoded
        # bits, but not the unicode parts.  We need to decode this manually.
        # unquote has special logic in which if it receives a unicode object it
        # will decode it to latin1.  This is hard coded.  To avoid this, we'll
        # encode the string with the passed in encoding before trying to
        # unquote it.
        byte_string = value.encode(encoding)
        return unquote_plus(byte_string).decode(encoding) 

def from_url(cls, url: str):
        split = urlsplit(url)

        if split.scheme != PARSEC_SCHEME:
            raise ValueError(f"Must start with `{PARSEC_SCHEME}://`")

        if split.query:
            # Note `parse_qs` takes care of percent-encoding
            params = parse_qs(
                split.query,
                keep_blank_values=True,
                strict_parsing=True,
                encoding="utf-8",
                errors="strict",
            )
        else:
            params = {}

        path = unquote_plus(split.path)

        kwargs = {
            **cls._from_url_parse_and_consume_params(params),
            **cls._from_url_parse_path(path),
        }

        return cls(hostname=split.hostname or "localhost", port=split.port, **kwargs) 

def extract_basicauth(authorization_header, encoding='utf-8'):
    splitted = authorization_header.split(' ')
    if len(splitted) != 2:
        return None

    auth_type, auth_string = splitted

    if 'basic' != auth_type.lower():
        return None

    try:
        b64_decoded = base64.b64decode(auth_string)
    except (TypeError, binascii.Error):
        return None
    try:
        auth_string_decoded = b64_decoded.decode(encoding)
    except UnicodeDecodeError:
        return None

    splitted = auth_string_decoded.split(':')

    if len(splitted) != 2:
        return None

    username, password = map(unquote_plus, splitted)
    return username, password 

def start_c2profile(request, info, user):
    if user['auth'] not in ['access_token', 'apitoken']:
        abort(status_code=403, message="Cannot access via Cookies. Use CLI or access via JS in browser")
    name = unquote_plus(info)
    try:
        query = await db_model.c2profile_query()
        profile = await db_objects.get(query, name=name)
    except Exception as e:
        print(e)
        return json({'status': 'error', 'error': 'failed to find C2 Profile'})
    status = await send_c2_rabbitmq_message(profile.name, "start", "", user['username'])
    return json(status)


# Start running a profile's server side code 

def unquote_str(value, encoding='utf-8'):
        # In python2, unquote() gives us a string back that has the urldecoded
        # bits, but not the unicode parts.  We need to decode this manually.
        # unquote has special logic in which if it receives a unicode object it
        # will decode it to latin1.  This is hard coded.  To avoid this, we'll
        # encode the string with the passed in encoding before trying to
        # unquote it.
        byte_string = value.encode(encoding)
        return unquote_plus(byte_string).decode(encoding) 

def urlunquote_plus(quoted_url):
    """
    A wrapper for Python's urllib.unquote_plus() function that can operate on
    the result of django.utils.http.urlquote_plus().
    """
    return force_text(urllib_parse.unquote_plus(force_str(quoted_url))) 

def get_c2profiles_by_type_in_current_operation(request, info, user):
    if user['auth'] not in ['access_token', 'apitoken']:
        abort(status_code=403, message="Cannot access via Cookies. Use CLI or access via JS in browser")
    ptype = unquote_plus(info)
    try:
        profiles = await get_c2profiles_by_type_function(ptype, user)
        return json({'status': 'success', 'profiles': profiles})
    except Exception as e:
        print(e)
        return json({'status': 'error', 'error': 'failed to get c2 profiles'})


# this function will be useful by other files, so make it easier to use