Python re.ASCII() Examples

def letter_case_props(self, case, in_group, negate=False):
        """Insert letter (ASCII or Unicode) case properties."""

        # Use traditional ASCII upper/lower case unless:
        #    1. The strings fed in are not binary
        #    2. And the the unicode flag was used
        if not in_group:
            v = self.posix_props(
                (self._negate if negate else self._empty) +
                (self._ascii_upper if case == _UPPER else self._ascii_lower)
            )
            v[0] = self._ls_bracket + v[0] + self._rs_bracket
        else:
            v = self.posix_props(
                (self._negate if negate else self._empty) +
                (self._ascii_upper if case == _UPPER else self._ascii_lower)
            )
        return v 

def _apply_search_backrefs(pattern, flags=0):
    """Apply the search backrefs to the search pattern."""

    if isinstance(pattern, (compat.string_type, compat.binary_type)):
        re_verbose = bool(VERBOSE & flags)
        re_unicode = None
        if compat.PY3 and bool(ASCII & flags):
            re_unicode = False
        elif bool(UNICODE & flags):
            re_unicode = True
        pattern = SearchTemplate(pattern, re_verbose, re_unicode).apply()
    elif isinstance(pattern, RE_TYPE):
        if flags:
            raise ValueError("Cannot process flags argument with a compiled pattern!")
    else:
        raise TypeError("Not a string or compiled pattern!")
    return pattern 

def parse150(resp):
    '''Parse the '150' response for a RETR request.
    Returns the expected transfer size or None; size is not guaranteed to
    be present in the 150 message.
    '''
    if resp[:3] != '150':
        raise error_reply(resp)
    global _150_re
    if _150_re is None:
        import re
        _150_re = re.compile(
            "150 .* \((\d+) bytes\)", re.IGNORECASE | re.ASCII)
    m = _150_re.match(resp)
    if not m:
        return None
    s = m.group(1)
    try:
        return int(s)
    except (OverflowError, ValueError):
        return int(s) 

def parse227(resp):
    '''Parse the '227' response for a PASV request.
    Raises error_proto if it does not contain '(h1,h2,h3,h4,p1,p2)'
    Return ('host.addr.as.numbers', port#) tuple.'''

    if resp[:3] != '227':
        raise error_reply(resp)
    global _227_re
    if _227_re is None:
        import re
        _227_re = re.compile(r'(\d+),(\d+),(\d+),(\d+),(\d+),(\d+)', re.ASCII)
    m = _227_re.search(resp)
    if not m:
        raise error_proto(resp)
    numbers = m.groups()
    host = '.'.join(numbers[:4])
    port = (int(numbers[4]) << 8) + int(numbers[5])
    return host, port 

def encode(self, inp):
        #
        #  Invoke binascii.b2a_base64 iteratively with
        #  short even length buffers, strip the trailing
        #  line feed from the result and append.  "Even"
        #  means a number that factors to both 6 and 8,
        #  so when it gets to the end of the 8-bit input
        #  there's no partial 6-bit output.
        #
        oup = b''
        if isinstance(inp, str):
            inp = inp.encode('ASCII')
        while inp:
            if len(inp) > 48:
                t = inp[:48]
                inp = inp[48:]
            else:
                t = inp
                inp = b''
            e = binascii.b2a_base64(t)
            if e:
                oup = oup + e[:-1]
        return oup 

def parse227(resp):
    '''Parse the '227' response for a PASV request.
    Raises error_proto if it does not contain '(h1,h2,h3,h4,p1,p2)'
    Return ('host.addr.as.numbers', port#) tuple.'''

    if resp[:3] != '227':
        raise error_reply(resp)
    global _227_re
    if _227_re is None:
        import re
        _227_re = re.compile(r'(\d+),(\d+),(\d+),(\d+),(\d+),(\d+)', re.ASCII)
    m = _227_re.search(resp)
    if not m:
        raise error_proto(resp)
    numbers = m.groups()
    host = '.'.join(numbers[:4])
    port = (int(numbers[4]) << 8) + int(numbers[5])
    return host, port 

def split_provision(value):
    """Return the name and optional version number of a provision.

    The version number, if given, will be returned as a `StrictVersion`
    instance, otherwise it will be `None`.

    >>> split_provision('mypkg')
    ('mypkg', None)
    >>> split_provision(' mypkg( 1.2 ) ')
    ('mypkg', StrictVersion ('1.2'))
    """
    global _provision_rx
    if _provision_rx is None:
        _provision_rx = re.compile(
            "([a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*)(?:\s*\(\s*([^)\s]+)\s*\))?$",
            re.ASCII)
    value = value.strip()
    m = _provision_rx.match(value)
    if not m:
        raise ValueError("illegal provides specification: %r" % value)
    ver = m.group(2) or None
    if ver:
        ver = distutils.version.StrictVersion(ver)
    return m.group(1), ver 

def parse150(resp):
    '''Parse the '150' response for a RETR request.
    Returns the expected transfer size or None; size is not guaranteed to
    be present in the 150 message.
    '''
    if resp[:3] != '150':
        raise error_reply(resp)
    global _150_re
    if _150_re is None:
        import re
        _150_re = re.compile(
            "150 .* \((\d+) bytes\)", re.IGNORECASE | re.ASCII)
    m = _150_re.match(resp)
    if not m:
        return None
    return int(m.group(1)) 

def parse227(resp):
    '''Parse the '227' response for a PASV request.
    Raises error_proto if it does not contain '(h1,h2,h3,h4,p1,p2)'
    Return ('host.addr.as.numbers', port#) tuple.'''

    if resp[:3] != '227':
        raise error_reply(resp)
    global _227_re
    if _227_re is None:
        import re
        _227_re = re.compile(r'(\d+),(\d+),(\d+),(\d+),(\d+),(\d+)', re.ASCII)
    m = _227_re.search(resp)
    if not m:
        raise error_proto(resp)
    numbers = m.groups()
    host = '.'.join(numbers[:4])
    port = (int(numbers[4]) << 8) + int(numbers[5])
    return host, port 

def encode(self, inp):
        #
        #  Invoke binascii.b2a_base64 iteratively with
        #  short even length buffers, strip the trailing
        #  line feed from the result and append.  "Even"
        #  means a number that factors to both 6 and 8,
        #  so when it gets to the end of the 8-bit input
        #  there's no partial 6-bit output.
        #
        oup = b''
        if isinstance(inp, str):
            inp = inp.encode('ASCII')
        while inp:
            if len(inp) > 48:
                t = inp[:48]
                inp = inp[48:]
            else:
                t = inp
                inp = b''
            e = binascii.b2a_base64(t)
            if e:
                oup = oup + e[:-1]
        return oup 

def parse227(resp):
    '''Parse the '227' response for a PASV request.
    Raises error_proto if it does not contain '(h1,h2,h3,h4,p1,p2)'
    Return ('host.addr.as.numbers', port#) tuple.'''

    if resp[:3] != '227':
        raise error_reply(resp)
    global _227_re
    if _227_re is None:
        import re
        _227_re = re.compile(r'(\d+),(\d+),(\d+),(\d+),(\d+),(\d+)', re.ASCII)
    m = _227_re.search(resp)
    if not m:
        raise error_proto(resp)
    numbers = m.groups()
    host = '.'.join(numbers[:4])
    port = (int(numbers[4]) << 8) + int(numbers[5])
    return host, port 

def split_provision(value):
    """Return the name and optional version number of a provision.

    The version number, if given, will be returned as a `StrictVersion`
    instance, otherwise it will be `None`.

    >>> split_provision('mypkg')
    ('mypkg', None)
    >>> split_provision(' mypkg( 1.2 ) ')
    ('mypkg', StrictVersion ('1.2'))
    """
    global _provision_rx
    if _provision_rx is None:
        _provision_rx = re.compile(
            "([a-zA-Z_]\w*(?:\.[a-zA-Z_]\w*)*)(?:\s*\(\s*([^)\s]+)\s*\))?$",
            re.ASCII)
    value = value.strip()
    m = _provision_rx.match(value)
    if not m:
        raise ValueError("illegal provides specification: %r" % value)
    ver = m.group(2) or None
    if ver:
        ver = distutils.version.StrictVersion(ver)
    return m.group(1), ver 

def flags(key):
	flag = 0
	if 'a' in key:
		flag += re.ASCII
	if 'i' in key:
		flag += re.IGNORECASE
	if 'l' in key:
		flag += re.LOCALE
	if 'm' in key:
		flag += re.MULTILINE
	if 's' in key:
		flag += re.DOTALL
	if 'x' in key:
		flag += re.VERBOSE
	return flag 

def pathhasvars(source):
    """Expand shell variables of form $var and ${var}.  Unknown variables
    are left unchanged."""
    global _varprog, _varprogb
    if isinstance(source, bytes):
        if b'$' not in source:
            return False
        if not _varprogb:
            import re
            #_varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII)
            _varprogb = re.compile(br'\$(\w+|\{[^}]*\})')
        search = _varprogb.search
        start = b'{'
        end = b'}'
    else:
        if '$' not in source:
            return False
        if not _varprog:
            import re
            #_varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII)
            _varprog = re.compile(r'\$(\w+|\{[^}]*\})')
        search = _varprog.search
        start = '{'
        end = '}'
    i = 0
    m = search(source, i)
    return not (not m) 

def validate(self, value, redis):
        '''
        Validates data obtained from a request and returns it in the apropiate
        format
        '''
        # cleanup
        if type(value) == str:
            value = value.strip()

        value = self.value_or_default(value)

        # validation
        self.validate_required(value)

        if self.regex and not re.match(self.regex, value, flags=re.ASCII):
            raise InvalidFieldError(self.name)

        if self.forbidden and value in self.forbidden:
            raise ReservedFieldError(self.name)

        if self.allowed and value not in self.allowed:
            raise InvalidFieldError(self.name)

        if self.index:
            key = self.key()

            old = debyte_string(redis.hget(key, value))
            old_value = getattr(self.obj, self.name)

            if old is not None and old != self.obj.id:
                raise NotUniqueFieldError(self.name)
            elif old_value != value:
                self.obj._old[self.name] = old_value

        return value 

def breakopt(arg):
    m = re.match(r"^\-\-?([0-9a-zA-Z][0-9a-zA-Z\-]*)(?:\=(.+))?$", arg, re.ASCII)
    if m:
        opt_name = m.group(1)
        try:
            return opt_name, m.group(2)
        except IndexError:
            return opt_name, None
    return None, None 

def unquote(s):
    """Turn a string in the form =AB to the ASCII character with value 0xab"""
    return chr(int(s[1:3], 16)) 

def _unquote_match(match):
    """Turn a match in the form =AB to the ASCII character with value 0xab"""
    s = match.group(0)
    return unquote(s)


# Header decoding is done a bit differently 

def header_decode(s):
    """Decode a string encoded with RFC 2045 MIME header `Q' encoding.

    This function does not parse a full MIME header value encoded with
    quoted-printable (like =?iso-8895-1?q?Hello_World?=) -- please use
    the high level email.header class for that functionality.
    """
    s = s.replace('_', ' ')
    return re.sub(r'=[a-fA-F0-9]{2}', _unquote_match, s, re.ASCII) 

def formataddr(pair, charset='utf-8'):
    """The inverse of parseaddr(), this takes a 2-tuple of the form
    (realname, email_address) and returns the string value suitable
    for an RFC 2822 From, To or Cc header.

    If the first element of pair is false, then the second element is
    returned unmodified.

    Optional charset if given is the character set that is used to encode
    realname in case realname is not ASCII safe.  Can be an instance of str or
    a Charset-like object which has a header_encode method.  Default is
    'utf-8'.
    """
    name, address = pair
    # The address MUST (per RFC) be ascii, so raise an UnicodeError if it isn't.
    address.encode('ascii')
    if name:
        try:
            name.encode('ascii')
        except UnicodeEncodeError:
            if isinstance(charset, str):
                charset = Charset(charset)
            encoded_name = charset.header_encode(name)
            return "%s <%s>" % (encoded_name, address)
        else:
            quotes = ''
            if specialsre.search(name):
                quotes = '"'
            name = escapesre.sub(r'\\\g<0>', name)
            return '%s%s%s <%s>' % (quotes, name, quotes, address)
    return address 

def open_data(self, url, data=None):
        """Use "data" URL."""
        if not isinstance(url, str):
            raise URLError('data error: proxy support for data protocol currently not implemented')
        # ignore POSTed data
        #
        # syntax of data URLs:
        # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
        # mediatype := [ type "/" subtype ] *( ";" parameter )
        # data      := *urlchar
        # parameter := attribute "=" value
        try:
            [type, data] = url.split(',', 1)
        except ValueError:
            raise IOError('data error', 'bad data URL')
        if not type:
            type = 'text/plain;charset=US-ASCII'
        semi = type.rfind(';')
        if semi >= 0 and '=' not in type[semi:]:
            encoding = type[semi+1:]
            type = type[:semi]
        else:
            encoding = ''
        msg = []
        msg.append('Date: %s'%time.strftime('%a, %d %b %Y %H:%M:%S GMT',
                                            time.gmtime(time.time())))
        msg.append('Content-type: %s' % type)
        if encoding == 'base64':
            # XXX is this encoding/decoding ok?
            data = base64.decodebytes(data.encode('ascii')).decode('latin-1')
        else:
            data = unquote(data)
        msg.append('Content-Length: %d' % len(data))
        msg.append('')
        msg.append(data)
        msg = '\n'.join(msg)
        headers = email.message_from_string(msg)
        f = io.StringIO(msg)
        #f.fileno = None     # needed for addinfourl
        return addinfourl(f, headers, url) 

def test_posix_in_group_ascii(self):
        """Test posix in a group for ASCII."""
        if PY3:
            pattern = bre.compile_search(r'Test [[:graph:]]', re.ASCII)
            pattern2 = bre.compile_search('Test [\u0021-\u007E]', re.ASCII)
        else:
            pattern = bre.compile_search(r'Test [[:graph:]]')
            pattern2 = bre.compile_search(r'Test [\u0021-\u007E]')
        self.assertEqual(pattern.pattern, pattern2.pattern) 

def test_ascii_upper_props(self):
        """Test ASCII uppercase properties."""

        pattern = bre.compile_search(br'EX\c+LE')
        m = pattern.match(br'EXAMPLE')
        self.assertTrue(m is not None) 

def test_ascii_upper_props_group(self):
        """Test ASCII uppercase properties in a character group."""

        pattern = bre.compile_search(br'EX[\c]+LE')
        m = pattern.match(br'EXAMPLE')
        self.assertTrue(m is not None) 

def test_ascii_lower_props(self):
        """Test ASCII lowercase properties."""

        pattern = bre.compile_search(br'EX\l+LE')
        m = pattern.match(br'EXampLE')
        self.assertTrue(m is not None) 

def test_ascii_lower_props_group(self):
        """Test ASCII uppercase properties in a char group."""

        pattern = bre.compile_search(br'EX[\l]+LE')
        m = pattern.match(br'EXampLE')
        self.assertTrue(m is not None) 

def test_ascii_props_mixed(self):
        """Test mixed ASCII properties."""

        pattern = bre.compile_search(br'EX\l\c\lLE')
        m = pattern.match(br'EXaMpLE')
        self.assertTrue(m is not None) 

def test_reverse_ascii_lower_props(self):
        """Test reverse ASCII lowercase properties."""

        pattern = bre.compile_search(br'EX\L+LE')
        m = pattern.match(br'EXAMPLE')
        self.assertTrue(m is not None) 

def test_reverse_ascii_lower_props_group(self):
        """Test reverse ASCII lowercase properties in a group."""

        pattern = bre.compile_search(br'EX[\L]+LE')
        m = pattern.match(br'EXAMPLE')
        self.assertTrue(m is not None) 

def test_reverse_ascii_upper_props(self):
        """Test reverse ASCII uppercase properties."""

        pattern = bre.compile_search(br'EX\C+LE')
        m = pattern.match(br'EXampLE')
        self.assertTrue(m is not None) 

def test_reverse_ascii_upper_props_group(self):
        """Test reverse ASCII uppercase properties in a group."""

        pattern = bre.compile_search(br'EX[\C]+LE')
        m = pattern.match(br'EXampLE')
        self.assertTrue(m is not None) 

def test_reverse_ascii_props_mixed(self):
        """Test reverse ASCII properties."""

        pattern = bre.compile_search(br'EX\C\L\CLE')
        m = pattern.match(br'EXaMpLE')
        self.assertTrue(m is not None) 

def test_unicode_shorthand_ascii_only(self):
        """Ensure that when the Unicode flag is not used, only ASCII properties are used."""

        flags = bre.ASCII if PY3 else 0
        pattern = bre.compile_search(r'ex\lmple', flags)
        m = pattern.match('exámple')
        self.assertTrue(m is None)
        m = pattern.match('example')
        self.assertTrue(m is not None) 

def test_unicode_string_flag(self):
        """Test finding Unicode/ASCII string flag."""

        if PY3:
            template = bre.SearchTemplate(r'Testing for (?ia) ASCII flag.', False, None)
            template.apply()
            self.assertFalse(template.unicode)
        else:
            template = bre.SearchTemplate(r'Testing for (?iu) Unicode flag.', False, None)
            template.apply()
            self.assertTrue(template.unicode) 

def test_unicode_string_flag_in_group(self):
        """Test ignoring Unicode/ASCII string flag in group."""

        if PY3:
            template = bre.SearchTemplate(r'Testing for [(?ia)] ASCII flag.', False, None)
            template.apply()
            self.assertTrue(template.unicode)
        else:
            template = bre.SearchTemplate(r'Testing for [(?iu)] Unicode flag.', False, None)
            template.apply()
            self.assertFalse(template.unicode) 

def test_unicode_string_flag_escaped(self):
        """Test ignoring Unicode/ASCII string flag in group."""

        if PY3:
            template = bre.SearchTemplate(r'Testing for \(?ia) ASCII flag.', False, None)
            template.apply()
            self.assertTrue(template.unicode)
        else:
            template = bre.SearchTemplate(r'Testing for \(?iu) Unicode flag.', False, None)
            template.apply()
            self.assertFalse(template.unicode) 

def test_unicode_string_flag_escaped_deep(self):
        """Test deep escaped Unicode flag."""

        if PY3:
            template = bre.SearchTemplate(r'Testing for \\\(?ia) ASCII flag.', False, None)
            template.apply()
            self.assertTrue(template.unicode)
        else:
            template = bre.SearchTemplate(r'Testing for \\\(?iu) Unicode flag.', False, None)
            template.apply()
            self.assertFalse(template.unicode) 

def test_replace_unicode_name_ascii_range(self):
        """Test replacing Unicode names in the ASCII range."""

        pattern = re.compile(r"(some)(.*?)(pattern)(!)")
        expand = bre.compile_replace(
            pattern,
            r'\1 \N{Latin small letter a}\l\N{Latin Capital Letter A} and '
            r'\LSPAN \N{Latin Capital Letter A}\E and Escaped \\N{Latin Capital Letter A}\E \3'
        )
        results = expand(pattern.match('some test pattern!'))

        self.assertEqual(
            'some aa and span a and Escaped \\N{Latin Capital Letter A} pattern',
            results
        ) 

def _connect(self):
        # Create unique tag for this session,
        # and compile tagged response matcher.

        self.tagpre = Int2AP(random.randint(4096, 65535))
        self.tagre = re.compile(br'(?P<tag>'
                        + self.tagpre
                        + br'\d+) (?P<type>[A-Z]+) (?P<data>.*)', re.ASCII)

        # Get server welcome message,
        # request and store CAPABILITY response.

        if __debug__:
            self._cmd_log_len = 10
            self._cmd_log_idx = 0
            self._cmd_log = {}           # Last `_cmd_log_len' interactions
            if self.debug >= 1:
                self._mesg('imaplib version %s' % __version__)
                self._mesg('new IMAP4 connection, tag=%s' % self.tagpre)

        self.welcome = self._get_response()
        if 'PREAUTH' in self.untagged_responses:
            self.state = 'AUTH'
        elif 'OK' in self.untagged_responses:
            self.state = 'NONAUTH'
        else:
            raise self.error(self.welcome)

        self._get_capabilities()
        if __debug__:
            if self.debug >= 3:
                self._mesg('CAPABILITIES: %r' % (self.capabilities,))

        for version in AllowedVersions:
            if not version in self.capabilities:
                continue
            self.PROTOCOL_VERSION = version
            return

        raise self.error('server not IMAP4 compliant') 

def _get_capabilities(self):
        typ, dat = self.capability()
        if dat == [None]:
            raise self.error('no CAPABILITY response from server')
        dat = str(dat[-1], "ASCII")
        dat = dat.upper()
        self.capabilities = tuple(dat.split()) 

def _new_tag(self):

        tag = self.tagpre + bytes(str(self.tagnum), 'ASCII')
        self.tagnum = self.tagnum + 1
        self.tagged_commands[tag] = None
        return tag 

def test_ascii_and_unicode_flag(self):
        # String patterns
        for flags in (0, re.UNICODE):
            pat = re.compile('\xc0', flags | re.IGNORECASE)
            self.assertNotEqual(pat.match('\xe0'), None)
            pat = re.compile('\w', flags)
            self.assertNotEqual(pat.match('\xe0'), None)
        pat = re.compile('\xc0', re.ASCII | re.IGNORECASE)
        self.assertEqual(pat.match('\xe0'), None)
        pat = re.compile('(?a)\xc0', re.IGNORECASE)
        self.assertEqual(pat.match('\xe0'), None)
        pat = re.compile('\w', re.ASCII)
        self.assertEqual(pat.match('\xe0'), None)
        pat = re.compile('(?a)\w')
        self.assertEqual(pat.match('\xe0'), None)
        # Bytes patterns
        for flags in (0, re.ASCII):
            pat = re.compile(b'\xc0', re.IGNORECASE)
            self.assertEqual(pat.match(b'\xe0'), None)
            pat = re.compile(b'\w')
            self.assertEqual(pat.match(b'\xe0'), None)
        # Incompatibilities
        self.assertRaises(ValueError, re.compile, b'\w', re.UNICODE)
        self.assertRaises(ValueError, re.compile, b'(?u)\w')
        self.assertRaises(ValueError, re.compile, '\w', re.UNICODE | re.ASCII)
        self.assertRaises(ValueError, re.compile, '(?u)\w', re.ASCII)
        self.assertRaises(ValueError, re.compile, '(?a)\w', re.UNICODE)
        self.assertRaises(ValueError, re.compile, '(?au)\w') 

def expanduser(path):
    """Expand ~ and ~user constructions.  If user or $HOME is unknown,
    do nothing."""
    if isinstance(path, bytes):
        tilde = b'~'
    else:
        tilde = '~'
    if not path.startswith(tilde):
        return path
    sep = _get_sep(path)
    i = path.find(sep, 1)
    if i < 0:
        i = len(path)
    if i == 1:
        if 'HOME' not in os.environ:
            import pwd
            userhome = pwd.getpwuid(os.getuid()).pw_dir
        else:
            userhome = os.environ['HOME']
    else:
        import pwd
        name = path[1:i]
        if isinstance(name, bytes):
            name = str(name, 'ASCII')
        try:
            pwent = pwd.getpwnam(name)
        except KeyError:
            return path
        userhome = pwent.pw_dir
    if isinstance(path, bytes):
        userhome = os.fsencode(userhome)
        root = b'/'
    else:
        root = '/'
    userhome = userhome.rstrip(root) or userhome
    return userhome + path[i:]


# Expand paths containing shell variable substitutions.
# This expands the forms $variable and ${variable} only.
# Non-existent variables are left unchanged. 

def unquote(s):
    """Turn a string in the form =AB to the ASCII character with value 0xab"""
    return chr(int(s[1:3], 16)) 

def _unquote_match(match):
    """Turn a match in the form =AB to the ASCII character with value 0xab"""
    s = match.group(0)
    return unquote(s)


# Header decoding is done a bit differently 

def open_data(self, url, data=None):
        """Use "data" URL."""
        if not isinstance(url, str):
            raise URLError('data error', 'proxy support for data protocol currently not implemented')
        # ignore POSTed data
        #
        # syntax of data URLs:
        # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
        # mediatype := [ type "/" subtype ] *( ";" parameter )
        # data      := *urlchar
        # parameter := attribute "=" value
        try:
            [type, data] = url.split(',', 1)
        except ValueError:
            raise IOError('data error', 'bad data URL')
        if not type:
            type = 'text/plain;charset=US-ASCII'
        semi = type.rfind(';')
        if semi >= 0 and '=' not in type[semi:]:
            encoding = type[semi+1:]
            type = type[:semi]
        else:
            encoding = ''
        msg = []
        msg.append('Date: %s'%time.strftime('%a, %d %b %Y %H:%M:%S GMT',
                                            time.gmtime(time.time())))
        msg.append('Content-type: %s' % type)
        if encoding == 'base64':
            # XXX is this encoding/decoding ok?
            data = base64.decodebytes(data.encode('ascii')).decode('latin1')
        else:
            data = unquote(data)
        msg.append('Content-Length: %d' % len(data))
        msg.append('')
        msg.append(data)
        msg = '\n'.join(msg)
        headers = email.message_from_string(msg)
        f = io.StringIO(msg)
        #f.fileno = None     # needed for addinfourl
        return addinfourl(f, headers, url) 

def expanduser(path):
    """Expand ~ and ~user constructions.  If user or $HOME is unknown,
    do nothing."""
    if isinstance(path, bytes):
        tilde = b'~'
    else:
        tilde = '~'
    if not path.startswith(tilde):
        return path
    sep = _get_sep(path)
    i = path.find(sep, 1)
    if i < 0:
        i = len(path)
    if i == 1:
        if 'HOME' not in os.environ:
            import pwd
            userhome = pwd.getpwuid(os.getuid()).pw_dir
        else:
            userhome = os.environ['HOME']
    else:
        import pwd
        name = path[1:i]
        if isinstance(name, bytes):
            name = str(name, 'ASCII')
        try:
            pwent = pwd.getpwnam(name)
        except KeyError:
            return path
        userhome = pwent.pw_dir
    if isinstance(path, bytes):
        userhome = os.fsencode(userhome)
        root = b'/'
    else:
        root = '/'
    userhome = userhome.rstrip(root)
    return (userhome + path[i:]) or root


# Expand paths containing shell variable substitutions.
# This expands the forms $variable and ${variable} only.
# Non-existent variables are left unchanged. 

def compat(self, token, iterable):
        startline = False
        indents = []
        toks_append = self.tokens.append
        toknum, tokval = token
        if toknum in (NAME, NUMBER):
            tokval += ' '
        if toknum in (NEWLINE, NL):
            startline = True
        for tok in iterable:
            toknum, tokval = tok[:2]

            if toknum in (NAME, NUMBER, ASYNC, AWAIT):
                tokval += ' '

            if toknum == INDENT:
                indents.append(tokval)
                continue
            elif toknum == DEDENT:
                indents.pop()
                continue
            elif toknum in (NEWLINE, NL):
                startline = True
            elif startline and indents:
                toks_append(indents[-1])
                startline = False
            toks_append(tokval)

# Commented out because re.ASCII not in Python 2.
#cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII)
#blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) 

def ascii_regex(pattern):
    flags = re.ASCII if six.PY3 else 0
    return re.compile(pattern, flags) 

def __init__(self, negate=False, flags=0):
        self._categories = set()
        self._whitelist_chars = set()
        self._blacklist_chars = set()
        self._negate = negate
        self._ignorecase = flags & re.IGNORECASE
        self._unicode = (not flags & re.ASCII) \
            if six.PY3 else bool(flags & re.UNICODE)