Python shlex.shlex() Examples

def parse_config_string(config_string, issue_warnings=True):
    """
    Parses a config string (comma-separated key=value components) into a dict.
    """
    config_dict = {}
    my_splitter = shlex.shlex(config_string, posix=True)
    my_splitter.whitespace = ','
    my_splitter.whitespace_split = True
    for kv_pair in my_splitter:
        kv_pair = kv_pair.strip()
        if not kv_pair:
            continue
        kv_tuple = kv_pair.split('=', 1)
        if len(kv_tuple) == 1:
            if issue_warnings:
                TheanoConfigWarning.warn(
                    ("Config key '%s' has no value, ignoring it"
                        % kv_tuple[0]),
                    stacklevel=1)
        else:
            k, v = kv_tuple
            # subsequent values for k will override earlier ones
            config_dict[k] = v
    return config_dict 

def single_quote(s):
    """Escape a string with single quotes in order to be parsed as a single element by shlex

    Parameters
    ----------
    s : str
        The string to quote

    Returns
    -------
    str
       The quoted string
    """
    if not s:
        return "''"
    if find_unsafe(s) is None:
        return s

    # use single quotes, and put single quotes into double quotes
    # the string $'b is then quoted as '$'"'"'b'
    return "'" + s.replace("'", "'\"'\"'") + "'" 

def double_quote(s):
    """Escape a string with double quotes in order to be parsed as a single element by shlex

    Parameters
    ----------
    s : str
        The string to quote

    Returns
    -------
    str
       The quoted string
    """
    if not s:
        return '""'
    if find_unsafe(s) is None:
        return s

    # use double quotes, and put double quotes into single quotes
    # the string $"b is then quoted as "$"'"'"b"
    return '"' + s.replace('"', '"\'"\'"') + '"' 

def parse_config_string(config_string, issue_warnings=True):
    """
    Parses a config string (comma-separated key=value components) into a dict.
    """
    config_dict = {}
    my_splitter = shlex.shlex(config_string, posix=True)
    my_splitter.whitespace = ','
    my_splitter.whitespace_split = True
    for kv_pair in my_splitter:
        kv_pair = kv_pair.strip()
        if not kv_pair:
            continue
        kv_tuple = kv_pair.split('=', 1)
        if len(kv_tuple) == 1:
            if issue_warnings:
                TheanoConfigWarning.warn(
                    ("Config key '%s' has no value, ignoring it"
                        % kv_tuple[0]),
                    stacklevel=1)
        else:
            k, v = kv_tuple
            # subsequent values for k will override earlier ones
            config_dict[k] = v
    return config_dict 

def tokenize(self, line: str) -> List[str]:
        """Lex a string into a list of tokens.

        Comments are removed, and shortcuts and aliases are expanded.

        Raises ValueError if there are unclosed quotation marks.
        """

        # strip C-style comments
        # shlex will handle the python/shell style comments for us
        line = re.sub(self.comment_pattern, self._comment_replacer, line)

        # expand shortcuts and aliases
        line = self._expand(line)

        # split on whitespace
        lexer = shlex.shlex(line, posix=False)
        lexer.whitespace_split = True

        # custom lexing
        tokens = self._split_on_punctuation(list(lexer))
        return tokens 

def testQuote(self):
        safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./'
        unicode_sample = '\xe9\xe0\xdf'  # e + acute accent, a + grave, sharp s
        unsafe = '"`$\\!' + unicode_sample

        self.assertEqual(shlex.quote(''), "''")
        self.assertEqual(shlex.quote(safeunquoted), safeunquoted)
        self.assertEqual(shlex.quote('test file name'), "'test file name'")
        for u in unsafe:
            self.assertEqual(shlex.quote('test%sname' % u),
                             "'test%sname'" % u)
        for u in unsafe:
            self.assertEqual(shlex.quote("test%s'name'" % u),
                             "'test%s'\"'\"'name'\"'\"''" % u)

# Allow this test to be used with old shlex.py 

def parseH(self, file):
        lex = shlex.shlex(file)
        lex.commenters = "//"
        token = " "
        while token is not None:
            token = lex.get_token()
            if token == "" or token is None:
                token = None
            else:
                if token=='define':
                    n = lex.get_token()
                    i = int(lex.get_token())
                    self.ids[n] = i
                    if i in self.names:
                        # Dupe ID really isn't a problem - most consumers
                        # want to go from name->id, and this is OK.
                        # It means you can't go from id->name though.
                        pass
                        # ignore AppStudio special ones
                        #if not n.startswith("_APS_"):
                        #    print "Duplicate id",i,"for",n,"is", self.names[i]
                    else:
                        self.names[i] = n
                    if self.next_id<=i:
                        self.next_id = i+1 

def processFileList(self, sFileList):
        """ Process the files in a file list.
        """
        flist = self.openInputFile(sFileList)
        lines = flist.readlines()
        flist.close()
        for l in lines:
            # Use shlex to parse the line like a shell.
            lex = shlex.shlex(l, posix=True)
            lex.whitespace_split = True
            lex.commenters = '#'
            # No escapes, so that backslash can be part of the path
            lex.escape = ''
            args = list(lex)
            if args:
                self.processArguments(args) 

def currentQuotedString(self):
        # Handle quoted strings - pity shlex doesn't handle it.
        assert self.token.startswith('"'), self.token
        bits = [self.token]
        while 1:
            tok = self.getToken()
            if not tok.startswith('"'):
                self.ungetToken()
                break
            bits.append(tok)
        sval = "".join(bits)[1:-1] # Remove end quotes.            
        # Fixup quotes in the body, and all (some?) quoted characters back
        # to their raw value.
        for i, o in ('""', '"'), ("\\r", "\r"), ("\\n", "\n"), ("\\t", "\t"):
            sval = sval.replace(i, o)
        return sval 

def _expand_args(command):
    """Parses command strings and returns a Popen-ready list."""

    # Prepare arguments.
    if isinstance(command, STR_TYPES):
        if sys.version_info[0] == 2:
            splitter = shlex.shlex(command.encode('utf-8'))
        elif sys.version_info[0] == 3:
            splitter = shlex.shlex(command)
        else:
            splitter = shlex.shlex(command.encode('utf-8'))
        splitter.whitespace = '|'
        splitter.whitespace_split = True
        command = []

        while True:
            token = splitter.get_token()
            if token:
                command.append(token)
            else:
                break

        command = list(map(shlex.split, command))

    return command 

def double_quote(s):
    """Escape a string with double quotes in order to be parsed as a single element by shlex

    Parameters
    ----------
    s : str
        The string to quote

    Returns
    -------
    str
       The quoted string
    """
    if not s:
        return '""'
    if find_unsafe(s) is None:
        return s

    # use double quotes, and put double quotes into single quotes
    # the string $"b is then quoted as "$"'"'"b"
    return '"' + s.replace('"', '"\'"\'"') + '"' 

def single_quote(s):
    """Escape a string with single quotes in order to be parsed as a single element by shlex

    Parameters
    ----------
    s : str
        The string to quote

    Returns
    -------
    str
       The quoted string
    """
    if not s:
        return "''"
    if find_unsafe(s) is None:
        return s

    # use single quotes, and put single quotes into double quotes
    # the string $'b is then quoted as '$'"'"'b'
    return "'" + s.replace("'", "'\"'\"'") + "'" 

def _expand_args(command):
    """Parses command strings and returns a Popen-ready list."""

    # Prepare arguments.
    if isinstance(command, STR_TYPES):
        if sys.version_info[0] == 2:
            splitter = shlex.shlex(command.encode("utf-8"))
        elif sys.version_info[0] == 3:
            splitter = shlex.shlex(command)
        else:
            splitter = shlex.shlex(command.encode("utf-8"))
        splitter.whitespace = "|"
        splitter.whitespace_split = True
        command = []

        while True:
            token = splitter.get_token()
            if token:
                command.append(token)
            else:
                break

        command = list(map(shlex.split, command))

    return command 

def testQuote(self):
        safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./'
        unicode_sample = '\xe9\xe0\xdf'  # e + acute accent, a + grave, sharp s
        unsafe = '"`$\\!' + unicode_sample

        self.assertEqual(shlex.quote(''), "''")
        self.assertEqual(shlex.quote(safeunquoted), safeunquoted)
        self.assertEqual(shlex.quote('test file name'), "'test file name'")
        for u in unsafe:
            self.assertEqual(shlex.quote('test%sname' % u),
                             "'test%sname'" % u)
        for u in unsafe:
            self.assertEqual(shlex.quote("test%s'name'" % u),
                             "'test%s'\"'\"'name'\"'\"''" % u)

# Allow this test to be used with old shlex.py 

def parse_config_string(config_string, issue_warnings=True):
    """
    Parses a config string (comma-separated key=value components) into a dict.
    """
    config_dict = {}
    my_splitter = shlex.shlex(config_string, posix=True)
    my_splitter.whitespace = ','
    my_splitter.whitespace_split = True
    for kv_pair in my_splitter:
        kv_pair = kv_pair.strip()
        if not kv_pair:
            continue
        kv_tuple = kv_pair.split('=', 1)
        if len(kv_tuple) == 1:
            if issue_warnings:
                MsafConfigWarning.warn(
                    ("Config key '%s' has no value, ignoring it" %
                     kv_tuple[0]), stacklevel=1)
        else:
            k, v = kv_tuple
            # subsequent values for k will override earlier ones
            config_dict[k] = v
    return config_dict 

def splitTest(self, data, comments):
        for i in range(len(data)):
            l = shlex.split(data[i][0], comments=comments)
            self.assertEqual(l, data[i][1:],
                             "%s: %s != %s" %
                             (data[i][0], l, data[i][1:])) 

def prettySplit(command):
        cmdstring = " ".join(command)

        lex = shlex.shlex(cmdstring)
        lex.whitespace_split = True
        lex.whitespace += "(),"

        command[:] = list(lex) 

def oldSplit(self, s):
        ret = []
        lex = shlex.shlex(io.StringIO(s))
        tok = lex.get_token()
        while tok:
            ret.append(tok)
            tok = lex.get_token()
        return ret 

def _parse_flink_line(line,final_flags):
	lexer=shlex.shlex(line,posix=True)
	lexer.whitespace_split=True
	t=lexer.get_token()
	tmp_flags=[]
	while t:
		t=_parse_flink_token(lexer,t,tmp_flags)
	final_flags.extend(tmp_flags)
	return final_flags 

def onecmd_custom(self, cmd, args):
        if cmd in self.modules_loaded_tree:
            shlex_obj = shlex.shlex(args, posix=False)
            shlex_obj.quotes = '\''
            shlex_obj.whitespace_split = True
            shlex_obj.commenters = ''
            args = list(shlex_obj)
            args = normalize_args(args)
            parsed_response = self.modules_loaded[cmd].run(args)
        else:
            parsed_response = '"#' + cmd + '" Module not found.'
        return parsed_response 

def read_script(cur, arg, **_):
    args = shlex.split(arg)
    if len(args) != 1:
        raise TypeError(".read accepts exactly one path")
    path = args[0]
    with open(path, "r") as f:
        script = f.read()
        cur.executescript(script)
    return [(None, None, None, "")] 

def oldSplit(self, s):
        ret = []
        lex = shlex.shlex(io.StringIO(s))
        tok = lex.get_token()
        while tok:
            ret.append(tok)
            tok = lex.get_token()
        return ret 

def parameter_string_to_list(par):
    import shlex
    parameter_string = shlex.shlex(par)
    parameter_string.quotes = '"'
    parameter_string.whitespace_split = True
    parameter_string.commenters = ''
    parameters = list(parameter_string)
    return parameters 

def load_extension(cur, arg, **_):
    args = shlex.split(arg)
    if len(args) != 1:
        raise TypeError(".load accepts exactly one path")
    path = args[0]
    conn = cur.connection
    conn.enable_load_extension(True)
    conn.load_extension(path)
    return [(None, None, None, "")] 

def __call__(self, value):
        """The actual transformation"""
        transform = lambda s: self.cast(s.strip(self.strip))

        splitter = shlex(value, posix=True)
        splitter.whitespace = self.delimiter
        splitter.whitespace_split = True

        return self.post_process(transform(s) for s in splitter) 

def __init__(self, cast=text_type, delimiter=',', strip=string.whitespace, post_process=list):
        """
        Parameters:
        cast -- callable that transforms the item just before it's added to the list.
        delimiter -- string of delimiters chars passed to shlex.
        strip -- string of non-relevant characters to be passed to str.strip after the split.
        post_process -- callable to post process all casted values. Default is `list`.
        """
        self.cast = cast
        self.delimiter = delimiter
        self.strip = strip
        self.post_process = post_process 

def _parse_flink_line(line,final_flags):
	lexer=shlex.shlex(line,posix=True)
	lexer.whitespace_split=True
	t=lexer.get_token()
	tmp_flags=[]
	while t:
		t=_parse_flink_token(lexer,t,tmp_flags)
	final_flags.extend(tmp_flags)
	return final_flags 

def is_valid_command(self, word: str) -> Tuple[bool, str]:
        """Determine whether a word is a valid name for a command.

        Commands can not include redirection characters, whitespace,
        or termination characters. They also cannot start with a
        shortcut.

        If word is not a valid command, return False and error text
        This string is suitable for inclusion in an error message of your
        choice:

        valid, errmsg = statement_parser.is_valid_command('>')
        if not valid:
            errmsg = "Alias {}".format(errmsg)
        """
        valid = False

        if not word:
            return False, 'cannot be an empty string'

        for (shortcut, _) in self.shortcuts:
            if word.startswith(shortcut):
                # Build an error string with all shortcuts listed
                errmsg = 'cannot start with a shortcut: '
                errmsg += ', '.join(shortcut for (shortcut, _) in self.shortcuts)
                return False, errmsg

        errmsg = 'cannot contain: whitespace, quotes, '
        errchars = []
        errchars.extend(constants.REDIRECTION_CHARS)
        errchars.extend(self.terminators)
        errmsg += ', '.join([shlex.quote(x) for x in errchars])

        match = self._command_pattern.search(word)
        if match:
            if word == match.group(1):
                valid = True
                errmsg = ''
        return valid, errmsg 

def _analyzer(line):
    '''
        A lexical analyzer for simple shell-like syntaxes. Split given line into fields.

    :param line: Line to split.
    :returs    : List of fields.
    :rtype     : ``list``
    '''
    lex                  = shlex.shlex(line)
    lex.quotes           = '"'
    lex.whitespace_split = True
    lex.commenters       = ''

    return list(lex) 

def split_args(line):
    """Version of shlex.split that silently accept incomplete strings.

    Parameters
    ----------
    line : str
        The string to split

    Returns
    -------
    [str]
        The line split in separated arguments
    """
    lex = shlex.shlex(line, posix=True)
    lex.whitespace_split = True
    lex.commenters = ''
    res = []
    try:
        while True:
            res.append(next(lex))
    except ValueError:  # No closing quotation
        pass
    except StopIteration:  # End of loop
        pass
    if lex.token:
        res.append(lex.token)
    return res