Python os.path.commonprefix() Examples

The following are code examples for showing how to use os.path.commonprefix(). They are extracted from open source Python projects. You can vote up the examples you like or vote down the exmaples you don't like. You can also save this page to your account.

Example 1
Project: devsecops-example-helloworld   Author: boozallen   File: util.py    (license) View Source Project 6 votes vote down vote up
def _common_shorten_repr(*args):
    args = tuple(map(safe_repr, args))
    maxlen = max(map(len, args))
    if maxlen <= _MAX_LENGTH:
        return args

    prefix = commonprefix(args)
    prefixlen = len(prefix)

    common_len = _MAX_LENGTH - \
                 (maxlen - prefixlen + _MIN_BEGIN_LEN + _PLACEHOLDER_LEN)
    if common_len > _MIN_COMMON_LEN:
        assert _MIN_BEGIN_LEN + _PLACEHOLDER_LEN + _MIN_COMMON_LEN + \
               (maxlen - prefixlen) < _MAX_LENGTH
        prefix = _shorten(prefix, _MIN_BEGIN_LEN, common_len)
        return tuple(prefix + s[prefixlen:] for s in args)

    prefix = _shorten(prefix, _MIN_BEGIN_LEN, _MIN_COMMON_LEN)
    return tuple(prefix + _shorten(s[prefixlen:], _MIN_DIFF_LEN, _MIN_END_LEN)
                 for s in args) 
Example 2
Project: endpointer   Author: devjack   File: specification.py    (license) View Source Project 6 votes vote down vote up
def _traverse_paths_for(self, filename):

        # Current directory
        if path.isfile(path.abspath(filename)):
            base_directory = path.commonprefix([
                path.dirname(path.abspath(filename)),
                getcwd()
            ])
            self.dirs.append(base_directory)
            return base_directory
        # Try base dir's for files we've previously loaded
        # Lets definitions refer to each other relatively
        for known_file in self.files:
            full_known_file_path = path.abspath(known_file)
            known_directory = path.dirname(full_known_file_path)
            if path.isfile(known_directory + "/" + filename):
                return known_directory

        raise KeyError("Definition not found for %s" % filename) 
Example 3
Project: download-manager   Author: thispc   File: pyLoadGui.py    (license) View Source Project 6 votes vote down vote up
def slotClipboardChange(self):
        """
            called if clipboard changes
        """
        if self.checkClipboard:
            text = self.clipboard.text()
            pattern = re.compile(r"(http|https|ftp)://[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(([0-9]{1,5})?/.*)?")
            matches = pattern.finditer(text)
            
            # thanks to: jmansour //#139
            links = [str(match.group(0)) for match in matches]
            if len(links) == 0:
                return
                
            filenames = [link.rpartition("/")[2] for link in links]
            
            packagename = commonprefix(filenames)
            if len(packagename) == 0:
                packagename = filenames[0]

            self.slotAddPackage(packagename, links) 
Example 4
Project: sflearn   Author: GeorgeArgyros   File: angluin_fst.py    (license) View Source Project 6 votes vote down vote up
def _fill_ot_entry(self, row, col):
        """
        Fill an entry of the observation table.
        Only save the part of the output generated by the col parameter.

        Args:
            row(tuple(int)): A tuple of integers specifiying the row to fill.
            col(tuple(int)): A tuple of integers specifying the column to fill.
        """
        prefix = self.membership_query(row)
        full_output = self.membership_query(row + col)

        common_prefix_len = len(commonprefix([prefix, full_output]))
        self.ot[row, col] = full_output[common_prefix_len:]

    #########################################################################
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
Example 5
Project: scrapy_rss   Author: woxcab   File: utils.py    (license) View Source Project 6 votes vote down vote up
def _common_shorten_repr(*args):
        args = tuple(map(safe_repr, args))
        maxlen = max(map(len, args))
        if maxlen <= _MAX_LENGTH:
            return args

        prefix = commonprefix(args)
        prefixlen = len(prefix)

        common_len = _MAX_LENGTH - \
                     (maxlen - prefixlen + _MIN_BEGIN_LEN + _PLACEHOLDER_LEN)
        if common_len > _MIN_COMMON_LEN:
            assert _MIN_BEGIN_LEN + _PLACEHOLDER_LEN + _MIN_COMMON_LEN + \
                   (maxlen - prefixlen) < _MAX_LENGTH
            prefix = _shorten(prefix, _MIN_BEGIN_LEN, common_len)
            return tuple(prefix + s[prefixlen:] for s in args)

        prefix = _shorten(prefix, _MIN_BEGIN_LEN, _MIN_COMMON_LEN)
        return tuple(prefix + _shorten(s[prefixlen:], _MIN_DIFF_LEN, _MIN_END_LEN)
                     for s in args) 
Example 6
Project: ouroboros   Author: pybee   File: util.py    (license) View Source Project 6 votes vote down vote up
def _common_shorten_repr(*args):
    args = tuple(map(safe_repr, args))
    maxlen = max(map(len, args))
    if maxlen <= _MAX_LENGTH:
        return args

    prefix = commonprefix(args)
    prefixlen = len(prefix)

    common_len = _MAX_LENGTH - \
                 (maxlen - prefixlen + _MIN_BEGIN_LEN + _PLACEHOLDER_LEN)
    if common_len > _MIN_COMMON_LEN:
        assert _MIN_BEGIN_LEN + _PLACEHOLDER_LEN + _MIN_COMMON_LEN + \
               (maxlen - prefixlen) < _MAX_LENGTH
        prefix = _shorten(prefix, _MIN_BEGIN_LEN, common_len)
        return tuple(prefix + s[prefixlen:] for s in args)

    prefix = _shorten(prefix, _MIN_BEGIN_LEN, _MIN_COMMON_LEN)
    return tuple(prefix + _shorten(s[prefixlen:], _MIN_DIFF_LEN, _MIN_END_LEN)
                 for s in args) 
Example 7
Project: kbe_server   Author: xiaohaoppy   File: util.py    (license) View Source Project 6 votes vote down vote up
def _common_shorten_repr(*args):
    args = tuple(map(safe_repr, args))
    maxlen = max(map(len, args))
    if maxlen <= _MAX_LENGTH:
        return args

    prefix = commonprefix(args)
    prefixlen = len(prefix)

    common_len = _MAX_LENGTH - \
                 (maxlen - prefixlen + _MIN_BEGIN_LEN + _PLACEHOLDER_LEN)
    if common_len > _MIN_COMMON_LEN:
        assert _MIN_BEGIN_LEN + _PLACEHOLDER_LEN + _MIN_COMMON_LEN + \
               (maxlen - prefixlen) < _MAX_LENGTH
        prefix = _shorten(prefix, _MIN_BEGIN_LEN, common_len)
        return tuple(prefix + s[prefixlen:] for s in args)

    prefix = _shorten(prefix, _MIN_BEGIN_LEN, _MIN_COMMON_LEN)
    return tuple(prefix + _shorten(s[prefixlen:], _MIN_DIFF_LEN, _MIN_END_LEN)
                 for s in args) 
Example 8
Project: automation-examples   Author: globus   File: cleanup_cache.py    (license) View Source Project 6 votes vote down vote up
def select_dir_to_delete(transfer_client, task):
    """Find the common directory under which all the files live. If one exists,
    it will be deleted recursively, even if not all files under it were
    transferred. If there is no common directory, each file that was
    transferred will be deleted"""
    successful_file_transfers = \
        transfer_client.endpoint_manager_task_successful_transfers(
            task["task_id"]
        )
    print("Transfer Task({}): {} -> {}\n was submitted by {}\n".
          format(task["task_id"], task["source_endpoint"],
                 task["destination_endpoint"],
                 task["owner_string"]))

    files_list = [
        globr["source_path"] for globr in successful_file_transfers]
    print("files list is ", files_list)

    common_dir = dirname(commonprefix(files_list))
    return files_list, common_dir 
Example 9
Project: libmozdata   Author: mozilla   File: versions.py    (license) View Source Project 5 votes vote down vote up
def __getMatchingVersion(version, versions_dates):
    date = None
    longest_match = []
    longest_match_v = None
    for v, d in versions_dates:
        match = commonprefix([v.split('.'), str(version).split('.')])
        if len(match) > 0 and (len(match) > len(longest_match) or (len(match) == len(longest_match) and int(v[-1]) <= int(longest_match_v[-1]))):
            longest_match = match
            longest_match_v = v
            date = d

    return date 
Example 10
Project: pomu   Author: Hummer12007   File: package.py    (license) View Source Project 5 votes vote down vote up
def strip_root(self, d_path):
        """Strip the root component of d_path"""
        # the path should be either relative, or a child of root
        if d_path.startswith('/'):
            if path.commonprefix([d_path, self.root]) != self.root:
                raise ValueError('Path should be a subdirectory of root')
            return strip_prefix(strip_prefix(d_path, self.root), '/')
        return d_path 
Example 11
Project: beansoup   Author: fxtlabs   File: transactions.py    (license) View Source Project 5 votes vote down vote up
def score_model(self, model_txn, txn):
        """Score an existing transaction for its ability to provide a model
        for an incomplete transaction.

        Args:
          model_txn: The transaction to be scored.
          txn: The incomplete transaction.
        Returns:
          A float number representing the score, normalized in [0,1].
        """
        def get_description(txn):
            return ('{} {}'.format(txn.payee or '', txn.narration or '')).strip()

        # If the target transaction does not have a description, there is
        # nothing we can do
        txn_description = get_description(txn)
        n_max = len(txn_description)
        if n_max > 1:
            # Only consider model transactions whose posting to the target
            # account has the same sign as the transaction to be completed
            posting = [p for p in model_txn.postings if p.account == self.account][0]
            if number.same_sign(posting.units.number, txn.postings[0].units.number):
                model_txn_description = get_description(model_txn)
                n_match = len(path.commonprefix(
                    [model_txn_description, txn_description]))
                score = float(n_match) / float(n_max)
                return score
        return 0 
Example 12
Project: download-manager   Author: thispc   File: filters.py    (license) View Source Project 5 votes vote down vote up
def relpath(path, start=curdir):
        """Return a relative version of a path"""
        if not path:
            raise ValueError("no path specified")
        start_list = abspath(start).split(sep)
        path_list = abspath(path).split(sep)
        # Work out how much of the filepath is shared by start and path.
        i = len(commonprefix([start_list, path_list]))
        rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
        if not rel_list:
            return curdir
        return join(*rel_list) 
Example 13
Project: sflearn   Author: GeorgeArgyros   File: angluin_fst_lookahead.py    (license) View Source Project 5 votes vote down vote up
def _remove_common_prefix(main, prefix):
    """
    Return the suffix of main after removing its common prefix with "prefix"
    Args:
        main (list): List to return suffix
        prefix (list): list to match prefix

    Returns:
        list: suffix of main after removing common prefix with prefix list.
    """
    common_part = commonprefix([main, prefix])
    return main[len(common_part):] 
Example 14
Project: sflearn   Author: GeorgeArgyros   File: angluin_fst_lookahead.py    (license) View Source Project 5 votes vote down vote up
def _fill_ot_entry(self, row, col):
        """
        Fill an entry of the observation table.
        Only save the part of the output generated by the col parameter.

        Args:
            row(tuple(int)): A tuple of integers specifiying the row to fill.
            col(tuple(int)): A tuple of integers specifying the column to fill.
        """

        prefix = self.membership_query(row)
        full_output = self.membership_query(row + col)

        prefix_len = len(commonprefix([prefix, full_output]))
        self.ot[row, col] = full_output[prefix_len:] 
Example 15
Project: sflearn   Author: GeorgeArgyros   File: angluin_fst_lookahead.py    (license) View Source Project 5 votes vote down vote up
def _process_counterexample(self, ce):
        """
        Counterexample processing method. The method is similar with the
        Shabaz-Groz counterexample processing with an additional module to
        check for counterexamples resulting from lookahead transitions.

        Args:
            ce (list): counterexample input
        """
        # Process lookaheads
        self._check_lookahead(ce)

        #Finding longest prefix among strings in access_strings
        maxlen = 0
        for row in self.ot.access_strings:
            if not row:
                continue
            # Seems that commonprefix works for tuple/list pairs but convert
            # just to be sure.
            prefix = commonprefix([ce, list(row)])
            if len(prefix) > maxlen:
                maxlen = len(prefix)

        # Add the all the suffixes as experiments in distinguishing strings
        suff = ()
        for c in reversed(ce[maxlen:]):
            suff = (c,) + suff
            # Add the experiment if not already there
            if suff not in self.ot.dist_strings:
                self.ot.dist_strings.append(suff)

            # Fill the entries in the observation table
            for row in self.ot.access_strings + self.ot.transitions:
                self._fill_ot_entry(row, suff)

            # Fill the lookahead transitions
            for (src, inp, _) in self.ot.lookaheads:
                self._fill_ot_entry(src+inp, suff) 
Example 16
Project: sflearn   Author: GeorgeArgyros   File: angluin_fst.py    (license) View Source Project 5 votes vote down vote up
def _process_ce_sg(self, ce):
        """
        Counterexample processing using the Shabaz-Groz algorithm.

        Args:
            ce (list): counterexample input
        """

        #Finding longest prefix among strings in S_m
        maxlen = 0
        for row in self.ot.access_strings:
            if not row:
                continue
            # Seems that commonprefix works for tuple/list pairs but convert
            # just to be sure.
            prefix = commonprefix([ce, list(row)])
            if len(prefix) > maxlen:
                maxlen = len(prefix)

        # Add the all the suffixes as experiments in E_m
        suff = ()
        for c in reversed(ce[maxlen:]):
            suff = (c,) + suff
            # Add the experiment if not already there
            if suff not in self.ot.dist_strings:
                self.ot.dist_strings.append(suff)

            # Fill the entries in the observation table
            for row in self.ot.access_strings + self.ot.transitions:
                self._fill_ot_entry(row, suff)


    #########################################################################
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
Example 17
Project: sardana   Author: sardana-org   File: macroserver.py    (license) View Source Project 5 votes vote down vote up
def refresh(self):
        self.macro_path = mp = self._ms().get_property("MacroPath")[
            "MacroPath"]
        self.base_macro_path = osp.commonprefix(self.macro_path)
        self.rel_macro_path = [osp.relpath for p in mp, self.base_macro_path] 
Example 18
Project: libSigNetSim   Author: vincent-noel   File: Source.py    (license) View Source Project 5 votes vote down vote up
def makeRelativePath(self, path):

		if self.__sourceType == self.LOCAL and (isabs(self.__source) or commonprefix([self.__source, path]) != ""):
			self.__source = relpath(self.__source, path) 
Example 19
Project: sublime-text-3-packages   Author: nickjj   File: regexopt.py    (MIT License) View Source Project 4 votes vote down vote up
def regex_opt_inner(strings, open_paren):
    """Return a regex that matches any string in the sorted list of strings."""
    close_paren = open_paren and ')' or ''
    # print strings, repr(open_paren)
    if not strings:
        # print '-> nothing left'
        return ''
    first = strings[0]
    if len(strings) == 1:
        # print '-> only 1 string'
        return open_paren + escape(first) + close_paren
    if not first:
        # print '-> first string empty'
        return open_paren + regex_opt_inner(strings[1:], '(?:') \
            + '?' + close_paren
    if len(first) == 1:
        # multiple one-char strings? make a charset
        oneletter = []
        rest = []
        for s in strings:
            if len(s) == 1:
                oneletter.append(s)
            else:
                rest.append(s)
        if len(oneletter) > 1:  # do we have more than one oneletter string?
            if rest:
                # print '-> 1-character + rest'
                return open_paren + regex_opt_inner(rest, '') + '|' \
                    + make_charset(oneletter) + close_paren
            # print '-> only 1-character'
            return make_charset(oneletter)
    prefix = commonprefix(strings)
    if prefix:
        plen = len(prefix)
        # we have a prefix for all strings
        # print '-> prefix:', prefix
        return open_paren + escape(prefix) \
            + regex_opt_inner([s[plen:] for s in strings], '(?:') \
            + close_paren
    # is there a suffix?
    strings_rev = [s[::-1] for s in strings]
    suffix = commonprefix(strings_rev)
    if suffix:
        slen = len(suffix)
        # print '-> suffix:', suffix[::-1]
        return open_paren \
            + regex_opt_inner(sorted(s[:-slen] for s in strings), '(?:') \
            + escape(suffix[::-1]) + close_paren
    # recurse on common 1-string prefixes
    # print '-> last resort'
    return open_paren + \
        '|'.join(regex_opt_inner(list(group[1]), '')
                 for group in groupby(strings, lambda s: s[0] == first[0])) \
        + close_paren 
Example 20
Project: macos-st-packages   Author: zce   File: regexopt.py    (license) View Source Project 4 votes vote down vote up
def regex_opt_inner(strings, open_paren):
    """Return a regex that matches any string in the sorted list of strings."""
    close_paren = open_paren and ')' or ''
    # print strings, repr(open_paren)
    if not strings:
        # print '-> nothing left'
        return ''
    first = strings[0]
    if len(strings) == 1:
        # print '-> only 1 string'
        return open_paren + escape(first) + close_paren
    if not first:
        # print '-> first string empty'
        return open_paren + regex_opt_inner(strings[1:], '(?:') \
            + '?' + close_paren
    if len(first) == 1:
        # multiple one-char strings? make a charset
        oneletter = []
        rest = []
        for s in strings:
            if len(s) == 1:
                oneletter.append(s)
            else:
                rest.append(s)
        if len(oneletter) > 1:  # do we have more than one oneletter string?
            if rest:
                # print '-> 1-character + rest'
                return open_paren + regex_opt_inner(rest, '') + '|' \
                    + make_charset(oneletter) + close_paren
            # print '-> only 1-character'
            return make_charset(oneletter)
    prefix = commonprefix(strings)
    if prefix:
        plen = len(prefix)
        # we have a prefix for all strings
        # print '-> prefix:', prefix
        return open_paren + escape(prefix) \
            + regex_opt_inner([s[plen:] for s in strings], '(?:') \
            + close_paren
    # is there a suffix?
    strings_rev = [s[::-1] for s in strings]
    suffix = commonprefix(strings_rev)
    if suffix:
        slen = len(suffix)
        # print '-> suffix:', suffix[::-1]
        return open_paren \
            + regex_opt_inner(sorted(s[:-slen] for s in strings), '(?:') \
            + escape(suffix[::-1]) + close_paren
    # recurse on common 1-string prefixes
    # print '-> last resort'
    return open_paren + \
        '|'.join(regex_opt_inner(list(group[1]), '')
                 for group in groupby(strings, lambda s: s[0] == first[0])) \
        + close_paren 
Example 21
Project: leetcode   Author: thomasyimgit   File: regexopt.py    (license) View Source Project 4 votes vote down vote up
def regex_opt_inner(strings, open_paren):
    """Return a regex that matches any string in the sorted list of strings."""
    close_paren = open_paren and ')' or ''
    # print strings, repr(open_paren)
    if not strings:
        # print '-> nothing left'
        return ''
    first = strings[0]
    if len(strings) == 1:
        # print '-> only 1 string'
        return open_paren + escape(first) + close_paren
    if not first:
        # print '-> first string empty'
        return open_paren + regex_opt_inner(strings[1:], '(?:') \
            + '?' + close_paren
    if len(first) == 1:
        # multiple one-char strings? make a charset
        oneletter = []
        rest = []
        for s in strings:
            if len(s) == 1:
                oneletter.append(s)
            else:
                rest.append(s)
        if len(oneletter) > 1:  # do we have more than one oneletter string?
            if rest:
                # print '-> 1-character + rest'
                return open_paren + regex_opt_inner(rest, '') + '|' \
                    + make_charset(oneletter) + close_paren
            # print '-> only 1-character'
            return open_paren + make_charset(oneletter) + close_paren
    prefix = commonprefix(strings)
    if prefix:
        plen = len(prefix)
        # we have a prefix for all strings
        # print '-> prefix:', prefix
        return open_paren + escape(prefix) \
            + regex_opt_inner([s[plen:] for s in strings], '(?:') \
            + close_paren
    # is there a suffix?
    strings_rev = [s[::-1] for s in strings]
    suffix = commonprefix(strings_rev)
    if suffix:
        slen = len(suffix)
        # print '-> suffix:', suffix[::-1]
        return open_paren \
            + regex_opt_inner(sorted(s[:-slen] for s in strings), '(?:') \
            + escape(suffix[::-1]) + close_paren
    # recurse on common 1-string prefixes
    # print '-> last resort'
    return open_paren + \
        '|'.join(regex_opt_inner(list(group[1]), '')
                 for group in groupby(strings, lambda s: s[0] == first[0])) \
        + close_paren 
Example 22
Project: grammarinator   Author: renatahodovan   File: parser_builder.py    (license) View Source Project 4 votes vote down vote up
def build_grammars(out, antlr):
    """
    Build lexer and grammar from ANTLRv4 grammar files in Python3 target.

    :param out: Output directory.
    :param antlr: Path to the ANTLR4 tool (Java jar binary).
    :return: List of references/names of the lexer, parser and listener classes of the target.
    """
    try:
        # TODO: support Java parsers too.
        languages = {
            'python': {'antlr_arg': '-Dlanguage=Python3',
                       'ext': 'py',
                       'listener_format': 'Listener',
                       'sources': ['ANTLRv4Lexer.g4', 'ANTLRv4Parser.g4', 'LexBasic.g4', 'LexerAdaptor.py']}
        }

        # Copy the grammars from the package to the given working directory.
        for resource in languages['python']['sources']:
            with open(join(out, resource), 'wb') as f:
                f.write(get_data(__package__, join('resources', 'antlr', resource)))

        grammars = tuple([file for file in languages['python']['sources'] if file.endswith('.g4')])

        # Generate parser and lexer in the target language and return either with
        # python class ref or the name of java classes.
        cmd = 'java -jar {antlr} {lang} {grammars}'.format(antlr=antlr,
                                                           lang=languages['python']['antlr_arg'],
                                                           grammars=' '.join(grammars))

        with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, cwd=out) as proc:
            stdout, stderr = proc.communicate()
            if proc.returncode:
                logger.error('Building grammars %r failed!\n%s\n%s\n', grammars,
                             stdout.decode('utf-8', 'ignore'),
                             stderr.decode('utf-8', 'ignore'))
                raise CalledProcessError(returncode=proc.returncode, cmd=cmd, output=stdout + stderr)

        files = listdir(out)
        filename = basename(grammars[0])

        def file_endswith(end_pattern):
            return splitext(split(list(
                filter(lambda x: len(commonprefix([filename, x])) > 0 and x.endswith(end_pattern), files))[0])[1])[0]

        # Extract the name of lexer and parser from their path.
        lexer = file_endswith('Lexer.{ext}'.format(ext=languages['python']['ext']))
        parser = file_endswith('Parser.{ext}'.format(ext=languages['python']['ext']))
        # The name of the generated listeners differs if Python or other language target is used.
        listener = file_endswith('{listener_format}.{ext}'.format(listener_format=languages['python']['listener_format'], ext=languages['python']['ext']))

        return (getattr(__import__(x, globals(), locals(), [x], 0), x) for x in [lexer, parser, listener])
    except Exception as e:
        logger.error('Exception while loading parser modules', exc_info=e)
        raise e 
Example 23
Project: chalktalk_docs   Author: loremIpsum1771   File: regexopt.py    (license) View Source Project 4 votes vote down vote up
def regex_opt_inner(strings, open_paren):
    """Return a regex that matches any string in the sorted list of strings."""
    close_paren = open_paren and ')' or ''
    # print strings, repr(open_paren)
    if not strings:
        # print '-> nothing left'
        return ''
    first = strings[0]
    if len(strings) == 1:
        # print '-> only 1 string'
        return open_paren + escape(first) + close_paren
    if not first:
        # print '-> first string empty'
        return open_paren + regex_opt_inner(strings[1:], '(?:') \
            + '?' + close_paren
    if len(first) == 1:
        # multiple one-char strings? make a charset
        oneletter = []
        rest = []
        for s in strings:
            if len(s) == 1:
                oneletter.append(s)
            else:
                rest.append(s)
        if len(oneletter) > 1:  # do we have more than one oneletter string?
            if rest:
                # print '-> 1-character + rest'
                return open_paren + regex_opt_inner(rest, '') + '|' \
                    + make_charset(oneletter) + close_paren
            # print '-> only 1-character'
            return make_charset(oneletter)
    prefix = commonprefix(strings)
    if prefix:
        plen = len(prefix)
        # we have a prefix for all strings
        # print '-> prefix:', prefix
        return open_paren + escape(prefix) \
            + regex_opt_inner([s[plen:] for s in strings], '(?:') \
            + close_paren
    # is there a suffix?
    strings_rev = [s[::-1] for s in strings]
    suffix = commonprefix(strings_rev)
    if suffix:
        slen = len(suffix)
        # print '-> suffix:', suffix[::-1]
        return open_paren \
            + regex_opt_inner(sorted(s[:-slen] for s in strings), '(?:') \
            + escape(suffix[::-1]) + close_paren
    # recurse on common 1-string prefixes
    # print '-> last resort'
    return open_paren + \
        '|'.join(regex_opt_inner(list(group[1]), '')
                 for group in groupby(strings, lambda s: s[0] == first[0])) \
        + close_paren 
Example 24
Project: sflearn   Author: GeorgeArgyros   File: angluin_fst_lookahead.py    (license) View Source Project 4 votes vote down vote up
def _check_lookahead(self, inp):
        """
        Check a counterexample for lookahead transitions using prefix-closed
        queries. If an unknown lookahead is found it is added on the observation
        table.

        Args:
            inp (list): Counterexample input.
        """
        # Make a prefix closed membership query and gather the result
        prefix = []
        prefix_set = [[]]
        prefix_set_input = [[]]
        for c in inp:
            prefix.append(c)
            prefix_set_input.append(prefix)
            prefix_set.append(self.membership_query(prefix))

        for i in xrange(1, len(prefix_set)):
            if commonprefix([prefix_set[i], prefix_set[i-1]]) != prefix_set[i-1]:
                logging.debug('Lookahead detected at position %s : %s, %s',
                              i, prefix_set[i-1], prefix_set[i])

                la_out = _remove_common_prefix(prefix_set[i], prefix_set[i-1])
                j = None
                for j in reversed(xrange(i)):
                    if commonprefix([prefix_set[i], prefix_set[j]]) == prefix_set[j]:
                        la_inp = inp[j:i]
                        break

                la_out = _remove_common_prefix(prefix_set[i], prefix_set[j])
                access_string = self._run_in_hypothesis(inp, j)
                out_as = self.membership_query(access_string)
                out_complete = self.membership_query(list(access_string)+la_inp)

                # If The access string for the lookahead state is wrong, we will
                # add the lookahead path once this is fixed in a next iteration.
                if _remove_common_prefix(out_complete, out_as) != la_out:
                    logging.debug('Lookahead detected but access string is '+ \
                                  'wrong, skipping.')
                    continue
                if  self.ot.add_lookahead_transition(access_string,
                                                     tuple(la_inp),
                                                     tuple(la_out)):
                    # Fill all table entries for the lookahead transition
                    for col in self.ot.dist_strings:
                        self._fill_ot_entry(access_string + tuple(la_inp), col)
                    # New lookahead added, no need for further processing.
                    break 
Example 25
Project: sublimeTextConfig   Author: luoye-fe   File: regexopt.py    (license) View Source Project 4 votes vote down vote up
def regex_opt_inner(strings, open_paren):
    """Return a regex that matches any string in the sorted list of strings."""
    close_paren = open_paren and ')' or ''
    # print strings, repr(open_paren)
    if not strings:
        # print '-> nothing left'
        return ''
    first = strings[0]
    if len(strings) == 1:
        # print '-> only 1 string'
        return open_paren + escape(first) + close_paren
    if not first:
        # print '-> first string empty'
        return open_paren + regex_opt_inner(strings[1:], '(?:') \
            + '?' + close_paren
    if len(first) == 1:
        # multiple one-char strings? make a charset
        oneletter = []
        rest = []
        for s in strings:
            if len(s) == 1:
                oneletter.append(s)
            else:
                rest.append(s)
        if len(oneletter) > 1:  # do we have more than one oneletter string?
            if rest:
                # print '-> 1-character + rest'
                return open_paren + regex_opt_inner(rest, '') + '|' \
                    + make_charset(oneletter) + close_paren
            # print '-> only 1-character'
            return make_charset(oneletter)
    prefix = commonprefix(strings)
    if prefix:
        plen = len(prefix)
        # we have a prefix for all strings
        # print '-> prefix:', prefix
        return open_paren + escape(prefix) \
            + regex_opt_inner([s[plen:] for s in strings], '(?:') \
            + close_paren
    # is there a suffix?
    strings_rev = [s[::-1] for s in strings]
    suffix = commonprefix(strings_rev)
    if suffix:
        slen = len(suffix)
        # print '-> suffix:', suffix[::-1]
        return open_paren \
            + regex_opt_inner(sorted(s[:-slen] for s in strings), '(?:') \
            + escape(suffix[::-1]) + close_paren
    # recurse on common 1-string prefixes
    # print '-> last resort'
    return open_paren + \
        '|'.join(regex_opt_inner(list(group[1]), '')
                 for group in groupby(strings, lambda s: s[0] == first[0])) \
        + close_paren 
Example 26
Project: Repobot   Author: Desgard   File: regexopt.py    (license) View Source Project 4 votes vote down vote up
def regex_opt_inner(strings, open_paren):
    """Return a regex that matches any string in the sorted list of strings."""
    close_paren = open_paren and ')' or ''
    # print strings, repr(open_paren)
    if not strings:
        # print '-> nothing left'
        return ''
    first = strings[0]
    if len(strings) == 1:
        # print '-> only 1 string'
        return open_paren + escape(first) + close_paren
    if not first:
        # print '-> first string empty'
        return open_paren + regex_opt_inner(strings[1:], '(?:') \
            + '?' + close_paren
    if len(first) == 1:
        # multiple one-char strings? make a charset
        oneletter = []
        rest = []
        for s in strings:
            if len(s) == 1:
                oneletter.append(s)
            else:
                rest.append(s)
        if len(oneletter) > 1:  # do we have more than one oneletter string?
            if rest:
                # print '-> 1-character + rest'
                return open_paren + regex_opt_inner(rest, '') + '|' \
                    + make_charset(oneletter) + close_paren
            # print '-> only 1-character'
            return open_paren + make_charset(oneletter) + close_paren
    prefix = commonprefix(strings)
    if prefix:
        plen = len(prefix)
        # we have a prefix for all strings
        # print '-> prefix:', prefix
        return open_paren + escape(prefix) \
            + regex_opt_inner([s[plen:] for s in strings], '(?:') \
            + close_paren
    # is there a suffix?
    strings_rev = [s[::-1] for s in strings]
    suffix = commonprefix(strings_rev)
    if suffix:
        slen = len(suffix)
        # print '-> suffix:', suffix[::-1]
        return open_paren \
            + regex_opt_inner(sorted(s[:-slen] for s in strings), '(?:') \
            + escape(suffix[::-1]) + close_paren
    # recurse on common 1-string prefixes
    # print '-> last resort'
    return open_paren + \
        '|'.join(regex_opt_inner(list(group[1]), '')
                 for group in groupby(strings, lambda s: s[0] == first[0])) \
        + close_paren 
Example 27
Project: kubeface   Author: hammerlab   File: stringable.py    (license) View Source Project 4 votes vote down vote up
def prefixes(self, max_prefixes=1, **fields_dict):
        for (key, value) in fields_dict.items():
            assert key in self.field_names, key
            assert value is None or isinstance(value, list), type(value)

        def make_prefixes(
                template_pieces,
                max_prefixes=max_prefixes,
                fields_dict=fields_dict):
            result = [[]]
            if not template_pieces:
                return result

            (literal, field_name) = template_pieces[0]
            if literal:
                for piece in result:
                    piece.append(literal)

            values = fields_dict.get(field_name)
            if values is None:
                values = self.valid_values.get(field_name)
            if values is not None:
                if len(result) * len(values) > max_prefixes:
                    common_prefix = commonprefix(values)
                    for piece in result:
                        piece.append(common_prefix)
                else:
                    new_result = []
                    for value in values:
                        new_fields_dict = dict(fields_dict)
                        new_fields_dict[field_name] = [value]
                        rest = make_prefixes(
                            template_pieces[1:],
                            max_prefixes=max_prefixes / (
                                len(result) * len(values)),
                            fields_dict=new_fields_dict)
                        for some_rest in rest:
                            new_result.extend(
                                [x + [value] + some_rest for x in result])
                    result = new_result
            return result

        prefix_components = make_prefixes(self.template_pieces)
        assert len(prefix_components) <= max_prefixes
        return [''.join(x) for x in prefix_components] 
Example 28
Project: enkiWS   Author: juliettef   File: regexopt.py    (license) View Source Project 4 votes vote down vote up
def regex_opt_inner(strings, open_paren):
    """Return a regex that matches any string in the sorted list of strings."""
    close_paren = open_paren and ')' or ''
    # print strings, repr(open_paren)
    if not strings:
        # print '-> nothing left'
        return ''
    first = strings[0]
    if len(strings) == 1:
        # print '-> only 1 string'
        return open_paren + escape(first) + close_paren
    if not first:
        # print '-> first string empty'
        return open_paren + regex_opt_inner(strings[1:], '(?:') \
            + '?' + close_paren
    if len(first) == 1:
        # multiple one-char strings? make a charset
        oneletter = []
        rest = []
        for s in strings:
            if len(s) == 1:
                oneletter.append(s)
            else:
                rest.append(s)
        if len(oneletter) > 1:  # do we have more than one oneletter string?
            if rest:
                # print '-> 1-character + rest'
                return open_paren + regex_opt_inner(rest, '') + '|' \
                    + make_charset(oneletter) + close_paren
            # print '-> only 1-character'
            return open_paren + make_charset(oneletter) + close_paren
    prefix = commonprefix(strings)
    if prefix:
        plen = len(prefix)
        # we have a prefix for all strings
        # print '-> prefix:', prefix
        return open_paren + escape(prefix) \
            + regex_opt_inner([s[plen:] for s in strings], '(?:') \
            + close_paren
    # is there a suffix?
    strings_rev = [s[::-1] for s in strings]
    suffix = commonprefix(strings_rev)
    if suffix:
        slen = len(suffix)
        # print '-> suffix:', suffix[::-1]
        return open_paren \
            + regex_opt_inner(sorted(s[:-slen] for s in strings), '(?:') \
            + escape(suffix[::-1]) + close_paren
    # recurse on common 1-string prefixes
    # print '-> last resort'
    return open_paren + \
        '|'.join(regex_opt_inner(list(group[1]), '')
                 for group in groupby(strings, lambda s: s[0] == first[0])) \
        + close_paren 
Example 29
Project: picireny   Author: renatahodovan   File: parser_builder.py    (license) View Source Project 4 votes vote down vote up
def build_grammars(grammars, out, antlr, lang='python'):
    """
    Build lexer and grammar from ANTLRv4 grammar files in Python3 target.

    :param grammars: Tuple of grammar files.
    :param out: Output directory.
    :param antlr: Path to the ANTLR4 tool (Java jar binary).
    :param lang: The target language of the parser.
    :return: List of references/names of the lexer, parser and listener classes of the target.
    """

    # Generate parser and lexer in the target language and return either with
    # python class ref or the name of java classes.
    global grammar_cache
    if lang not in grammar_cache:
        grammar_cache[lang] = {}
    if grammars in grammar_cache[lang]:
        logger.debug('%r is already built with %s target.', grammars, lang)
        return grammar_cache[lang][grammars]

    try:
        languages = {
            'python': {'antlr_arg': '-Dlanguage=Python3', 'ext': 'py', 'listener_format': 'Listener'},
            'java': {'antlr_arg': '', 'ext': 'java', 'listener_format': 'BaseListener'},
        }

        cmd = 'java -jar {antlr} {lang} -o {out} {grammars}'.format(antlr=antlr,
                                                                    lang=languages[lang]['antlr_arg'],
                                                                    out=out,
                                                                    grammars=' '.join(grammars))
        with Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True, cwd=out) as proc:
            stdout, stderr = proc.communicate()
            if proc.returncode:
                logger.error('Building grammars %r failed!\n%s\n%s\n', grammars, stdout, stderr)
                raise CalledProcessError(returncode=proc.returncode, cmd=cmd, output=stdout + stderr)

        files = listdir(out)
        filename = basename(grammars[0])

        def file_endswith(end_pattern):
            return splitext(split(list(
                filter(lambda x: len(commonprefix([filename, x])) > 0 and x.endswith(end_pattern), files))[0])[1])[0]

        # Extract the name of lexer and parser from their path.
        lexer = file_endswith('Lexer.{ext}'.format(ext=languages[lang]['ext']))
        parser = file_endswith('Parser.{ext}'.format(ext=languages[lang]['ext']))
        # The name of the generated listeners differs if Python or other language target is used.
        listener = file_endswith('{listener_format}.{ext}'.format(listener_format=languages[lang]['listener_format'], ext=languages[lang]['ext']))

        if lang == 'python':
            grammar_cache[lang][grammars] = [getattr(__import__(x, globals(), locals(), [x], 0), x) for x in [lexer, parser, listener]]
        else:
            grammar_cache[lang][grammars] = [lexer, parser, listener]

        return grammar_cache[lang][grammars]
    except Exception as e:
        logger.error('Exception while loading parser modules', exc_info=e)
        raise e 
Example 30
Project: python-flask-security   Author: weinbergdavid   File: regexopt.py    (license) View Source Project 4 votes vote down vote up
def regex_opt_inner(strings, open_paren):
    """Return a regex that matches any string in the sorted list of strings."""
    close_paren = open_paren and ')' or ''
    # print strings, repr(open_paren)
    if not strings:
        # print '-> nothing left'
        return ''
    first = strings[0]
    if len(strings) == 1:
        # print '-> only 1 string'
        return open_paren + escape(first) + close_paren
    if not first:
        # print '-> first string empty'
        return open_paren + regex_opt_inner(strings[1:], '(?:') \
            + '?' + close_paren
    if len(first) == 1:
        # multiple one-char strings? make a charset
        oneletter = []
        rest = []
        for s in strings:
            if len(s) == 1:
                oneletter.append(s)
            else:
                rest.append(s)
        if len(oneletter) > 1:  # do we have more than one oneletter string?
            if rest:
                # print '-> 1-character + rest'
                return open_paren + regex_opt_inner(rest, '') + '|' \
                    + make_charset(oneletter) + close_paren
            # print '-> only 1-character'
            return make_charset(oneletter)
    prefix = commonprefix(strings)
    if prefix:
        plen = len(prefix)
        # we have a prefix for all strings
        # print '-> prefix:', prefix
        return open_paren + escape(prefix) \
            + regex_opt_inner([s[plen:] for s in strings], '(?:') \
            + close_paren
    # is there a suffix?
    strings_rev = [s[::-1] for s in strings]
    suffix = commonprefix(strings_rev)
    if suffix:
        slen = len(suffix)
        # print '-> suffix:', suffix[::-1]
        return open_paren \
            + regex_opt_inner(sorted(s[:-slen] for s in strings), '(?:') \
            + escape(suffix[::-1]) + close_paren
    # recurse on common 1-string prefixes
    # print '-> last resort'
    return open_paren + \
        '|'.join(regex_opt_inner(list(group[1]), '')
                 for group in groupby(strings, lambda s: s[0] == first[0])) \
        + close_paren 
Example 31
Project: blender   Author: gastrodia   File: regexopt.py    (license) View Source Project 4 votes vote down vote up
def regex_opt_inner(strings, open_paren):
    """Return a regex that matches any string in the sorted list of strings."""
    close_paren = open_paren and ')' or ''
    # print strings, repr(open_paren)
    if not strings:
        # print '-> nothing left'
        return ''
    first = strings[0]
    if len(strings) == 1:
        # print '-> only 1 string'
        return open_paren + escape(first) + close_paren
    if not first:
        # print '-> first string empty'
        return open_paren + regex_opt_inner(strings[1:], '(?:') \
            + '?' + close_paren
    if len(first) == 1:
        # multiple one-char strings? make a charset
        oneletter = []
        rest = []
        for s in strings:
            if len(s) == 1:
                oneletter.append(s)
            else:
                rest.append(s)
        if len(oneletter) > 1:  # do we have more than one oneletter string?
            if rest:
                # print '-> 1-character + rest'
                return open_paren + regex_opt_inner(rest, '') + '|' \
                    + make_charset(oneletter) + close_paren
            # print '-> only 1-character'
            return make_charset(oneletter)
    prefix = commonprefix(strings)
    if prefix:
        plen = len(prefix)
        # we have a prefix for all strings
        # print '-> prefix:', prefix
        return open_paren + escape(prefix) \
            + regex_opt_inner([s[plen:] for s in strings], '(?:') \
            + close_paren
    # is there a suffix?
    strings_rev = [s[::-1] for s in strings]
    suffix = commonprefix(strings_rev)
    if suffix:
        slen = len(suffix)
        # print '-> suffix:', suffix[::-1]
        return open_paren \
            + regex_opt_inner(sorted(s[:-slen] for s in strings), '(?:') \
            + escape(suffix[::-1]) + close_paren
    # recurse on common 1-string prefixes
    # print '-> last resort'
    return open_paren + \
        '|'.join(regex_opt_inner(list(group[1]), '')
                 for group in groupby(strings, lambda s: s[0] == first[0])) \
        + close_paren 
Example 32
Project: yatta_reader   Author: sound88   File: regexopt.py    (license) View Source Project 4 votes vote down vote up
def regex_opt_inner(strings, open_paren):
    """Return a regex that matches any string in the sorted list of strings."""
    close_paren = open_paren and ')' or ''
    # print strings, repr(open_paren)
    if not strings:
        # print '-> nothing left'
        return ''
    first = strings[0]
    if len(strings) == 1:
        # print '-> only 1 string'
        return open_paren + escape(first) + close_paren
    if not first:
        # print '-> first string empty'
        return open_paren + regex_opt_inner(strings[1:], '(?:') \
            + '?' + close_paren
    if len(first) == 1:
        # multiple one-char strings? make a charset
        oneletter = []
        rest = []
        for s in strings:
            if len(s) == 1:
                oneletter.append(s)
            else:
                rest.append(s)
        if len(oneletter) > 1:  # do we have more than one oneletter string?
            if rest:
                # print '-> 1-character + rest'
                return open_paren + regex_opt_inner(rest, '') + '|' \
                    + make_charset(oneletter) + close_paren
            # print '-> only 1-character'
            return open_paren + make_charset(oneletter) + close_paren
    prefix = commonprefix(strings)
    if prefix:
        plen = len(prefix)
        # we have a prefix for all strings
        # print '-> prefix:', prefix
        return open_paren + escape(prefix) \
            + regex_opt_inner([s[plen:] for s in strings], '(?:') \
            + close_paren
    # is there a suffix?
    strings_rev = [s[::-1] for s in strings]
    suffix = commonprefix(strings_rev)
    if suffix:
        slen = len(suffix)
        # print '-> suffix:', suffix[::-1]
        return open_paren \
            + regex_opt_inner(sorted(s[:-slen] for s in strings), '(?:') \
            + escape(suffix[::-1]) + close_paren
    # recurse on common 1-string prefixes
    # print '-> last resort'
    return open_paren + \
        '|'.join(regex_opt_inner(list(group[1]), '')
                 for group in groupby(strings, lambda s: s[0] == first[0])) \
        + close_paren