Python sqlparse.tokens.Keyword() Examples

The following are code examples for showing how to use sqlparse.tokens.Keyword(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: xuemc   Author: skycucumber   File: grouping.py    GNU General Public License v2.0 6 votes vote down vote up
def group_where(tlist):
    [group_where(sgroup) for sgroup in tlist.get_sublists()
     if not isinstance(sgroup, sql.Where)]
    idx = 0
    token = tlist.token_next_match(idx, T.Keyword, 'WHERE')
    stopwords = ('ORDER', 'GROUP', 'LIMIT', 'UNION', 'EXCEPT', 'HAVING')
    while token:
        tidx = tlist.token_index(token)
        end = tlist.token_next_match(tidx + 1, T.Keyword, stopwords)
        if end is None:
            end = tlist._groupable_tokens[-1]
        else:
            end = tlist.tokens[tlist.token_index(end) - 1]
        group = tlist.group_tokens(sql.Where,
                                   tlist.tokens_between(token, end),
                                   ignore_ws=True)
        idx = tlist.token_index(group)
        token = tlist.token_next_match(idx, T.Keyword, 'WHERE') 
Example 2
Project: xuemc   Author: skycucumber   File: sql.py    GNU General Public License v2.0 6 votes vote down vote up
def get_type(self):
        """Returns the type of a statement.

        The returned value is a string holding an upper-cased reprint of
        the first DML or DDL keyword. If the first token in this group
        isn't a DML or DDL keyword "UNKNOWN" is returned.
        """
        first_token = self.token_first()
        if first_token is None:
            # An "empty" statement that either has not tokens at all
            # or only whitespace tokens.
            return 'UNKNOWN'

        elif first_token.ttype in (T.Keyword.DML, T.Keyword.DDL):
            return first_token.normalized

        return 'UNKNOWN' 
Example 3
Project: index-digest   Author: macbre   File: linter_0093_having_clause.py    MIT License 6 votes vote down vote up
def query_has_having_clause(query):
    """
    Checks if provided query uses HAVING clause
    :type query str
    :rtype bool
    """
    if not is_select_query(query):
        return False

    query = preprocess_query(query)
    tokens = get_query_tokens(query)

    for token in tokens:
        if token.ttype is Keyword and str(token).upper() == 'HAVING':
            return True

    return False 
Example 4
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: sql.py    BSD 2-Clause "Simplified" License 6 votes vote down vote up
def get_type(self):
        """Returns the type of a statement.

        The returned value is a string holding an upper-cased reprint of
        the first DML or DDL keyword. If the first token in this group
        isn't a DML or DDL keyword "UNKNOWN" is returned.

        Whitespaces and comments at the beginning of the statement
        are ignored.
        """
        first_token = self.token_first(ignore_comments=True)
        if first_token is None:
            # An "empty" statement that either has not tokens at all
            # or only whitespace tokens.
            return 'UNKNOWN'

        elif first_token.ttype in (T.Keyword.DML, T.Keyword.DDL):
            return first_token.normalized

        return 'UNKNOWN' 
Example 5
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: tokenutils.py    BSD 2-Clause "Simplified" License 6 votes vote down vote up
def is_dmlddl_parenthesis(token):
    """
        DMLかDDLの括弧判定
    """
    if not is_parenthesis(token):
        return False


    open_punc = token.token_next_match(0, T.Punctuation, '(')
    first = token_next_enable(token, open_punc)
    if first and first.ttype in (T.Keyword.DML, T.Keyword.DDL):
        return True

    if is_with(first):
        return True

    if is_parenthesis(first):
        return is_dmlddl_parenthesis(first)

    return False 
Example 6
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 6 votes vote down vote up
def _get_alias(self, token):
        tkw = token.token_next_match(0, T.Keyword, 'AS')
        if tkw is not None:
            return tu.token_next_enable(token, tkw)

        left = tu.token_next_enable(token)
        if not left:
            return None

        def is_space(tkn):
            return tkn.is_whitespace() and tkn.value

        spl = token.token_matching(token.token_index(left), [is_space])
        if spl:
            return tu.token_next_enable(token, spl)

        if tu.is_parenthesis(left):
            tkn = tu.token_next_enable(token, left)
            if tkn and (tu.is_identifier(tkn) or (tkn.ttype in T.Name)):
                # (・・・)ALIAS の場合
                space = sql.Token(T.Whitespace, "\t") # スペースを付与
                token.insert_after(left, space)
                return tkn

        return None 
Example 7
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 6 votes vote down vote up
def __custom_process_insert_values_lr(self, tlist):
        #INSERT の場合VALUES前後に空白1つをセット
        values_token = tlist.token_next_match(0, T.Keyword, "VALUES")
        if values_token:
            prv = tlist.token_prev(values_token, skip_ws=False)
            if prv and prv.is_whitespace():
                prv.value = " "
                prv = tlist.token_prev(prv, skip_ws=False)
                while prv and prv.is_whitespace():
                    prv.value = ""
                    prv = tlist.token_prev(prv, skip_ws=False)
            else:
                tlist.insert_before(values_token, sql.Token(T.Whitespace, " "))

            nxt = tlist.token_next(values_token, skip_ws=False)
            if nxt and nxt.is_whitespace():
                nxt.value = " "
                nxt = tlist.token_next(nxt, skip_ws=False)
                while nxt and nxt.is_whitespace():
                    nxt.value = ""
                    nxt = tlist.token_next(nxt, skip_ws=False)
            else:
                tlist.insert_after(values_token, sql.Token(T.Whitespace, " ")) 
Example 8
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 6 votes vote down vote up
def group_when(tlist):
    def proc(tlist):
        [proc(sgroup) for sgroup in tlist.get_sublists()
         if not isinstance(sgroup, When)]

        if not tu.is_case(tlist):
            return
        idx = 0
        token = tlist.token_next_match(idx, T.Keyword, 'WHEN')
        stopwords = ('THEN', 'END')
        while token:
            tidx = tlist.token_index(token)
            end = tlist.token_next_match(tidx + 1, T.Keyword, stopwords)
            if end is None:
                end = tlist._groupable_tokens[-1]
            else:
                end = tlist.tokens[tlist.token_index(end) - 1]
            tgroup = tlist.group_tokens(When,
                                       tlist.tokens_between(token, end),
                                       ignore_ws=True)
            idx = tlist.token_index(tgroup)
            token = tlist.token_next_match(idx, T.Keyword, 'WHEN')

    proc = SqlFormatterException.to_wrap_try_except(proc, 0)
    proc(tlist) 
Example 9
Project: DbDeo   Author: tushartushar   File: SQLParse.py    MIT License 6 votes vote down vote up
def knowStmtType(self):
        create_seen = False
        for token in self.parsed:
            if token.ttype is DML and token.value.upper() == 'SELECT':
                return SQLStmtType.SELECT
            if token.ttype is DDL and token.value.upper() == 'CREATE':
                create_seen = True
                continue
            if token.ttype is Keyword and token.value.upper() == 'TABLE' and create_seen:
                return SQLStmtType.CREATE
            if token.ttype is Keyword and token.value.upper() == 'INDEX' and create_seen:
                return SQLStmtType.CREATE_INDEX
            if token.ttype is DML and token.value.upper() == 'INSERT':
                return SQLStmtType.INSERT
            if token.ttype is DML and token.value.upper() == 'UPDATE':
                return SQLStmtType.UPDATE
        return SQLStmtType.DEFAULT 
Example 10
Project: DbDeo   Author: tushartushar   File: SQLParse.py    MIT License 6 votes vote down vote up
def extractTableIdentifiers(self, token_stream):
        for item in token_stream:
            if isinstance(item, IdentifierList):
                for identifier in item.get_identifiers():
                    try:
                        yield identifier.get_name()
                    except:
                        pass
            elif isinstance(item, Identifier):
                try:
                    yield item.get_name()
                except:
                    pass
            # It's a bug to check for Keyword here, but in the example
            # above some tables names are identified as keywords...
            elif item.ttype is Keyword:
                yield item.value 
Example 11
Project: ravel   Author: ravel-net   File: app.py    Apache License 2.0 5 votes vote down vote up
def discoverComponents(sql):
    """Search for installable/removable components within a string containing
       one or more SQL statemnets"""
    components = []
    parsed = sqlparse.parse(sql)
    for statement in parsed:
        for token in statement.tokens:
            name = None
            typ = None

            # remove newlines, extra spaces for regex
            stmt = str(statement).replace("\n", " ")
            stmt = " ".join(stmt.split())

            for comp in sqlComponents:
                if token.match(Keyword, comp.typ):
                    name = comp.match(stmt)
                    typ = comp.typ

            if name is not None:
                component = AppComponent(name, typ)
                if component not in components:
                    components.append(component)

    # sort alphabetically, should fix drop issues when 'rule on table'
    # is dropped before 'table'
    return sorted(components, key=lambda x: x.typ) 
Example 12
Project: xuemc   Author: skycucumber   File: functions.py    GNU General Public License v2.0 5 votes vote down vote up
def __call__(self, stream):
        for token_type, value in stream:
            if token_type not in Whitespace:
                return token_type in Keyword and value == self.type 
Example 13
Project: xuemc   Author: skycucumber   File: filters.py    GNU General Public License v2.0 5 votes vote down vote up
def _split_kwds(self, tlist):
        split_words = ('FROM', 'STRAIGHT_JOIN$', 'JOIN$', 'AND', 'OR',
                       'GROUP', 'ORDER', 'UNION', 'VALUES',
                       'SET', 'BETWEEN', 'EXCEPT', 'HAVING')

        def _next_token(i):
            t = tlist.token_next_match(i, T.Keyword, split_words,
                                       regex=True)
            if t and t.value.upper() == 'BETWEEN':
                t = _next_token(tlist.token_index(t) + 1)
                if t and t.value.upper() == 'AND':
                    t = _next_token(tlist.token_index(t) + 1)
            return t

        idx = 0
        token = _next_token(idx)
        added = set()
        while token:
            prev = tlist.token_prev(tlist.token_index(token), False)
            offset = 1
            if prev and prev.is_whitespace() and prev not in added:
                tlist.tokens.pop(tlist.token_index(prev))
                offset += 1
            uprev = unicode(prev)
            if (prev and (uprev.endswith('\n') or uprev.endswith('\r'))):
                nl = tlist.token_next(token)
            else:
                nl = self.nl()
                added.add(nl)
                tlist.insert_before(token, nl)
                offset += 1
            token = _next_token(tlist.token_index(nl) + offset) 
Example 14
Project: xuemc   Author: skycucumber   File: filters.py    GNU General Public License v2.0 5 votes vote down vote up
def _split_statements(self, tlist):
        idx = 0
        token = tlist.token_next_by_type(idx, (T.Keyword.DDL, T.Keyword.DML))
        while token:
            prev = tlist.token_prev(tlist.token_index(token), False)
            if prev and prev.is_whitespace():
                tlist.tokens.pop(tlist.token_index(prev))
            # only break if it's not the first token
            if prev:
                nl = self.nl()
                tlist.insert_before(token, nl)
            token = tlist.token_next_by_type(tlist.token_index(token) + 1,
                                             (T.Keyword.DDL, T.Keyword.DML)) 
Example 15
Project: xuemc   Author: skycucumber   File: filters.py    GNU General Public License v2.0 5 votes vote down vote up
def _process_where(self, tlist):
        token = tlist.token_next_match(0, T.Keyword, 'WHERE')
        try:
            tlist.insert_before(token, self.nl())
        except ValueError:  # issue121, errors in statement
            pass
        self.indent += 1
        self._process_default(tlist)
        self.indent -= 1 
Example 16
Project: xuemc   Author: skycucumber   File: filters.py    GNU General Public License v2.0 5 votes vote down vote up
def _process_having(self, tlist):
        token = tlist.token_next_match(0, T.Keyword, 'HAVING')
        try:
            tlist.insert_before(token, self.nl())
        except ValueError:  # issue121, errors in statement
            pass
        self.indent += 1
        self._process_default(tlist)
        self.indent -= 1 
Example 17
Project: xuemc   Author: skycucumber   File: filters.py    GNU General Public License v2.0 5 votes vote down vote up
def _process_case(self, tlist):
        is_first = True
        num_offset = None
        case = tlist.tokens[0]
        outer_offset = self._get_offset(case) - len(case.value)
        self.offset += outer_offset
        for cond, value in tlist.get_cases():
            if is_first:
                tcond = list(cond[0].flatten())[0]
                is_first = False
                num_offset = self._get_offset(tcond) - len(tcond.value)
                self.offset += num_offset
                continue
            if cond is None:
                token = value[0]
            else:
                token = cond[0]
            tlist.insert_before(token, self.nl())
        # Line breaks on group level are done. Now let's add an offset of
        # 5 (=length of "when", "then", "else") and process subgroups.
        self.offset += 5
        self._process_default(tlist)
        self.offset -= 5
        if num_offset is not None:
            self.offset -= num_offset
        end = tlist.token_next_match(0, T.Keyword, 'END')
        tlist.insert_before(end, self.nl())
        self.offset -= outer_offset 
Example 18
Project: xuemc   Author: skycucumber   File: filters.py    GNU General Public License v2.0 5 votes vote down vote up
def process(self, stack, stream):
        index = 7
        stream = list(stream)
        stream.reverse()

        # Run over all tokens in the stream from the end
        for token_type, value in stream:
            index -= 1

#            if index and token_type in Keyword:
            if index and token_type in Keyword and value == 'LIMIT':
                return stream[4 - index][1]

        return -1 
Example 19
Project: xuemc   Author: skycucumber   File: grouping.py    GNU General Public License v2.0 5 votes vote down vote up
def group_if(tlist):
    _group_matching(tlist, T.Keyword, 'IF', T.Keyword, 'END IF', sql.If, True) 
Example 20
Project: xuemc   Author: skycucumber   File: grouping.py    GNU General Public License v2.0 5 votes vote down vote up
def group_foreach(tlist):
    _group_matching(tlist, T.Keyword, 'FOREACH', T.Keyword, 'END LOOP',
                    sql.For, True) 
Example 21
Project: xuemc   Author: skycucumber   File: grouping.py    GNU General Public License v2.0 5 votes vote down vote up
def group_begin(tlist):
    _group_matching(tlist, T.Keyword, 'BEGIN', T.Keyword, 'END',
                    sql.Begin, True) 
Example 22
Project: xuemc   Author: skycucumber   File: grouping.py    GNU General Public License v2.0 5 votes vote down vote up
def group_as(tlist):

    def _right_valid(token):
        # Currently limited to DML/DDL. Maybe additional more non SQL reserved
        # keywords should appear here (see issue8).
        return not token.ttype in (T.DML, T.DDL)

    def _left_valid(token):
        if token.ttype is T.Keyword and token.value in ('NULL',):
            return True
        return token.ttype is not T.Keyword

    _group_left_right(tlist, T.Keyword, 'AS', sql.Identifier,
                      check_right=_right_valid,
                      check_left=_left_valid) 
Example 23
Project: xuemc   Author: skycucumber   File: grouping.py    GNU General Public License v2.0 5 votes vote down vote up
def group_comparison(tlist):

    def _parts_valid(token):
        return (token.ttype in (T.String.Symbol, T.String.Single,
                                T.Name, T.Number, T.Number.Float,
                                T.Number.Integer, T.Literal,
                                T.Literal.Number.Integer, T.Name.Placeholder)
                or isinstance(token, (sql.Identifier, sql.Parenthesis))
                or (token.ttype is T.Keyword
                    and token.value.upper() in ['NULL', ]))
    _group_left_right(tlist, T.Operator.Comparison, None, sql.Comparison,
                      check_left=_parts_valid, check_right=_parts_valid) 
Example 24
Project: xuemc   Author: skycucumber   File: grouping.py    GNU General Public License v2.0 5 votes vote down vote up
def group_case(tlist):
    _group_matching(tlist, T.Keyword, 'CASE', T.Keyword, 'END', sql.Case,
                    include_semicolon=True, recurse=True) 
Example 25
Project: xuemc   Author: skycucumber   File: grouping.py    GNU General Public License v2.0 5 votes vote down vote up
def group_order(tlist):
    idx = 0
    token = tlist.token_next_by_type(idx, T.Keyword.Order)
    while token:
        prev = tlist.token_prev(token)
        if isinstance(prev, sql.Identifier):
            ido = tlist.group_tokens(sql.Identifier,
                                     tlist.tokens_between(prev, token))
            idx = tlist.token_index(ido) + 1
        else:
            idx = tlist.token_index(token) + 1
        token = tlist.token_next_by_type(idx, T.Keyword.Order) 
Example 26
Project: xuemc   Author: skycucumber   File: sql.py    GNU General Public License v2.0 5 votes vote down vote up
def __init__(self, ttype, value):
        self.value = value
        if ttype in T.Keyword:
            self.normalized = value.upper()
        else:
            self.normalized = value
        self.ttype = ttype
        self.is_keyword = ttype in T.Keyword
        self.parent = None 
Example 27
Project: xuemc   Author: skycucumber   File: sql.py    GNU General Public License v2.0 5 votes vote down vote up
def get_alias(self):
        """Returns the alias for this identifier or ``None``."""

        # "name AS alias"
        kw = self.token_next_match(0, T.Keyword, 'AS')
        if kw is not None:
            return self._get_first_name(kw, keywords=True)

        # "name alias" or "complicated column expression alias"
        if len(self.tokens) > 2:
            return self._get_first_name(reverse=True)

        return None 
Example 28
Project: xuemc   Author: skycucumber   File: sql.py    GNU General Public License v2.0 5 votes vote down vote up
def get_ordering(self):
        """Returns the ordering or ``None`` as uppercase string."""
        ordering = self.token_next_by_type(0, T.Keyword.Order)
        if ordering is None:
            return None
        return ordering.value.upper() 
Example 29
Project: waf-a-mole   Author: AvalZ   File: tokenizer.py    MIT License 5 votes vote down vote up
def __init__(self):
        self._allowed_tokens = [
            tks.Other,
            tks.Keyword,
            tks.Name,
            tks.String,
            tks.Number,
            tks.Punctuation,
            tks.Operator,
            tks.Comparison,
            tks.Wildcard,
            tks.Comment.Single,
            tks.Comment.Multiline,
            tks.Operator.Logical,
        ] 
Example 30
Project: waf-a-mole   Author: AvalZ   File: tokenizer.py    MIT License 5 votes vote down vote up
def produce_feat_vector(self, sql_query: str, normalize=False):
        """It returns the feature vector as histogram of tokens, produced from the input query.
        
        Arguments:
            sql_query (str) : An input SQL query
        
        Keyword Arguments:
            normalize (bool) : True for producing a normalized hitogram. (default: (False))
        
        Raises:
            TypeError: params has wrong types
        
        Returns:
            numpy ndarray : histogram of tokens
        """
        type_check(sql_query, str, "sql_query")
        if normalize is not None:
            type_check(normalize, bool, "normalize")

        parsed = list(sqlparse.parse(sql_query)[0].flatten())
        allowed = self._allowed_tokens
        tokens = self._produce_tokens(parsed)
        dict_token = OrderedDict(zip(allowed, [0 for _ in range(len(allowed))]))
        for t in tokens:
            if t in dict_token:
                dict_token[t] += 1
            else:
                parent = t
                while parent is not None and parent not in dict_token:
                    parent = parent.parent
                if parent is None:
                    continue
                dict_token[parent] += 1
        values = dict_token.values()
        feature_vector = np.array([i for i in values])
        if normalize:
            norm = np.linalg.norm(feature_vector)
            feature_vector = feature_vector / norm
        return feature_vector 
Example 31
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: functions.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def __call__(self, stream):
        for token_type, value in stream:
            if token_type not in Whitespace:
                return token_type in Keyword and value == self.type 
Example 32
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def _split_kwds(self, tlist):
        split_words = ('FROM', 'STRAIGHT_JOIN$', 'JOIN$', 'AND', 'OR',
                       'GROUP', 'ORDER', 'UNION', 'VALUES',
                       'SET', 'BETWEEN', 'EXCEPT', 'HAVING')

        def _next_token(i):
            t = tlist.token_next_match(i, T.Keyword, split_words,
                                       regex=True)
            if t and t.value.upper() == 'BETWEEN':
                t = _next_token(tlist.token_index(t) + 1)
                if t and t.value.upper() == 'AND':
                    t = _next_token(tlist.token_index(t) + 1)
            return t

        idx = 0
        token = _next_token(idx)
        added = set()
        while token:
            prev = tlist.token_prev(tlist.token_index(token), False)
            offset = 1
            if prev and prev.is_whitespace() and prev not in added:
                tlist.tokens.pop(tlist.token_index(prev))
                offset += 1
            uprev = str(prev)
            if (prev and (uprev.endswith('\n') or uprev.endswith('\r'))):
                nl = tlist.token_next(token)
            else:
                nl = self.nl()
                added.add(nl)
                tlist.insert_before(token, nl)
                offset += 1
            token = _next_token(tlist.token_index(nl) + offset) 
Example 33
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def _split_statements(self, tlist):
        idx = 0
        token = tlist.token_next_by_type(idx, (T.Keyword.DDL, T.Keyword.DML))
        while token:
            prev = tlist.token_prev(tlist.token_index(token), False)
            if prev and prev.is_whitespace():
                tlist.tokens.pop(tlist.token_index(prev))
            # only break if it's not the first token
            if prev:
                nl = self.nl()
                tlist.insert_before(token, nl)
            token = tlist.token_next_by_type(tlist.token_index(token) + 1,
                                             (T.Keyword.DDL, T.Keyword.DML)) 
Example 34
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def _process_where(self, tlist):
        token = tlist.token_next_match(0, T.Keyword, 'WHERE')
        try:
            tlist.insert_before(token, self.nl())
        except ValueError:  # issue121, errors in statement
            pass
        self.indent += 1
        self._process_default(tlist)
        self.indent -= 1 
Example 35
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def _process_having(self, tlist):
        token = tlist.token_next_match(0, T.Keyword, 'HAVING')
        try:
            tlist.insert_before(token, self.nl())
        except ValueError:  # issue121, errors in statement
            pass
        self.indent += 1
        self._process_default(tlist)
        self.indent -= 1 
Example 36
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def _process_case(self, tlist):
        is_first = True
        num_offset = None
        case = tlist.tokens[0]
        outer_offset = self._get_offset(case) - len(case.value)
        self.offset += outer_offset
        for cond, value in tlist.get_cases():
            if is_first:
                tcond = list(cond[0].flatten())[0]
                is_first = False
                num_offset = self._get_offset(tcond) - len(tcond.value)
                self.offset += num_offset
                continue
            if cond is None:
                token = value[0]
            else:
                token = cond[0]
            tlist.insert_before(token, self.nl())
        # Line breaks on group level are done. Now let's add an offset of
        # 5 (=length of "when", "then", "else") and process subgroups.
        self.offset += 5
        self._process_default(tlist)
        self.offset -= 5
        if num_offset is not None:
            self.offset -= num_offset
        end = tlist.token_next_match(0, T.Keyword, 'END')
        tlist.insert_before(end, self.nl())
        self.offset -= outer_offset 
Example 37
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def process(self, stack, stream):
        index = 7
        stream = list(stream)
        stream.reverse()

        # Run over all tokens in the stream from the end
        for token_type, value in stream:
            index -= 1

#            if index and token_type in Keyword:
            if index and token_type in Keyword and value == 'LIMIT':
                return stream[4 - index][1]

        return -1 
Example 38
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def group_if(tlist):
    _group_matching(tlist, T.Keyword, 'IF', T.Keyword, 'END IF', sql.If, True) 
Example 39
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def group_foreach(tlist):
    _group_matching(tlist, T.Keyword, 'FOREACH', T.Keyword, 'END LOOP',
                    sql.For, True) 
Example 40
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def group_begin(tlist):
    _group_matching(tlist, T.Keyword, 'BEGIN', T.Keyword, 'END',
                    sql.Begin, True) 
Example 41
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def group_as(tlist):

    def _right_valid(token):
        # Currently limited to DML/DDL. Maybe additional more non SQL reserved
        # keywords should appear here (see issue8).
        return not token.ttype in (T.DML, T.DDL)

    def _left_valid(token):
        if token.ttype is T.Keyword and token.value in ('NULL',):
            return True
        return token.ttype is not T.Keyword

    _group_left_right(tlist, T.Keyword, 'AS', sql.Identifier,
                      check_right=_right_valid,
                      check_left=_left_valid) 
Example 42
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def group_comparison(tlist):

    def _parts_valid(token):
        return (token.ttype in (T.String.Symbol, T.String.Single,
                                T.Name, T.Number, T.Number.Float,
                                T.Number.Integer, T.Literal,
                                T.Literal.Number.Integer, T.Name.Placeholder)
                or isinstance(token, (sql.Identifier, sql.Parenthesis))
                or (token.ttype is T.Keyword
                    and token.value.upper() in ['NULL', ]))
    _group_left_right(tlist, T.Operator.Comparison, None, sql.Comparison,
                      check_left=_parts_valid, check_right=_parts_valid) 
Example 43
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def group_case(tlist):
    _group_matching(tlist, T.Keyword, 'CASE', T.Keyword, 'END', sql.Case,
                    include_semicolon=True, recurse=True) 
Example 44
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def group_where(tlist):
    def end_match(token):
        stopwords = ('ORDER', 'GROUP', 'LIMIT', 'UNION', 'EXCEPT', 'HAVING',
                     'WHEN', # for Oracle10g merge
                     'CONNECT', # for Oracle connect by
                     )
        if token.match(T.Keyword, stopwords):
            return True
        if token.match(T.DML, ('DELETE')): # for Oracle10g merge
            return True
        if token.match(T.DML, ('START')): # for Oracle connect by
            return True

        return False

    [group_where(sgroup) for sgroup in tlist.get_sublists()
     if not isinstance(sgroup, sql.Where)]
    idx = 0
    token = tlist.token_next_match(idx, T.Keyword, 'WHERE')
    while token:
        tidx = tlist.token_index(token)
        end = tlist.token_matching(tidx + 1, (end_match, ))
        if end is None:
            end = tlist._groupable_tokens[-1]
        else:
            end = tlist.tokens[tlist.token_index(end) - 1]
        group = tlist.group_tokens(sql.Where,
                                   tlist.tokens_between(token, end),
                                   ignore_ws=True)
        idx = tlist.token_index(group)
        token = tlist.token_next_match(idx, T.Keyword, 'WHERE') 
Example 45
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def group_order(tlist):
    idx = 0
    token = tlist.token_next_by_type(idx, T.Keyword.Order)
    while token:
        prev = tlist.token_prev(token)
        if isinstance(prev, sql.Identifier):
            ido = tlist.group_tokens(sql.Identifier,
                                     tlist.tokens_between(prev, token))
            idx = tlist.token_index(ido) + 1
        else:
            idx = tlist.token_index(token) + 1
        token = tlist.token_next_by_type(idx, T.Keyword.Order) 
Example 46
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: sql.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def __init__(self, ttype, value):
        self.value = value
        if ttype in T.Keyword:
            self.normalized = value.upper()
        else:
            self.normalized = value
        self.ttype = ttype
        self.is_keyword = ttype in T.Keyword
        self.parent = None 
Example 47
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: sql.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def get_alias(self):
        """Returns the alias for this identifier or ``None``."""

        # "name AS alias"
        kw = self.token_next_match(0, T.Keyword, 'AS')
        if kw is not None:
            return self._get_first_name(kw, keywords=True)

        # "name alias" or "complicated column expression alias"
        if len(self.tokens) > 2 \
           and self.token_next_by_type(0, T.Whitespace) is not None:
            return self._get_first_name(reverse=True)

        return None 
Example 48
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: sql.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def get_ordering(self):
        """Returns the ordering or ``None`` as uppercase string."""
        ordering = self.token_next_by_type(0, T.Keyword.Order)
        if ordering is None:
            return None
        return ordering.value.upper() 
Example 49
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: tokenutils.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def is_null_keyword(token):
    """
        「NULL」文字列判定
    """
    return token.match(T.Keyword, "NULL") 
Example 50
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: tokenutils.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def is_distinct_keyword(token):
    """
        「DISTINCT」判定
    """
    return token.match(T.Keyword, "DISTINCT") 
Example 51
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: tokenutils.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def is_from_keyword(token):
    """
        「FROM」判定
    """
    return token.match(T.Keyword, "FROM") 
Example 52
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: tokenutils.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def is_by_keyword(token):
    """
        「BY」判定
    """
    return token.match(T.Keyword, "BY") 
Example 53
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: tokenutils.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def is_into_keyword(token):
    """
        INTO判定
    """
    return token.match(T.Keyword, "INTO") 
Example 54
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: tokenutils.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def is_values_keyword(token):
    """
        VALUES判定
    """
    return token.match(T.Keyword, "VALUES") 
Example 55
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: tokenutils.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def is_between_keyword(token):
    """
        「BETWEEN」判定
    """
    return token.match(T.Keyword, "BETWEEN") 
Example 56
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: tokenutils.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def is_and_keyword(token):
    """
        AND演算子判定
    """
    return token.match(T.Keyword, "AND") 
Example 57
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: tokenutils.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def is_using_keyword(token):
    """
        USING判定
    """
    return token.match(T.Keyword, "USING") 
Example 58
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: tokenutils.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def is_logical_operator_keyword(token):
    """
        AND・OR演算子判定
    """
    return token.match(T.Keyword, ("AND", "OR")) 
Example 59
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def _get_center_token(self, token):
        return token.token_next_match(0, T.Keyword, "AS") 
Example 60
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def _process_when(self, tlist):
        token = tlist.token_next_match(0, T.Keyword, 'WHEN')
        try:
            tlist.insert_before(token, self.nl())
            tlist.insert_after(token, self.nl_with_indent(1))
        except ValueError:  # issue121, errors in statement
            pass

        self._process_default(tlist)
        self._adjust_comparisons_indent(self._get_comparisons(tlist)) 
Example 61
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def _process_where(self, tlist):
        token = tlist.token_next_match(0, T.Keyword, 'WHERE')
        try:
            tlist.insert_before(token, self.nl())
            tlist.insert_after(token, self.nl_with_indent(1))
        except ValueError:  # issue121, errors in statement
            pass

        self._process_default(tlist)
        self._adjust_comparisons_indent(self._get_comparisons(tlist)) 
Example 62
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def _process_case(self, tlist):
        def is_prev_comma(token):
            for prev in self._flatten_tokens_prev(token):
                if not tu.is_enable(prev):
                    continue
                return tu.is_comma(prev)
            return False

        commma_next = is_prev_comma(tlist)
        cases = tu.get_cases(tlist)
        if not commma_next:
            case = tlist.tokens[0]
            tlist.insert_before(case, self.nl_with_indent(1))

        self.offset += 2
        is_first = True
        for cond, value in cases:
            if is_first:
                is_first = False
                if not value:
                    if cond:
                        tlist.insert_before(cond[0], self.nl_with_indent(-1))
                    continue

            if cond:
                tlist.insert_before(cond[0], self.nl())
                tlist.insert_after(cond[0], self.nl_with_indent(1))

            if value:
                tlist.insert_before(value[0], self.nl())
                tlist.insert_after(value[0], self.nl_with_indent(1))

        self._process_default(tlist)

        self.offset -= 2

        end = tlist.token_next_match(0, T.Keyword, 'END')
        tlist.insert_before(end, self.nl_with_indent(1))

        if not commma_next:
            tlist.insert_after(end, self.nl()) 
Example 63
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def _process_on(self, tlist):
        """
            ON句
        """
        token = tlist.token_next_match(0, T.Keyword, 'ON')
        try:
            tlist.insert_before(token, self.nl())
            tlist.insert_after(token, self.nl_with_indent(1))
        except ValueError:  # issue121, errors in statement
            pass

        self._process_default(tlist)
        self._adjust_comparisons_indent(self._get_comparisons(tlist)) 
Example 64
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: sql.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def token_with(self):
        return self.token_next_match(0, T.Keyword, 'WITH') 
Example 65
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def _test_word(self, word, token):
        return tu.equals_ignore_case(token.value, word) and (token.ttype in T.Name or token.ttype in T.Keyword) 
Example 66
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def _is_join_keyword(self, token):
        """
            JOIN判定
        """
        return tu.endswith_ignore_case(token.value, "JOIN") \
                    and (token.ttype in T.Name or token.ttype in T.Keyword) 
Example 67
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def group_on(tlist):

    def end_match(token):
        stopwords = ('WHERE', 'ORDER', 'GROUP', 'LIMIT', 'UNION', 'EXCEPT', 'HAVING',
                     'WHEN', # for Oracle10g merge
                     'CONNECT', # for Oracle connect by
                     )
        if token.match(T.Keyword, stopwords):
            return True

        if tu.is_phrase(token):
            if token.match_phrase(('ORDER', 'BY')) or token.match_phrase(('GROUP', 'BY')):
                return True


        return tu.is_join(token) or tu.is_mergewhen(token)

    def proc(tlist):
        [proc(sgroup) for sgroup in tlist.get_sublists()
         if not isinstance(sgroup, On)]
        idx = 0
        token = tlist.token_next_match(idx, T.Keyword, 'ON')

        while token:
            tidx = tlist.token_index(token)
            end = tlist.token_matching(tidx + 1, (end_match,))
            if end is None:
                end = tlist._groupable_tokens[-1]
            else:
                end = tlist.tokens[tlist.token_index(end) - 1]
            tgroup = tlist.group_tokens(On,
                                       tlist.tokens_between(token, end),
                                       ignore_ws=True)
            idx = tlist.token_index(tgroup)
            token = tlist.token_next_match(idx, T.Keyword, 'ON')

    proc = SqlFormatterException.to_wrap_try_except(proc, 0)
    proc(tlist) 
Example 68
Project: DbDeo   Author: tushartushar   File: SQLParse.py    MIT License 5 votes vote down vote up
def extractFromPart(self, parsed):
        from_seen = False
        for item in parsed.tokens:
            if from_seen:
                if self.isSubSelect(item):
                    for x in self.extractFromPart(item):
                        yield x
                elif item.ttype is Keyword:
                    raise StopIteration
                else:
                    yield item
            elif item.ttype is Keyword and item.value.upper() == 'FROM':
                from_seen = True 
Example 69
Project: DbDeo   Author: tushartushar   File: SQLParse.py    MIT License 5 votes vote down vote up
def traverseForWhere(self):
        where_seen = False
        whereExpn = []
        for item in self.parsed.tokens:
            if item.is_group:
                for token in item.tokens:
                    if where_seen:
                        if token.ttype is Keyword and not (token.value.upper() == 'AND' or token.value.upper() == 'OR'):
                            return whereExpn
                        else:
                            whereExpn.append(token)
                    elif token.ttype is Keyword and token.value.upper() == 'WHERE':
                        where_seen = True
        return whereExpn 
Example 70
Project: xuemc   Author: skycucumber   File: filters.py    GNU General Public License v2.0 4 votes vote down vote up
def process(self, stack, stream):
        mode = 0
        oldValue = ""
        parenthesis = 0

        for token_type, value in stream:
            # Ignore comments
            if token_type in Comment:
                continue

            # We have not detected a SELECT statement
            if mode == 0:
                if token_type in Keyword and value == 'SELECT':
                    mode = 1

            # We have detected a SELECT statement
            elif mode == 1:
                if value == 'FROM':
                    if oldValue:
                        yield oldValue

                    mode = 3    # Columns have been checked

                elif value == 'AS':
                    oldValue = ""
                    mode = 2

                elif (token_type == Punctuation
                      and value == ',' and not parenthesis):
                    if oldValue:
                        yield oldValue
                    oldValue = ""

                elif token_type not in Whitespace:
                    if value == '(':
                        parenthesis += 1
                    elif value == ')':
                        parenthesis -= 1

                    oldValue += value

            # We are processing an AS keyword
            elif mode == 2:
                # We check also for Keywords because a bug in SQLParse
                if token_type == Name or token_type == Keyword:
                    yield value
                    mode = 1


# ---------------------------
# postprocess 
Example 71
Project: xuemc   Author: skycucumber   File: filter.py    GNU General Public License v2.0 4 votes vote down vote up
def _change_splitlevel(self, ttype, value):
        "Get the new split level (increase, decrease or remain equal)"
        # PostgreSQL
        if (ttype == T.Name.Builtin
            and value.startswith('$') and value.endswith('$')):
            if self._in_dbldollar:
                self._in_dbldollar = False
                return -1
            else:
                self._in_dbldollar = True
                return 1
        elif self._in_dbldollar:
            return 0

        # ANSI
        if ttype not in T.Keyword:
            return 0

        unified = value.upper()

        if unified == 'DECLARE' and self._is_create:
            self._in_declare = True
            return 1

        if unified == 'BEGIN':
            self._begin_depth += 1
            if self._in_declare or self._is_create:
                # FIXME(andi): This makes no sense.
                return 1
            return 0

        if unified == 'END':
            # Should this respect a preceeding BEGIN?
            # In CASE ... WHEN ... END this results in a split level -1.
            self._begin_depth = max(0, self._begin_depth - 1)
            return -1

        if ttype is T.Keyword.DDL and unified.startswith('CREATE'):
            self._is_create = True
            return 0

        if (unified in ('IF', 'FOR')
            and self._is_create and self._begin_depth > 0):
            return 1

        # Default
        return 0 
Example 72
Project: xuemc   Author: skycucumber   File: sql.py    GNU General Public License v2.0 4 votes vote down vote up
def match(self, ttype, values, regex=False):
        """Checks whether the token matches the given arguments.

        *ttype* is a token type. If this token doesn't match the given token
        type.
        *values* is a list of possible values for this token. The values
        are OR'ed together so if only one of the values matches ``True``
        is returned. Except for keyword tokens the comparison is
        case-sensitive. For convenience it's ok to pass in a single string.
        If *regex* is ``True`` (default is ``False``) the given values are
        treated as regular expressions.
        """
        type_matched = self.ttype is ttype
        if not type_matched or values is None:
            return type_matched

        if regex:
            if isinstance(values, basestring):
                values = set([values])

            if self.ttype is T.Keyword:
                values = set(re.compile(v, re.IGNORECASE) for v in values)
            else:
                values = set(re.compile(v) for v in values)

            for pattern in values:
                if pattern.search(self.value):
                    return True
            return False

        if isinstance(values, basestring):
            if self.is_keyword:
                return values.upper() == self.normalized
            return values == self.value

        if self.is_keyword:
            for v in values:
                if v.upper() == self.normalized:
                    return True
            return False

        return self.value in values 
Example 73
Project: xuemc   Author: skycucumber   File: sql.py    GNU General Public License v2.0 4 votes vote down vote up
def get_cases(self):
        """Returns a list of 2-tuples (condition, value).

        If an ELSE exists condition is None.
        """
        CONDITION = 1
        VALUE = 2

        ret = []
        mode = CONDITION

        for token in self.tokens:
            # Set mode from the current statement
            if token.match(T.Keyword, 'CASE'):
                continue

            elif token.match(T.Keyword, 'WHEN'):
                ret.append(([], []))
                mode = CONDITION

            elif token.match(T.Keyword, 'THEN'):
                mode = VALUE

            elif token.match(T.Keyword, 'ELSE'):
                ret.append((None, []))
                mode = VALUE

            elif token.match(T.Keyword, 'END'):
                mode = None

            # First condition without preceding WHEN
            if mode and not ret:
                ret.append(([], []))

            # Append token depending of the current mode
            if mode == CONDITION:
                ret[-1][0].append(token)

            elif mode == VALUE:
                ret[-1][1].append(token)

        # Return cases list
        return ret 
Example 74
Project: waf-a-mole   Author: AvalZ   File: tokenizer.py    MIT License 4 votes vote down vote up
def create_dataset_from_file(
        self, filepath: str, label: int, limit: int = None, unique_rows=True
    ):
        """Create dataset from fil containing sql queries.
        
        Arguments:
            filepath (str) : path of sql queries dataset
            label (int) : labels to assign to each sample
        
        Keyword Arguments:
            limit (int) : if None, it specifies how many queries to use (default: (None))
            unique_rows (bool) : True for removing all the duplicates (default: (True))
        
        Raises:
            TypeError: params has wrong types
            FileNotFoundError: filepath not pointing to regular file
            TypeError: limit is not None and not int
        
        Returns:
            (numpy ndarray, list) : X and y
        """
        type_check(filepath, str, "filepath")
        type_check(label, int, "label")
        type_check(unique_rows, bool, "unique_rows")
        if limit is not None:
            type_check(limit, int, "limit")

        file_exists(filepath)
        X = []
        with open(filepath, "r") as f:
            i = 0
            for line in f:
                if limit is not None and i > limit:
                    break
                line = line.strip()
                X.append(self.produce_feat_vector(line))
                i += 1
        if unique_rows:
            X = np.unique(X, axis=0)
        else:
            X = np.array(X)
        y = [label for _ in X]
        return X, y 
Example 75
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filters.py    BSD 2-Clause "Simplified" License 4 votes vote down vote up
def process(self, stack, stream):
        mode = 0
        oldValue = ""
        parenthesis = 0

        for token_type, value in stream:
            # Ignore comments
            if token_type in Comment:
                continue

            # We have not detected a SELECT statement
            if mode == 0:
                if token_type in Keyword and value == 'SELECT':
                    mode = 1

            # We have detected a SELECT statement
            elif mode == 1:
                if value == 'FROM':
                    if oldValue:
                        yield oldValue

                    mode = 3    # Columns have been checked

                elif value == 'AS':
                    oldValue = ""
                    mode = 2

                elif (token_type == Punctuation
                      and value == ',' and not parenthesis):
                    if oldValue:
                        yield oldValue
                    oldValue = ""

                elif token_type not in Whitespace:
                    if value == '(':
                        parenthesis += 1
                    elif value == ')':
                        parenthesis -= 1

                    oldValue += value

            # We are processing an AS keyword
            elif mode == 2:
                # We check also for Keywords because a bug in SQLParse
                if token_type == Name or token_type == Keyword:
                    yield value
                    mode = 1


# ---------------------------
# postprocess 
Example 76
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: filter.py    BSD 2-Clause "Simplified" License 4 votes vote down vote up
def _change_splitlevel(self, ttype, value):
        "Get the new split level (increase, decrease or remain equal)"
        # PostgreSQL
        if (ttype == T.Name.Builtin
            and value.startswith('$') and value.endswith('$')):
            if self._in_dbldollar:
                self._in_dbldollar = False
                return -1
            else:
                self._in_dbldollar = True
                return 1
        elif self._in_dbldollar:
            return 0

        # ANSI
        if ttype not in T.Keyword:
            return 0

        unified = value.upper()

        if unified == 'DECLARE' and self._is_create and self._begin_depth == 0:
            self._in_declare = True
            return 1

        if unified == 'BEGIN':
            self._begin_depth += 1
            if self._in_declare or self._is_create:
                # FIXME(andi): This makes no sense.
                return 1
            return 0

        if unified in ('END IF', 'END FOR'):
            return -1

        if unified == 'END':
            # Should this respect a preceeding BEGIN?
            # In CASE ... WHEN ... END this results in a split level -1.
            self._begin_depth = max(0, self._begin_depth - 1)
            return -1

        if ttype is T.Keyword.DDL and unified.startswith('CREATE'):
            self._is_create = True
            return 0

        if (unified in ('IF', 'FOR')
            and self._is_create and self._begin_depth > 0):
            return 1

        # Default
        return 0 
Example 77
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: sql.py    BSD 2-Clause "Simplified" License 4 votes vote down vote up
def match(self, ttype, values, regex=False):
        """Checks whether the token matches the given arguments.

        *ttype* is a token type. If this token doesn't match the given token
        type.
        *values* is a list of possible values for this token. The values
        are OR'ed together so if only one of the values matches ``True``
        is returned. Except for keyword tokens the comparison is
        case-sensitive. For convenience it's ok to pass in a single string.
        If *regex* is ``True`` (default is ``False``) the given values are
        treated as regular expressions.
        """
        type_matched = self.ttype is ttype
        if not type_matched or values is None:
            return type_matched

        if regex:
            if isinstance(values, str):
                values = set([values])

            if self.ttype is T.Keyword:
                values = set(re.compile(v, re.IGNORECASE) for v in values)
            else:
                values = set(re.compile(v) for v in values)

            for pattern in values:
                if pattern.search(self.value):
                    return True
            return False

        if isinstance(values, str):
            if self.is_keyword:
                return values.upper() == self.normalized
            return values == self.value

        if self.is_keyword:
            for v in values:
                if v.upper() == self.normalized:
                    return True
            return False

        return self.value in values 
Example 78
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: sql.py    BSD 2-Clause "Simplified" License 4 votes vote down vote up
def get_cases(self):
        """Returns a list of 2-tuples (condition, value).

        If an ELSE exists condition is None.
        """
        CONDITION = 1
        VALUE = 2

        ret = []
        mode = CONDITION

        for token in self.tokens:
            # Set mode from the current statement
            if token.match(T.Keyword, 'CASE'):
                continue

            elif token.match(T.Keyword, 'WHEN'):
                ret.append(([], []))
                mode = CONDITION

            elif token.match(T.Keyword, 'THEN'):
                mode = VALUE

            elif token.match(T.Keyword, 'ELSE'):
                ret.append((None, []))
                mode = VALUE

            elif token.match(T.Keyword, 'END'):
                mode = None

            # First condition without preceding WHEN
            if mode and not ret:
                ret.append(([], []))

            # Append token depending of the current mode
            if mode == CONDITION:
                ret[-1][0].append(token)

            elif mode == VALUE:
                ret[-1][1].append(token)

        # Return cases list
        return ret 
Example 79
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: tokenutils.py    BSD 2-Clause "Simplified" License 4 votes vote down vote up
def get_cases(case):
    """Returns a list of 2-tuples (condition, value).

    If an ELSE exists condition is None.
    """


    ret = []
    mode = CONDITION

    for token in case.tokens:
        # Set mode from the current statement
        if token.match(T.Keyword, 'CASE'):
            continue

        elif is_when(token):
            ret.append(([], []))
            mode = CONDITION

        elif token.match(T.Keyword, 'THEN'):
            mode = VALUE

        elif token.match(T.Keyword, 'ELSE'):
            ret.append((None, []))
            mode = VALUE

        elif token.match(T.Keyword, 'END'):
            mode = None

        # First condition without preceding WHEN
        if mode and not ret:
            ret.append(([], []))

        # Append token depending of the current mode
        if mode == CONDITION:
            ret[-1][0].append(token)

        elif mode == VALUE:
            ret[-1][1].append(token)

    # Return cases list
    return ret 
Example 80
Project: Sublime-uroboroSQL-formatter   Author: future-architect   File: grouping.py    BSD 2-Clause "Simplified" License 4 votes vote down vote up
def group_connectby_startwith(tlist):
    def start_match(token):
        if tu.is_phrase(token):
            return token.match_phrase(("CONNECT", "BY")) or token.match_phrase(("START", "WITH"))


    def end_match(token):
        stopwords = ('ORDER', 'GROUP', 'LIMIT', 'UNION', 'EXCEPT', 'HAVING',
                     'WHEN', # for Oracle10g merge
                     'CONNECT', # for Oracle connect by
                     )
        if token.match(T.Keyword, stopwords):
            return True
        if token.match(T.DML, ('DELETE')): # for Oracle10g merge
            return True
        if token.match(T.DML, ('START')): # for Oracle connect by
            return True

        if tu.is_phrase(token):
            if token.match_phrase(("CONNECT", "BY")) or token.match_phrase(("START", "WITH")):
                return True

        return False

    def proc(tlist):
        [proc(sgroup) for sgroup in tlist.get_sublists()
         if (not isinstance(sgroup, ConnectBy)) and (not isinstance(sgroup, StartWith))]
        idx = 0
        token = tlist.token_matching(idx, (start_match, ))
        while token:
            tidx = tlist.token_index(token)
            end = tlist.token_matching(tidx + 1, (end_match, ))
            if end is None:
                end = tlist._groupable_tokens[-1]
            else:
                end = tlist.tokens[tlist.token_index(end) - 1]

            group_class = None
            if token.match_phrase(("CONNECT", "BY")):
                group_class = ConnectBy
            elif token.match_phrase(("START", "WITH")):
                group_class = StartWith
            tgroup = tlist.group_tokens(group_class,
                                       tlist.tokens_between(token, end),
                                       ignore_ws=True)

            idx = tlist.token_index(tgroup)
            token = tlist.token_matching(idx, (start_match, ))

    proc = SqlFormatterException.to_wrap_try_except(proc, 0)
    proc(tlist)