Python nltk.compat.binary_type() Examples

The following are code examples for showing how to use nltk.compat.binary_type(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: razzy-spinner   Author: rafasashi   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_tokenize(tgrep_string):
    '''
    Tokenizes a TGrep search string into separate tokens.
    '''
    parser = _build_tgrep_parser(False)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string)) 
Example 2
Project: razzy-spinner   Author: rafasashi   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_compile(tgrep_string):
    '''
    Parses (and tokenizes, if necessary) a TGrep search string into a
    lambda function.
    '''
    parser = _build_tgrep_parser(True)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string, parseAll=True))[0] 
Example 3
Project: razzy-spinner   Author: rafasashi   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_positions(pattern, trees, search_leaves=True):
    """
    Return the tree positions in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree positions)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [position for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 4
Project: razzy-spinner   Author: rafasashi   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_nodes(pattern, trees, search_leaves=True):
    """
    Return the tree nodes in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree nodes)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [tree[position] for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 5
Project: OpenBottle   Author: xiaozhuchacha   File: tgrep.py    MIT License 5 votes vote down vote up
def tgrep_tokenize(tgrep_string):
    '''
    Tokenizes a TGrep search string into separate tokens.
    '''
    parser = _build_tgrep_parser(False)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string)) 
Example 6
Project: OpenBottle   Author: xiaozhuchacha   File: tgrep.py    MIT License 5 votes vote down vote up
def tgrep_compile(tgrep_string):
    '''
    Parses (and tokenizes, if necessary) a TGrep search string into a
    lambda function.
    '''
    parser = _build_tgrep_parser(True)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string, parseAll=True))[0] 
Example 7
Project: OpenBottle   Author: xiaozhuchacha   File: tgrep.py    MIT License 5 votes vote down vote up
def tgrep_positions(pattern, trees, search_leaves=True):
    """
    Return the tree positions in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree positions)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [position for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 8
Project: OpenBottle   Author: xiaozhuchacha   File: tgrep.py    MIT License 5 votes vote down vote up
def tgrep_nodes(pattern, trees, search_leaves=True):
    """
    Return the tree nodes in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree nodes)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [tree[position] for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 9
Project: OpenBottle   Author: xiaozhuchacha   File: tgrep.py    MIT License 5 votes vote down vote up
def tgrep_tokenize(tgrep_string):
    '''
    Tokenizes a TGrep search string into separate tokens.
    '''
    parser = _build_tgrep_parser(False)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string)) 
Example 10
Project: OpenBottle   Author: xiaozhuchacha   File: tgrep.py    MIT License 5 votes vote down vote up
def tgrep_positions(pattern, trees, search_leaves=True):
    """
    Return the tree positions in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree positions)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [position for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 11
Project: OpenBottle   Author: xiaozhuchacha   File: tgrep.py    MIT License 5 votes vote down vote up
def tgrep_nodes(pattern, trees, search_leaves=True):
    """
    Return the tree nodes in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree nodes)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [tree[position] for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 12
Project: FancyWord   Author: EastonLee   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_tokenize(tgrep_string):
    '''
    Tokenizes a TGrep search string into separate tokens.
    '''
    parser = _build_tgrep_parser(False)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string)) 
Example 13
Project: FancyWord   Author: EastonLee   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_compile(tgrep_string):
    '''
    Parses (and tokenizes, if necessary) a TGrep search string into a
    lambda function.
    '''
    parser = _build_tgrep_parser(True)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string, parseAll=True))[0] 
Example 14
Project: FancyWord   Author: EastonLee   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_positions(pattern, trees, search_leaves=True):
    """
    Return the tree positions in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree positions)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [position for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 15
Project: FancyWord   Author: EastonLee   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_nodes(pattern, trees, search_leaves=True):
    """
    Return the tree nodes in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree nodes)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [tree[position] for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield []


# run module doctests 
Example 16
Project: honours_project   Author: JFriel   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_tokenize(tgrep_string):
    '''
    Tokenizes a TGrep search string into separate tokens.
    '''
    parser = _build_tgrep_parser(False)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string)) 
Example 17
Project: honours_project   Author: JFriel   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_compile(tgrep_string):
    '''
    Parses (and tokenizes, if necessary) a TGrep search string into a
    lambda function.
    '''
    parser = _build_tgrep_parser(True)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string, parseAll=True))[0] 
Example 18
Project: honours_project   Author: JFriel   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_positions(pattern, trees, search_leaves=True):
    """
    Return the tree positions in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree positions)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [position for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 19
Project: honours_project   Author: JFriel   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_nodes(pattern, trees, search_leaves=True):
    """
    Return the tree nodes in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree nodes)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [tree[position] for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 20
Project: honours_project   Author: JFriel   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_tokenize(tgrep_string):
    '''
    Tokenizes a TGrep search string into separate tokens.
    '''
    parser = _build_tgrep_parser(False)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string)) 
Example 21
Project: honours_project   Author: JFriel   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_positions(pattern, trees, search_leaves=True):
    """
    Return the tree positions in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree positions)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [position for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 22
Project: honours_project   Author: JFriel   File: tgrep.py    GNU General Public License v3.0 5 votes vote down vote up
def tgrep_nodes(pattern, trees, search_leaves=True):
    """
    Return the tree nodes in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree nodes)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [tree[position] for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 23
Project: serverless-chatbots-workshop   Author: datteswararao   File: tgrep.py    Apache License 2.0 5 votes vote down vote up
def tgrep_tokenize(tgrep_string):
    '''
    Tokenizes a TGrep search string into separate tokens.
    '''
    parser = _build_tgrep_parser(False)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string)) 
Example 24
Project: serverless-chatbots-workshop   Author: datteswararao   File: tgrep.py    Apache License 2.0 5 votes vote down vote up
def tgrep_compile(tgrep_string):
    '''
    Parses (and tokenizes, if necessary) a TGrep search string into a
    lambda function.
    '''
    parser = _build_tgrep_parser(True)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string, parseAll=True))[0] 
Example 25
Project: serverless-chatbots-workshop   Author: datteswararao   File: tgrep.py    Apache License 2.0 5 votes vote down vote up
def tgrep_positions(pattern, trees, search_leaves=True):
    """
    Return the tree positions in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree positions)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [position for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 26
Project: serverless-chatbots-workshop   Author: datteswararao   File: tgrep.py    Apache License 2.0 5 votes vote down vote up
def tgrep_nodes(pattern, trees, search_leaves=True):
    """
    Return the tree nodes in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree nodes)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [tree[position] for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 27
Project: serverless-chatbots-workshop   Author: datteswararao   File: tgrep.py    Apache License 2.0 5 votes vote down vote up
def tgrep_tokenize(tgrep_string):
    '''
    Tokenizes a TGrep search string into separate tokens.
    '''
    parser = _build_tgrep_parser(False)
    if isinstance(tgrep_string, binary_type):
        tgrep_string = tgrep_string.decode()
    return list(parser.parseString(tgrep_string)) 
Example 28
Project: serverless-chatbots-workshop   Author: datteswararao   File: tgrep.py    Apache License 2.0 5 votes vote down vote up
def tgrep_positions(pattern, trees, search_leaves=True):
    """
    Return the tree positions in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree positions)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [position for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield [] 
Example 29
Project: serverless-chatbots-workshop   Author: datteswararao   File: tgrep.py    Apache License 2.0 5 votes vote down vote up
def tgrep_nodes(pattern, trees, search_leaves=True):
    """
    Return the tree nodes in the trees which match the given pattern.

    :param pattern: a tgrep search pattern
    :type pattern: str or output of tgrep_compile()
    :param trees: a sequence of NLTK trees (usually ParentedTrees)
    :type trees: iter(ParentedTree) or iter(Tree)
    :param search_leaves: whether ot return matching leaf nodes
    :type search_leaves: bool
    :rtype: iter(tree nodes)
    """

    if isinstance(pattern, (binary_type, text_type)):
        pattern = tgrep_compile(pattern)

    for tree in trees:
        try:
            if search_leaves:
                positions = tree.treepositions()
            else:
                positions = treepositions_no_leaves(tree)
            yield [tree[position] for position in positions
                      if pattern(tree[position])]
        except AttributeError:
            yield []