Python ply.lex.lexer() Examples
The following are 30
code examples of ply.lex.lexer().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
ply.lex
, or try the search function
.
Example #1
Source File: yacc.py From SublimeKSP with GNU General Public License v3.0 | 6 votes |
def parse(self,input=None,lexer=None,debug=0,tracking=0,tokenfunc=None): if debug or yaccdevel: if isinstance(debug,int): debug = PlyLogger(sys.stderr) return self.parsedebug(input,lexer,debug,tracking,tokenfunc) elif tracking: return self.parseopt(input,lexer,debug,tracking,tokenfunc) else: return self.parseopt_notrack(input,lexer,debug,tracking,tokenfunc) # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # parsedebug(). # # This is the debugging enabled version of parse(). All changes made to the # parsing engine should be made here. For the non-debugging version, # copy this code to a method parseopt() and delete all of the sections # enclosed in: # # #--! DEBUG # statements # #--! DEBUG # # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Example #2
Source File: cpp.py From misp42splunk with GNU Lesser General Public License v3.0 | 6 votes |
def __init__(self,lexer=None): if lexer is None: lexer = lex.lexer self.lexer = lexer self.macros = { } self.path = [] self.temp_path = [] # Probe the lexer for selected tokens self.lexprobe() tm = time.localtime() self.define("__DATE__ \"%s\"" % time.strftime("%b %d %Y",tm)) self.define("__TIME__ \"%s\"" % time.strftime("%H:%M:%S",tm)) self.parser = None # ----------------------------------------------------------------------------- # tokenize() # # Utility function. Given a string of text, tokenize into a list of tokens # -----------------------------------------------------------------------------
Example #3
Source File: yacc.py From oss-ftp with MIT License | 6 votes |
def parse(self,input=None,lexer=None,debug=0,tracking=0,tokenfunc=None): if debug or yaccdevel: if isinstance(debug,int): debug = PlyLogger(sys.stderr) return self.parsedebug(input,lexer,debug,tracking,tokenfunc) elif tracking: return self.parseopt(input,lexer,debug,tracking,tokenfunc) else: return self.parseopt_notrack(input,lexer,debug,tracking,tokenfunc) # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # parsedebug(). # # This is the debugging enabled version of parse(). All changes made to the # parsing engine should be made here. For the non-debugging version, # copy this code to a method parseopt() and delete all of the sections # enclosed in: # # #--! DEBUG # statements # #--! DEBUG # # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Example #4
Source File: cpp.py From SwiftKitten with MIT License | 6 votes |
def __init__(self,lexer=None): if lexer is None: lexer = lex.lexer self.lexer = lexer self.macros = { } self.path = [] self.temp_path = [] # Probe the lexer for selected tokens self.lexprobe() tm = time.localtime() self.define("__DATE__ \"%s\"" % time.strftime("%b %d %Y",tm)) self.define("__TIME__ \"%s\"" % time.strftime("%H:%M:%S",tm)) self.parser = None # ----------------------------------------------------------------------------- # tokenize() # # Utility function. Given a string of text, tokenize into a list of tokens # -----------------------------------------------------------------------------
Example #5
Source File: cpp.py From oss-ftp with MIT License | 6 votes |
def __init__(self,lexer=None): if lexer is None: lexer = lex.lexer self.lexer = lexer self.macros = { } self.path = [] self.temp_path = [] # Probe the lexer for selected tokens self.lexprobe() tm = time.localtime() self.define("__DATE__ \"%s\"" % time.strftime("%b %d %Y",tm)) self.define("__TIME__ \"%s\"" % time.strftime("%H:%M:%S",tm)) self.parser = None # ----------------------------------------------------------------------------- # tokenize() # # Utility function. Given a string of text, tokenize into a list of tokens # -----------------------------------------------------------------------------
Example #6
Source File: yacc.py From SwiftKitten with MIT License | 6 votes |
def parse(self,input=None,lexer=None,debug=0,tracking=0,tokenfunc=None): if debug or yaccdevel: if isinstance(debug,int): debug = PlyLogger(sys.stderr) return self.parsedebug(input,lexer,debug,tracking,tokenfunc) elif tracking: return self.parseopt(input,lexer,debug,tracking,tokenfunc) else: return self.parseopt_notrack(input,lexer,debug,tracking,tokenfunc) # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # parsedebug(). # # This is the debugging enabled version of parse(). All changes made to the # parsing engine should be made here. For the non-debugging version, # copy this code to a method parseopt() and delete all of the sections # enclosed in: # # #--! DEBUG # statements # #--! DEBUG # # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
Example #7
Source File: cpp.py From SublimeKSP with GNU General Public License v3.0 | 6 votes |
def __init__(self,lexer=None): if lexer is None: lexer = lex.lexer self.lexer = lexer self.macros = { } self.path = [] self.temp_path = [] # Probe the lexer for selected tokens self.lexprobe() tm = time.localtime() self.define("__DATE__ \"%s\"" % time.strftime("%b %d %Y",tm)) self.define("__TIME__ \"%s\"" % time.strftime("%H:%M:%S",tm)) self.parser = None # ----------------------------------------------------------------------------- # tokenize() # # Utility function. Given a string of text, tokenize into a list of tokens # -----------------------------------------------------------------------------
Example #8
Source File: cpp.py From rfmt with Apache License 2.0 | 5 votes |
def t_CPP_WS(t): r'\s+' t.lexer.lineno += t.value.count("\n") return t
Example #9
Source File: cpp.py From learn_python3_spider with MIT License | 5 votes |
def t_error(t): t.type = t.value[0] t.value = t.value[0] t.lexer.skip(1) return t
Example #10
Source File: yacc.py From SublimeKSP with GNU General Public License v3.0 | 5 votes |
def __init__(self,s,stack=None): self.slice = s self.stack = stack self.lexer = None self.parser= None
Example #11
Source File: cpp.py From learn_python3_spider with MIT License | 5 votes |
def t_CPP_COMMENT1(t): r'(/\*(.|\n)*?\*/)' ncr = t.value.count("\n") t.lexer.lineno += ncr # replace with one space or a number of '\n' t.type = 'CPP_WS'; t.value = '\n' * ncr if ncr else ' ' return t # Line comment
Example #12
Source File: cpp.py From learn_python3_spider with MIT License | 5 votes |
def t_CPP_WS(t): r'\s+' t.lexer.lineno += t.value.count("\n") return t
Example #13
Source File: cpp.py From retdec-regression-tests-framework with MIT License | 5 votes |
def group_lines(self,input): lex = self.lexer.clone() lines = [x.rstrip() for x in input.splitlines()] for i in xrange(len(lines)): j = i+1 while lines[i].endswith('\\') and (j < len(lines)): lines[i] = lines[i][:-1]+lines[j] lines[j] = "" j += 1 input = "\n".join(lines) lex.input(input) lex.lineno = 1 current_line = [] while True: tok = lex.token() if not tok: break current_line.append(tok) if tok.type in self.t_WS and '\n' in tok.value: yield current_line current_line = [] if current_line: yield current_line # ---------------------------------------------------------------------- # tokenstrip() # # Remove leading/trailing whitespace tokens from a token list # ----------------------------------------------------------------------
Example #14
Source File: cpp.py From retdec-regression-tests-framework with MIT License | 5 votes |
def error(self,file,line,msg): print("%s:%d %s" % (file,line,msg)) # ---------------------------------------------------------------------- # lexprobe() # # This method probes the preprocessor lexer object to discover # the token types of symbols that are important to the preprocessor. # If this works right, the preprocessor will simply "work" # with any suitable lexer regardless of how tokens have been named. # ----------------------------------------------------------------------
Example #15
Source File: cpp.py From retdec-regression-tests-framework with MIT License | 5 votes |
def tokenize(self,text): tokens = [] self.lexer.input(text) while True: tok = self.lexer.token() if not tok: break tokens.append(tok) return tokens # --------------------------------------------------------------------- # error() # # Report a preprocessor error/warning of some kind # ----------------------------------------------------------------------
Example #16
Source File: cpp.py From learn_python3_spider with MIT License | 5 votes |
def t_CPP_STRING(t): r'\"([^\\\n]|(\\(.|\n)))*?\"' t.lexer.lineno += t.value.count("\n") return t # Character constant 'c' or L'c'
Example #17
Source File: cpp.py From rfmt with Apache License 2.0 | 5 votes |
def t_CPP_CHAR(t): r'(L)?\'([^\\\n]|(\\(.|\n)))*?\'' t.lexer.lineno += t.value.count("\n") return t # Comment
Example #18
Source File: cpp.py From Safejumper-for-Desktop with GNU General Public License v2.0 | 5 votes |
def t_error(t): t.type = t.value[0] t.value = t.value[0] t.lexer.skip(1) return t
Example #19
Source File: cpp.py From rfmt with Apache License 2.0 | 5 votes |
def t_CPP_STRING(t): r'\"([^\\\n]|(\\(.|\n)))*?\"' t.lexer.lineno += t.value.count("\n") return t # Character constant 'c' or L'c'
Example #20
Source File: cpp.py From SwiftKitten with MIT License | 5 votes |
def error(self,file,line,msg): print("%s:%d %s" % (file,line,msg)) # ---------------------------------------------------------------------- # lexprobe() # # This method probes the preprocessor lexer object to discover # the token types of symbols that are important to the preprocessor. # If this works right, the preprocessor will simply "work" # with any suitable lexer regardless of how tokens have been named. # ----------------------------------------------------------------------
Example #21
Source File: cpp.py From SwiftKitten with MIT License | 5 votes |
def tokenize(self,text): tokens = [] self.lexer.input(text) while True: tok = self.lexer.token() if not tok: break tokens.append(tok) return tokens # --------------------------------------------------------------------- # error() # # Report a preprocessor error/warning of some kind # ----------------------------------------------------------------------
Example #22
Source File: cpp.py From learn_python3_spider with MIT License | 5 votes |
def tokenize(self,text): tokens = [] self.lexer.input(text) while True: tok = self.lexer.token() if not tok: break tokens.append(tok) return tokens # --------------------------------------------------------------------- # error() # # Report a preprocessor error/warning of some kind # ----------------------------------------------------------------------
Example #23
Source File: cpp.py From Safejumper-for-Desktop with GNU General Public License v2.0 | 5 votes |
def tokenize(self,text): tokens = [] self.lexer.input(text) while True: tok = self.lexer.token() if not tok: break tokens.append(tok) return tokens # --------------------------------------------------------------------- # error() # # Report a preprocessor error/warning of some kind # ----------------------------------------------------------------------
Example #24
Source File: cpp.py From SwiftKitten with MIT License | 5 votes |
def t_error(t): t.type = t.value[0] t.value = t.value[0] t.lexer.skip(1) return t
Example #25
Source File: cpp.py From SwiftKitten with MIT License | 5 votes |
def t_CPP_CHAR(t): r'(L)?\'([^\\\n]|(\\(.|\n)))*?\'' t.lexer.lineno += t.value.count("\n") return t # Comment
Example #26
Source File: cpp.py From SwiftKitten with MIT License | 5 votes |
def t_CPP_STRING(t): r'\"([^\\\n]|(\\(.|\n)))*?\"' t.lexer.lineno += t.value.count("\n") return t # Character constant 'c' or L'c'
Example #27
Source File: cpp.py From Safejumper-for-Desktop with GNU General Public License v2.0 | 5 votes |
def error(self,file,line,msg): print("%s:%d %s" % (file,line,msg)) # ---------------------------------------------------------------------- # lexprobe() # # This method probes the preprocessor lexer object to discover # the token types of symbols that are important to the preprocessor. # If this works right, the preprocessor will simply "work" # with any suitable lexer regardless of how tokens have been named. # ----------------------------------------------------------------------
Example #28
Source File: cpp.py From SwiftKitten with MIT License | 5 votes |
def t_CPP_WS(t): r'\s+' t.lexer.lineno += t.value.count("\n") return t
Example #29
Source File: cpp.py From Safejumper-for-Desktop with GNU General Public License v2.0 | 5 votes |
def group_lines(self,input): lex = self.lexer.clone() lines = [x.rstrip() for x in input.splitlines()] for i in xrange(len(lines)): j = i+1 while lines[i].endswith('\\') and (j < len(lines)): lines[i] = lines[i][:-1]+lines[j] lines[j] = "" j += 1 input = "\n".join(lines) lex.input(input) lex.lineno = 1 current_line = [] while True: tok = lex.token() if not tok: break current_line.append(tok) if tok.type in self.t_WS and '\n' in tok.value: yield current_line current_line = [] if current_line: yield current_line # ---------------------------------------------------------------------- # tokenstrip() # # Remove leading/trailing whitespace tokens from a token list # ----------------------------------------------------------------------
Example #30
Source File: yacc.py From SwiftKitten with MIT License | 5 votes |
def __init__(self,s,stack=None): self.slice = s self.stack = stack self.lexer = None self.parser= None