Python re.S Examples

The following are 30 code examples of re.S(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module re , or try the search function .
Example #1
Source File: template.py    From misp42splunk with GNU Lesser General Public License v3.0 7 votes vote down vote up
def get_module_source_metadata(cls, module_source, full_line_map=False):
        source_map = re.search(
            r"__M_BEGIN_METADATA(.+?)__M_END_METADATA", module_source, re.S
        ).group(1)
        source_map = json.loads(source_map)
        source_map["line_map"] = dict(
            (int(k), int(v)) for k, v in source_map["line_map"].items()
        )
        if full_line_map:
            f_line_map = source_map["full_line_map"] = []
            line_map = source_map["line_map"]

            curr_templ_line = 1
            for mod_line in range(1, max(line_map)):
                if mod_line in line_map:
                    curr_templ_line = line_map[mod_line]
                f_line_map.append(curr_templ_line)
        return source_map 
Example #2
Source File: sublist3r.py    From subtake with GNU General Public License v2.0 6 votes vote down vote up
def extract_domains(self, resp):
        tbl_regex = re.compile('<a name="hostanchor"><\/a>Host Records.*?<table.*?>(.*?)</table>', re.S)
        link_regex = re.compile('<td class="col-md-4">(.*?)<br>', re.S)
        links = []
        try:
            results_tbl = tbl_regex.findall(resp)[0]
        except IndexError:
            results_tbl = ''
        links_list = link_regex.findall(results_tbl)
        links = list(set(links_list))
        for link in links:
            subdomain = link.strip()
            if not subdomain.endswith(self.domain):
                continue
            if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
                self.subdomains.append(subdomain.strip())
        return links 
Example #3
Source File: ip_history.py    From Vxscan with Apache License 2.0 6 votes vote down vote up
def ipinfo(host):
    out = []
    if not re.search(r'\d+\.\d+\.\d+\.\d+', host):
        req = Requests()
        # noinspection PyBroadException
        try:
            r = req.get('https://viewdns.info/iphistory/?domain={}'.format(host))
            result = re.findall(r'(?<=<tr><td>)\d+\.\d+\.\d+\.\d+(?=</td><td>)', r.text, re.S | re.I)
            if result:
                for i in result:
                    if iscdn(i):
                        out.append(i)
        except Exception:
            pass

    return out 
Example #4
Source File: tools.py    From hadrian with Apache License 2.0 6 votes vote down vote up
def getmatch(self, haystack):
        if not isinstance(haystack, basestring):
            return None
        flags = 0
        if self.flags is not None:
            if "i" in self.flags or "I" in self.flags:
                flags |= re.I
            if "l" in self.flags or "L" in self.flags:
                flags |= re.L
            if "m" in self.flags or "M" in self.flags:
                flags |= re.M
            if "s" in self.flags or "S" in self.flags:
                flags |= re.S
            if "u" in self.flags or "U" in self.flags:
                flags |= re.U
            if "x" in self.flags or "X" in self.flags:
                flags |= re.X
        if re.match(self.pattern, haystack, flags=flags) is None:
            return None
        elif self.to is None:
            return Match(haystack, haystack)
        else:
            return Match(haystack, re.sub(self.pattern, self.to, haystack, flags=flags)) 
Example #5
Source File: lexer.py    From misp42splunk with GNU Lesser General Public License v3.0 6 votes vote down vote up
def compile_rules(environment):
    """Compiles all the rules from the environment into a list of rules."""
    e = re.escape
    rules = [
        (len(environment.comment_start_string), 'comment',
         e(environment.comment_start_string)),
        (len(environment.block_start_string), 'block',
         e(environment.block_start_string)),
        (len(environment.variable_start_string), 'variable',
         e(environment.variable_start_string))
    ]

    if environment.line_statement_prefix is not None:
        rules.append((len(environment.line_statement_prefix), 'linestatement',
                      r'^[ \t\v]*' + e(environment.line_statement_prefix)))
    if environment.line_comment_prefix is not None:
        rules.append((len(environment.line_comment_prefix), 'linecomment',
                      r'(?:^|(?<=\S))[^\S\r\n]*' +
                      e(environment.line_comment_prefix)))

    return [x[1:] for x in sorted(rules, reverse=True)] 
Example #6
Source File: ast.py    From misp42splunk with GNU Lesser General Public License v3.0 6 votes vote down vote up
def __init__(self, code, **exception_kwargs):
        self.codeargs = []
        self.args = []
        self.declared_identifiers = set()
        self.undeclared_identifiers = set()
        if isinstance(code, compat.string_types):
            if re.match(r"\S", code) and not re.match(r",\s*$", code):
                # if theres text and no trailing comma, insure its parsed
                # as a tuple by adding a trailing comma
                code += ","
            expr = pyparser.parse(code, "exec", **exception_kwargs)
        else:
            expr = code

        f = pyparser.FindTuple(self, PythonCode, **exception_kwargs)
        f.visit(expr) 
Example #7
Source File: ast.py    From misp42splunk with GNU Lesser General Public License v3.0 6 votes vote down vote up
def __init__(self, code, **exception_kwargs):
        self.codeargs = []
        self.args = []
        self.declared_identifiers = set()
        self.undeclared_identifiers = set()
        if isinstance(code, compat.string_types):
            if re.match(r"\S", code) and not re.match(r",\s*$", code):
                # if theres text and no trailing comma, insure its parsed
                # as a tuple by adding a trailing comma
                code += ","
            expr = pyparser.parse(code, "exec", **exception_kwargs)
        else:
            expr = code

        f = pyparser.FindTuple(self, PythonCode, **exception_kwargs)
        f.visit(expr) 
Example #8
Source File: template.py    From misp42splunk with GNU Lesser General Public License v3.0 6 votes vote down vote up
def get_module_source_metadata(cls, module_source, full_line_map=False):
        source_map = re.search(
            r"__M_BEGIN_METADATA(.+?)__M_END_METADATA", module_source, re.S
        ).group(1)
        source_map = json.loads(source_map)
        source_map["line_map"] = dict(
            (int(k), int(v)) for k, v in source_map["line_map"].items()
        )
        if full_line_map:
            f_line_map = source_map["full_line_map"] = []
            line_map = source_map["line_map"]

            curr_templ_line = 1
            for mod_line in range(1, max(line_map)):
                if mod_line in line_map:
                    curr_templ_line = line_map[mod_line]
                f_line_map.append(curr_templ_line)
        return source_map 
Example #9
Source File: util.py    From lyrics-in-terminal with MIT License 6 votes vote down vote up
def get_azlyrics(url):
    az_html = get_az_html(url)
    if isinstance(az_html, tuple):
        return az_html[0]

    az_regex = re.compile(r'<!-- Usage of azlyrics.com content by any third-party lyrics provider is prohibited by our licensing agreement. Sorry about that. -->(.*)<!-- MxM banner -->', re.S)

    ly = az_regex.search(az_html)
    if ly == None:
        # Az lyrics not found
        return 'Azlyrics missing...'

    rep = {'&quot;': '\"', '&amp;': '&', '\r' : ''}

    ly = re.sub(r'<[/]?\w*?>', '', ly.group(1)).strip()  
    # ly = ly.replace('&quot;', '\"').replace('&amp;', '&')
    # regex = re.compile('|'.join(substrings))
    ly = re.sub('|'.join(rep.keys()), lambda match: rep[match.group(0)], ly)
    lyrics_lines = ly.split('\n')

    return lyrics_lines 
Example #10
Source File: versioneer.py    From datasette with Apache License 2.0 6 votes vote down vote up
def versions_from_file(filename):
    """Try to determine the version from _version.py if present."""
    try:
        with open(filename) as f:
            contents = f.read()
    except EnvironmentError:
        raise NotThisMethod("unable to read _version.py")
    mo = re.search(
        r"version_json = '''\n(.*)'''  # END VERSION_JSON", contents, re.M | re.S
    )
    if not mo:
        mo = re.search(
            r"version_json = '''\r\n(.*)'''  # END VERSION_JSON", contents, re.M | re.S
        )
    if not mo:
        raise NotThisMethod("no version_json in _version.py")
    return json.loads(mo.group(1)) 
Example #11
Source File: gdbservermanager.py    From ffw with GNU General Public License v3.0 6 votes vote down vote up
def _getCrashDetails(self):
        ret = self.gdbOutput
        logging.info("get crash details, res: " + str(len(ret)))
        p = re.compile('#.*\(gdb\)', re.S)
        backtrace = re.search(p, ret, flags=0).group()
        #self.queue_stdout.put(backtrace)
        backtraceFrames = backtrace.split('\n')
        i = 0
        res = []
        while(i < len(backtraceFrames)):
            if backtraceFrames[i].startswith("#"):
                res.append(backtraceFrames[i].rstrip("\n\r"))
            i += 1

        serverCrashData = ServerCrashData(
            backtrace=res,
            analyzerOutput=ret,
            analyzerType="gdb"
        )
        gdbOutput = targetutils.getAsanOutput(self.config, self.pid)
        if gdbOutput is not None:
            serverCrashData.setAsan(gdbOutput)

        return serverCrashData 
Example #12
Source File: lexer.py    From recruit with Apache License 2.0 6 votes vote down vote up
def compile_rules(environment):
    """Compiles all the rules from the environment into a list of rules."""
    e = re.escape
    rules = [
        (len(environment.comment_start_string), 'comment',
         e(environment.comment_start_string)),
        (len(environment.block_start_string), 'block',
         e(environment.block_start_string)),
        (len(environment.variable_start_string), 'variable',
         e(environment.variable_start_string))
    ]

    if environment.line_statement_prefix is not None:
        rules.append((len(environment.line_statement_prefix), 'linestatement',
                      r'^[ \t\v]*' + e(environment.line_statement_prefix)))
    if environment.line_comment_prefix is not None:
        rules.append((len(environment.line_comment_prefix), 'linecomment',
                      r'(?:^|(?<=\S))[^\S\r\n]*' +
                      e(environment.line_comment_prefix)))

    return [x[1:] for x in sorted(rules, reverse=True)] 
Example #13
Source File: parser.py    From recruit with Apache License 2.0 6 votes vote down vote up
def split_arg_string(string):
    """Given an argument string this attempts to split it into small parts."""
    rv = []
    for match in re.finditer(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
                             r'|"([^"\\]*(?:\\.[^"\\]*)*)"'
                             r'|\S+)\s*', string, re.S):
        arg = match.group().strip()
        if arg[:1] == arg[-1:] and arg[:1] in '"\'':
            arg = arg[1:-1].encode('ascii', 'backslashreplace') \
                .decode('unicode-escape')
        try:
            arg = type(string)(arg)
        except UnicodeError:
            pass
        rv.append(arg)
    return rv 
Example #14
Source File: lexer.py    From recruit with Apache License 2.0 6 votes vote down vote up
def compile_rules(environment):
    """Compiles all the rules from the environment into a list of rules."""
    e = re.escape
    rules = [
        (len(environment.comment_start_string), 'comment',
         e(environment.comment_start_string)),
        (len(environment.block_start_string), 'block',
         e(environment.block_start_string)),
        (len(environment.variable_start_string), 'variable',
         e(environment.variable_start_string))
    ]

    if environment.line_statement_prefix is not None:
        rules.append((len(environment.line_statement_prefix), 'linestatement',
                      r'^[ \t\v]*' + e(environment.line_statement_prefix)))
    if environment.line_comment_prefix is not None:
        rules.append((len(environment.line_comment_prefix), 'linecomment',
                      r'(?:^|(?<=\S))[^\S\r\n]*' +
                      e(environment.line_comment_prefix)))

    return [x[1:] for x in sorted(rules, reverse=True)] 
Example #15
Source File: test_sharedconsole.py    From molotov with Apache License 2.0 6 votes vote down vote up
def test_simple_usage(self):
        test_loop = asyncio.get_event_loop()
        console = SharedConsole(interval=0.0)

        async def add_lines():
            console.print("one")
            console.print("two")
            console.print("3")
            try:
                1 + "e"
            except Exception as e:
                console.print_error(e)
                console.print_error(e, sys.exc_info()[2])
            await asyncio.sleep(0.2)
            await console.stop()

        with catch_output() as (stdout, stderr):
            adder = asyncio.ensure_future(add_lines())
            displayer = asyncio.ensure_future(console.display())
            test_loop.run_until_complete(asyncio.gather(adder, displayer))

        output = stdout.read()
        test_loop.close()
        self.assertTrue(re.match(OUTPUT, output, re.S | re.M) is not None, output) 
Example #16
Source File: versioneer.py    From mknotebooks with MIT License 6 votes vote down vote up
def versions_from_file(filename):
    """Try to determine the version from _version.py if present."""
    try:
        with open(filename) as f:
            contents = f.read()
    except EnvironmentError:
        raise NotThisMethod("unable to read _version.py")
    mo = re.search(
        r"version_json = '''\n(.*)'''  # END VERSION_JSON", contents, re.M | re.S
    )
    if not mo:
        mo = re.search(
            r"version_json = '''\r\n(.*)'''  # END VERSION_JSON", contents, re.M | re.S
        )
    if not mo:
        raise NotThisMethod("no version_json in _version.py")
    return json.loads(mo.group(1)) 
Example #17
Source File: usproxy.py    From IPProxyTool with MIT License 6 votes vote down vote up
def parse_page(self, response):
        pattern = re.compile(
		'<tr><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td><td.+?>(.*?)</td><td>(.*?)</td><td.+?>(.*?)</td><td.+?>(.*?)</td><td.+?>(.*?)</td></tr>',
                re.S)
        items = re.findall(pattern, response.body.decode())

        if items is not None:
            for item in items:
                proxy = Proxy()
                proxy.set_value(
                        ip = item[0],
                        port = item[1],
                        country = item[3],
                        anonymity = item[4],
                        source = self.name,
                )

                self.add_proxy(proxy) 
Example #18
Source File: gatherproxy.py    From IPProxyTool with MIT License 6 votes vote down vote up
def parse_page(self, response):
        pattern = re.compile('gp.insertPrx\((.*?)\)', re.S)
        items = re.findall(pattern, response.body.decode())
        for item in items:
            data = json.loads(item)
            #端口用的是十六进制
            port = data.get('PROXY_PORT')
            port = str(int(port, 16))

            proxy = Proxy()
            proxy.set_value(
                    ip = data.get('PROXY_IP'),
                    port = port,
                    country = data.get('PROXY_COUNTRY'),
                    anonymity = data.get('PROXY_TYPE'),
                    source = self.name,
            )

            self.add_proxy(proxy = proxy) 
Example #19
Source File: sixsixip.py    From IPProxyTool with MIT License 6 votes vote down vote up
def parse_page(self, response):
        pattern = re.compile('<tr><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td><td>(.*?)</td></tr>',
                             re.S)
        items = re.findall(pattern, response.body.decode())
        for i, item in enumerate(items):
            if i >= 1:
                proxy = Proxy()
                proxy.set_value(
                        ip = item[0],
                        port = item[1],
                        country = item[2],
                        anonymity = item[3],
                        source = self.name
                )

                self.add_proxy(proxy = proxy) 
Example #20
Source File: lexer.py    From jbox with MIT License 6 votes vote down vote up
def compile_rules(environment):
    """Compiles all the rules from the environment into a list of rules."""
    e = re.escape
    rules = [
        (len(environment.comment_start_string), 'comment',
         e(environment.comment_start_string)),
        (len(environment.block_start_string), 'block',
         e(environment.block_start_string)),
        (len(environment.variable_start_string), 'variable',
         e(environment.variable_start_string))
    ]

    if environment.line_statement_prefix is not None:
        rules.append((len(environment.line_statement_prefix), 'linestatement',
                      r'^[ \t\v]*' + e(environment.line_statement_prefix)))
    if environment.line_comment_prefix is not None:
        rules.append((len(environment.line_comment_prefix), 'linecomment',
                      r'(?:^|(?<=\S))[^\S\r\n]*' +
                      e(environment.line_comment_prefix)))

    return [x[1:] for x in sorted(rules, reverse=True)] 
Example #21
Source File: ast.py    From jbox with MIT License 6 votes vote down vote up
def __init__(self, code, **exception_kwargs):
        self.codeargs = []
        self.args = []
        self.declared_identifiers = set()
        self.undeclared_identifiers = set()
        if isinstance(code, compat.string_types):
            if re.match(r"\S", code) and not re.match(r",\s*$", code):
                # if theres text and no trailing comma, insure its parsed
                # as a tuple by adding a trailing comma
                code += ","
            expr = pyparser.parse(code, "exec", **exception_kwargs)
        else:
            expr = code

        f = pyparser.FindTuple(self, PythonCode, **exception_kwargs)
        f.visit(expr) 
Example #22
Source File: ast.py    From jbox with MIT License 6 votes vote down vote up
def __init__(self, code, **exception_kwargs):
        m = re.match(r'^(\w+)(?:\s+(.*?))?:\s*(#|$)', code.strip(), re.S)
        if not m:
            raise exceptions.CompileException(
                "Fragment '%s' is not a partial control statement" %
                code, **exception_kwargs)
        if m.group(3):
            code = code[:m.start(3)]
        (keyword, expr) = m.group(1, 2)
        if keyword in ['for', 'if', 'while']:
            code = code + "pass"
        elif keyword == 'try':
            code = code + "pass\nexcept:pass"
        elif keyword == 'elif' or keyword == 'else':
            code = "if False:pass\n" + code + "pass"
        elif keyword == 'except':
            code = "try:pass\n" + code + "pass"
        elif keyword == 'with':
            code = code + "pass"
        else:
            raise exceptions.CompileException(
                "Unsupported control keyword: '%s'" %
                keyword, **exception_kwargs)
        super(PythonFragment, self).__init__(code, **exception_kwargs) 
Example #23
Source File: versioneer.py    From aospy with Apache License 2.0 5 votes vote down vote up
def versions_from_file(filename):
    """Try to determine the version from _version.py if present."""
    try:
        with open(filename) as f:
            contents = f.read()
    except EnvironmentError:
        raise NotThisMethod("unable to read _version.py")
    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
                   contents, re.M | re.S)
    if not mo:
        mo = re.search(r"version_json = '''\r\n(.*)'''  # END VERSION_JSON",
                       contents, re.M | re.S)
    if not mo:
        raise NotThisMethod("no version_json in _version.py")
    return json.loads(mo.group(1)) 
Example #24
Source File: versioneer.py    From xrft with MIT License 5 votes vote down vote up
def versions_from_file(filename):
    """Try to determine the version from _version.py if present."""
    try:
        with open(filename) as f:
            contents = f.read()
    except EnvironmentError:
        raise NotThisMethod("unable to read _version.py")
    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
                   contents, re.M | re.S)
    if not mo:
        mo = re.search(r"version_json = '''\r\n(.*)'''  # END VERSION_JSON",
                       contents, re.M | re.S)
    if not mo:
        raise NotThisMethod("no version_json in _version.py")
    return json.loads(mo.group(1)) 
Example #25
Source File: versioneer.py    From NiBetaSeries with MIT License 5 votes vote down vote up
def versions_from_file(filename):
    """Try to determine the version from _version.py if present."""
    try:
        with open(filename) as f:
            contents = f.read()
    except EnvironmentError:
        raise NotThisMethod("unable to read _version.py")
    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
                   contents, re.M | re.S)
    if not mo:
        mo = re.search(r"version_json = '''\r\n(.*)'''  # END VERSION_JSON",
                       contents, re.M | re.S)
    if not mo:
        raise NotThisMethod("no version_json in _version.py")
    return json.loads(mo.group(1)) 
Example #26
Source File: versioneer.py    From QCElemental with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def versions_from_file(filename):
    """Try to determine the version from _version.py if present."""
    try:
        with open(filename) as f:
            contents = f.read()
    except EnvironmentError:
        raise NotThisMethod("unable to read _version.py")
    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
                   contents, re.M | re.S)
    if not mo:
        mo = re.search(r"version_json = '''\r\n(.*)'''  # END VERSION_JSON",
                       contents, re.M | re.S)
    if not mo:
        raise NotThisMethod("no version_json in _version.py")
    return json.loads(mo.group(1)) 
Example #27
Source File: sublist3r.py    From subtake with GNU General Public License v2.0 5 votes vote down vote up
def get_csrftoken(self, resp):
        csrf_regex = re.compile("<input type='hidden' name='csrfmiddlewaretoken' value='(.*?)' />", re.S)
        token = csrf_regex.findall(resp)[0]
        return token.strip() 
Example #28
Source File: sublist3r.py    From subtake with GNU General Public License v2.0 5 votes vote down vote up
def extract_domains(self, resp):
        link_regx = re.compile('<div class="enum.*?">.*?<a target="_blank" href=".*?">(.*?)</a>', re.S)
        try:
            links = link_regx.findall(resp)
            for link in links:
                subdomain = link.strip()
                if not subdomain.endswith(self.domain):
                    continue
                if subdomain not in self.subdomains and subdomain != self.domain:
                    if self.verbose:
                        self.print_("%s%s: %s%s" % (R, self.engine_name, W, subdomain))
                    self.subdomains.append(subdomain.strip())
        except Exception:
            pass 
Example #29
Source File: versioneer.py    From btle-sniffer with MIT License 5 votes vote down vote up
def versions_from_file(filename):
    """Try to determine the version from _version.py if present."""
    try:
        with open(filename) as f:
            contents = f.read()
    except EnvironmentError:
        raise NotThisMethod("unable to read _version.py")
    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
                   contents, re.M | re.S)
    if not mo:
        mo = re.search(r"version_json = '''\r\n(.*)'''  # END VERSION_JSON",
                       contents, re.M | re.S)
    if not mo:
        raise NotThisMethod("no version_json in _version.py")
    return json.loads(mo.group(1)) 
Example #30
Source File: versioneer.py    From matchpy with MIT License 5 votes vote down vote up
def versions_from_file(filename):
    """Try to determine the version from _version.py if present."""
    try:
        with open(filename) as f:
            contents = f.read()
    except EnvironmentError:
        raise NotThisMethod("unable to read _version.py")
    mo = re.search(r"version_json = '''\n(.*)'''  # END VERSION_JSON",
                   contents, re.M | re.S)
    if not mo:
        mo = re.search(r"version_json = '''\r\n(.*)'''  # END VERSION_JSON",
                       contents, re.M | re.S)
    if not mo:
        raise NotThisMethod("no version_json in _version.py")
    return json.loads(mo.group(1))