Python re.DOTALL Examples

def render_re(regex):
  """Renders a repr()-style value for a compiled regular expression."""
  actual_flags = []
  if regex.flags:
    flags = [
      (re.IGNORECASE, 'IGNORECASE'),
      (re.LOCALE, 'LOCALE'),
      (re.UNICODE, 'UNICODE'),
      (re.MULTILINE, 'MULTILINE'),
      (re.DOTALL, 'DOTALL'),
      (re.VERBOSE, 'VERBOSE'),
    ]
    for val, name in flags:
      if regex.flags & val:
        actual_flags.append(name)
  if actual_flags:
    return 're.compile(%r, %s)' % (regex.pattern, '|'.join(actual_flags))
  else:
    return 're.compile(%r)' % regex.pattern 

def ssh_edit_file(adress, user, passw, remotefile, regex, replace):
        client = paramiko.SSHClient()
        client.load_system_host_keys()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        trans = paramiko.Transport((adress, 22))
        trans.connect(username=user, password=passw)
        sftp = paramiko.SFTPClient.from_transport(trans)
        f_in = sftp.file(remotefile, "r")
        c_in = f_in.read()
        pattern = re.compile(regex, re.MULTILINE | re.DOTALL)
        c_out = pattern.sub(replace, c_in)
        f_out = sftp.file(remotefile, "w")
        f_out.write(c_out)
        f_in.close()
        f_out.close()
        sftp.close()
        trans.close() 

def show_system_resources(self):
        self.xapi.op(cmd="show system resources", cmd_xml=True)
        result = self.xapi.xml_root()
        regex = re.compile(r"load average: ([\d.]+).* ([\d.]+)%id.*Mem:.*?([\d.]+)k total.*?([\d]+)k free", re.DOTALL)
        match = regex.search(result)
        if match:
            """
            return cpu, mem_free, load
            """
            return {
                'load': Decimal(match.group(1)),
                'cpu': 100 - Decimal(match.group(2)),
                'mem_total': int(match.group(3)),
                'mem_free': int(match.group(4)),
            }
        else:
            raise err.PanDeviceError("Problem parsing show system resources",
                                     pan_device=self) 

def update_sys_cfg_file(uninstall_distro_dir_name):
    """
    Main function to remove uninstall distro specific operations.
    :return:
    """

    sys_cfg_file = os.path.join(config.usb_mount, "multibootusb", "syslinux.cfg")
    if not os.path.exists(sys_cfg_file):
        gen.log("syslinux.cfg file not found for updating changes.")
    else:
        gen.log("Updating syslinux.cfg file...")
        string = open(sys_cfg_file).read()
        string = re.sub(r'#start ' + re.escape(uninstall_distro_dir_name)
                        + '.*?' + '#end '
                        + re.escape(uninstall_distro_dir_name)
                        + r'\s*', '', string, flags=re.DOTALL)
        config_file = open(sys_cfg_file, "w")
        config_file.write(string)
        config_file.close() 

def update_grub_cfg_file(uninstall_distro_dir_name):
    """
    Main function to remove uninstall distro name from the grub.cfg file.
    :return:
    """

    grub_cfg_file = os.path.join(config.usb_mount, "multibootusb",
                                 "grub", "grub.cfg")
    if not os.path.exists(grub_cfg_file):
        gen.log("grub.cfg file not found for updating changes.")
    else:
        gen.log("Updating grub.cfg file...")
        string = open(grub_cfg_file).read()
        string = re.sub(r'#start ' + re.escape(uninstall_distro_dir_name)
                        + '.*?' + '#end '
                        + re.escape(uninstall_distro_dir_name)
                        + r'\s*', '', string, flags=re.DOTALL)
        config_file = open(grub_cfg_file, "w")
        config_file.write(string)
        config_file.close() 

def parse(self, response):
        soup = BeautifulSoup(response.content.decode("utf-8", "ignore"), "lxml")
        # image_divs = soup.find_all('script')
        image_divs = soup.find_all(name="script")
        for div in image_divs:
            # txt = div.text
            txt = str(div)
            # if not txt.startswith('AF_initDataCallback'):
            if "AF_initDataCallback" not in txt:
                continue
            if "ds:0" in txt or "ds:1" not in txt:
                continue
            # txt = re.sub(r"^AF_initDataCallback\({.*key: 'ds:(\d)'.+data:function\(\){return (.+)}}\);?$",
            #             "\\2", txt, 0, re.DOTALL)
            # meta = json.loads(txt)
            # data = meta[31][0][12][2]
            # uris = [img[1][3][0] for img in data if img[0] == 1]

            uris = re.findall(r"http.*?\.(?:jpg|png|bmp)", txt)
            return [{"file_url": uri} for uri in uris] 

def scanner(cls):
        if not getattr(cls, '_scanner', None):
            def h(tpe):
                return lambda sc, tk: cls.Token(tpe, tk)

            cls._scanner = re.Scanner([
                (r"(--|//).*?$",               h(cls.LINE_COMMENT)),
                (r"\/\*.+?\*\/",               h(cls.BLOCK_COMMENT)),
                (r'"(?:[^"\\]|\\.)*"',         h(cls.STRING)),
                (r"'(?:[^'\\]|\\.)*'",         h(cls.STRING)),
                (r"\$\$(?:[^\$\\]|\\.)*\$\$",  h(cls.STRING)),
                (r";",                         h(cls.SEMICOLON)),
                (r"\s+",                       h(cls.WHITESPACE)),
                (r".",                         h(cls.OTHER))
            ], re.MULTILINE | re.DOTALL)
        return cls._scanner 

def helper(self, term_instance):
        """
        Called at the start of a WAV file capture.  Calculates the length of the
        file and modifies `self.re_capture` with laser precision.
        """
        data = term_instance.capture
        self.wav_header = struct.unpack(
            '4si4s4sihhiihh4si', self.re_wav_header.match(data).group())
        self.wav_length = self.wav_header[1] + 8
        if not self.sent_message:
            channels = "mono"
            if self.wav_header[6] == 2:
                channels = "stereo"
            if self.wav_length != self.wav_header[12] + 44:
                # Corrupt WAV file
                message = _("WAV File is corrupted: Header data mismatch.")
                term_instance.send_message(message)
                term_instance.cancel_capture = True
            message = _("WAV File: %skHz (%s)" % (self.wav_header[7], channels))
            term_instance.send_message(message)
            self.sent_message = True
        # Update the capture regex with laser precision:
        self.re_capture = re.compile(
            b'(RIFF....WAVE.{%s})' % (self.wav_length-12), re.DOTALL) 

def __init__(self, exception, regexp=None):
        """
        Initialize an _AssertRaisesContextManager instance.

        Parameters
        ----------
        exception : class
            The expected Exception class.
        regexp : str, default None
            The regex to compare against the Exception message.
        """

        self.exception = exception

        if regexp is not None and not hasattr(regexp, "search"):
            regexp = re.compile(regexp, re.DOTALL)

        self.regexp = regexp 

def str_flags_to_int(str_flags):
    flags = 0
    if "i" in str_flags:
        flags |= re.IGNORECASE
    if "l" in str_flags:
        flags |= re.LOCALE
    if "m" in str_flags:
        flags |= re.MULTILINE
    if "s" in str_flags:
        flags |= re.DOTALL
    if "u" in str_flags:
        flags |= re.UNICODE
    if "x" in str_flags:
        flags |= re.VERBOSE

    return flags 

def add_decks(dt: datetime.datetime, competition_id: int, final: Dict[str, int], s: str) -> int:
    # The HTML of this page is so badly malformed that BeautifulSoup cannot really help us with this bit.
    rows = re.findall('<tr style=">(.*?)</tr>', s, re.MULTILINE | re.DOTALL)
    decks_added, ds = 0, []
    matches: List[bs4.element.Tag] = []
    for row in rows:
        cells = BeautifulSoup(row, 'html.parser').find_all('td')
        d = tournament_deck(cells, competition_id, dt, final)
        if d is not None:
            if d.get('id') is None or not match.load_matches_by_deck(d):
                decks_added += 1
                ds.append(d)
                matches += tournament_matches(d)
    add_ids(matches, ds)
    insert_matches_without_dupes(dt, matches)
    guess_archetypes(ds)
    return decks_added 

def medal_winners(s: str) -> Dict[str, int]:
    winners = {}
    # The HTML of this page is so badly malformed that BeautifulSoup cannot really help us with this bit.
    rows = re.findall('<tr style=">(.*?)</tr>', s, re.MULTILINE | re.DOTALL)
    for row in rows:
        player = BeautifulSoup(row, 'html.parser').find_all('td')[2]
        if player.find('img'):
            mtgo_username = aliased(player.a.contents[0])
            img = re.sub(r'styles/Chandra/images/(.*?)\.png', r'\1', player.img['src'])
            if img == WINNER:
                winners[mtgo_username] = 1
            elif img == SECOND:
                winners[mtgo_username] = 2
            elif img == TOP_4:
                winners[mtgo_username] = 3
            elif img == TOP_8:
                winners[mtgo_username] = 5
            elif img == 'verified':
                pass
            else:
                raise InvalidDataException('Unknown player image `{img}`'.format(img=img))
    return winners 

def seasons(season_url = common.args.url):
	seasons = []
	season_data = connection.getURL(season_url)
	try:
		playlist = re.compile('"playlists":\s*(\[.*?\])', re.DOTALL).findall(season_data)[0]
		season_menu = simplejson.loads(playlist)
		for season_item in season_menu:
			seasons.append((season_item['name'],  SITE, 'episodes', FEED % season_item['guid'], -1, -1))
	except:
		try:
			season_tree = BeautifulSoup(season_data, 'html.parser', parse_only = SoupStrainer('div'))
			season_source = season_tree.find('div', id = 'TPVideoPlaylistTaxonomyContainer')['source']
			playlist_url = PLAYLIST % season_source
			playlist_data = connection.getURL(playlist_url)
			playlist_data = playlist_data.replace('$pdk.NBCplayer.ShowPlayerTaxonomy.GetList(', '').replace(');', '')
			season_menu = simplejson.loads(playlist_data)
			for season_item in season_menu['playlistTaxonomy']:
				season_name =  season_item['reference']['name']
				season_url = FEED % season_item['reference']['feed']
				seasons.append((season_name, SITE, 'episodes', season_url, -1, -1))
		except Exception:
			pass
	return seasons 

def play_video(video_url = common.args.url):
	stored_size = 0
	video_data = connection.getURL(video_url)
	video_model = re.compile('model: *(\[.*\]),\s*videoPlayer: _player,', re.DOTALL).findall(video_data)[0]
	video_model = simplejson.loads(video_model)
	try:
		sbitrate = long(addon.getSetting('quality')) * 1000
	except Exception as e:
		print "Exception: ", e
	hbitrate = -1
	print sbitrate
	for item in video_model[0]['flavors']:
		if item['format'] == 'mp4' and item['security_profile'][0] == 'progressive':
			bitrate = item['bitrate']
			if bitrate > hbitrate and bitrate <= sbitrate:
				hbitrate = bitrate
				url = item['url']
	finalurl = url
	xbmcplugin.setResolvedUrl(pluginHandle, True, xbmcgui.ListItem(path = finalurl)) 

def parse(self, response):
        soup = BeautifulSoup(
            response.content.decode('utf-8', 'ignore'), 'lxml')
        image_divs = soup.find_all('script')
        for div in image_divs:
            txt = div.string
            if txt is None or not txt.startswith('AF_initDataCallback'):
                continue
            if 'ds:1' not in txt:
                continue
            txt = re.sub(r"^AF_initDataCallback\({.*key: 'ds:(\d)'.+data:function\(\){return (.+)}}\);?$",
                         "\\2", txt, 0, re.DOTALL)

            meta = json.loads(txt)
            data = meta[31][0][12][2]

            uris = [img[1][3][0] for img in data if img[0] == 1]
            return [{'file_url': uri} for uri in uris] 

def find_existing_bugs(version: str, api_key: str) -> Set[str]:
    search_query_params = {
        "whiteboard": BUG_WHITEBOARD_TAG,
        "include_fields": "description,summary",
    }
    response = requests.get(BUGZILLA_BUG_URL, params=search_query_params,
                            headers=bugzilla_request_header(api_key))
    response.raise_for_status()

    found_bugs = response.json()["bugs"]

    probes_with_bugs = set()
    for bug in found_bugs:
        if re.search(r"release: version (\d+)", bug["description"]).group(1) != version:
            continue
        probes_in_bug = re.search(r"```(.*)```", bug["description"], re.DOTALL).group(1).split()
        for probe_name in probes_in_bug:
            probes_with_bugs.add(probe_name)

    return probes_with_bugs 

def convert(html):
	"""converts an html string to markdown while preserving unsupported markup."""
	bs = BeautifulSoup(html, 'html.parser')
	_markdownify(bs)
	ret = unicode(bs).replace(u'\xa0', ' ')
	ret = re.sub(r'\n{3,}', r'\n\n', ret)
	# ! FIXME: hack
	ret = re.sub(r'&lt;&lt;&lt;FLOATING LINK: (.+)&gt;&gt;&gt;', r'<\1>', ret)
	# ! FIXME: hack
	sp = re.split(r'(&lt;&lt;&lt;BLOCKQUOTE: .*?&gt;&gt;&gt;)', ret, flags=re.DOTALL)
	for i,e in enumerate(sp):
		if e[:len('&lt;&lt;&lt;BLOCKQUOTE:')] == '&lt;&lt;&lt;BLOCKQUOTE:':
			sp[i] = '> ' + e[len('&lt;&lt;&lt;BLOCKQUOTE:') : -len('&gt;&gt;&gt;')]
			sp[i] = sp[i].replace('\n', '\n> ')
	ret = ''.join(sp)
	return ret.strip('\n') 

def search_file(attribute, string):

    if attribute in ['gen', 'gencost', 'bus', 'branch']:
        pattern = r'mpc\.{}\s*=\s*\[[\n]?(?P<data>.*?)[\n]?\];'.format(attribute)
    elif attribute in ['version', 'baseMVA']:
        pattern = r'mpc\.{}\s*=\s*(?P<data>.*?);'.format(attribute)
    elif attribute == 'bus_name':
        pattern = r'mpc\.{}\s*=\s*\{{[\n]?(?P<data>.*?)[\n]?\}};'.format('bus_name')
    else:
        logger.warning('Unable to parse mpc.%s. Please contact the developer.', attribute)
        return None

    match = re.search(pattern, string, re.DOTALL)
    if match is not None:
        return match.groupdict().get('data', None)
    else:
        return match 

def _signed_content(descriptor: Union['stem.descriptor.server_descriptor.RelayDescriptor', 'stem.descriptor.hidden_service.HiddenServiceDescriptorV3']) -> bytes:
    """
    Provides this descriptor's signing constant, appended with the portion of
    the descriptor that's signed.
    """

    import stem.descriptor.server_descriptor

    if isinstance(descriptor, stem.descriptor.server_descriptor.RelayDescriptor):
      prefix = SIG_PREFIX_SERVER_DESC
      regex = b'(.+router-sig-ed25519 )'
    elif isinstance(descriptor, stem.descriptor.hidden_service.HiddenServiceDescriptorV3):
      prefix = SIG_PREFIX_HS_V3
      regex = b'(.+)signature '
    else:
      raise ValueError('BUG: %s type unexpected' % type(descriptor).__name__)

    match = re.search(regex, descriptor.get_bytes(), re.DOTALL)

    if not match:
      raise ValueError('Malformed descriptor missing signature line')

    return prefix + match.group(1) 

def get_sdp_by_name(self, name, media_ports, **params):
        """
        Fetch the SDP content from the database and add missing values.
        """
        logger.debug("Loading sdp with: params = %s, media_ports %s", pprint.pformat(params), pprint.pformat(media_ports))
        ret = self._cur.execute("SELECT sdp FROM sdp WHERE name='?'")
        data = ret.fetchone()

        if data is None:
            # try to fetch the default sdp from the db
            ret = self._cur.execute("SELECT sdp FROM sdp WHERE name='default'")
            data = ret.fetchone()

        if data is None:
            data = (DEFAULT_SDP,)

        sdp = data[0]
        for n,v in media_ports.items():
            if v is None:
                sdp = re.sub("\[" + n +"\].*\[\/" + n + "\]", "", sdp, 0, re.DOTALL)
            else:
                params[n] = v

        sdp = sdp.format(**params)
        return bytes(sdp, "utf-8") 

def search_nzb_url(self):
        """Search for NZB Download URL and return the URL
        :return bool, str: """
        try:
            self.header = self.header.replace('_', ' ')
            res = requests.get(self.search_url.format(quote(self.header, encoding='utf-8')),
                               timeout=REQUESTS_TIMEOUT, headers={'Cookie': 'agreed=true'}, verify=False)
        except requests.exceptions.Timeout:
            print(Col.WARN + ' Timeout' + Col.OFF, flush=True)
            return False, None
        except requests.exceptions.ConnectionError:
            print(Col.WARN + ' Connection Error' + Col.OFF, flush=True)
            return False, None

        m = re.search(self.regex, res.text, re.DOTALL)
        if m is None:
            print(Col.WARN + ' NOT FOUND' + Col.OFF, flush=True)
            return False, None

        self.nzb_url = self.download_url.format(**m.groupdict())

        return True, self.nzb_url 

def assertErrorPage(self, status, message=None, pattern=''):
        """Compare the response body with a built in error page.

        The function will optionally look for the regexp pattern,
        within the exception embedded in the error page."""

        # This will never contain a traceback
        page = cherrypy._cperror.get_error_page(status, message=message)

        # First, test the response body without checking the traceback.
        # Stick a match-all group (.*) in to grab the traceback.
        def esc(text):
            return re.escape(ntob(text))
        epage = re.escape(page)
        epage = epage.replace(
            esc('<pre id="traceback"></pre>'),
            esc('<pre id="traceback">') + b'(.*)' + esc('</pre>'))
        m = re.match(epage, self.body, re.DOTALL)
        if not m:
            self._handlewebError(
                'Error page does not match; expected:\n' + page)
            return

        # Now test the pattern against the traceback
        if pattern is None:
            # Special-case None to mean that there should be *no* traceback.
            if m and m.group(1):
                self._handlewebError('Error page contains traceback')
        else:
            if (m is None) or (
                not re.search(ntob(re.escape(pattern), self.encoding),
                              m.group(1))):
                msg = 'Error page does not contain %s in traceback'
                self._handlewebError(msg % repr(pattern)) 

def get_enumeration_sparql(rq, v, endpoint, auth=None):
    """
    Returns a list of enumerated values for variable 'v' in query 'rq'
    """
    glogger.info('Retrieving enumeration for variable {}'.format(v))
    vcodes = []
    # tpattern_matcher = re.compile(".*(FROM\s+)?(?P<gnames>.*)\s+WHERE.*[\.\{][\n\t\s]*(?P<tpattern>.*\?" + re.escape(v) + ".*?\.).*", flags=re.DOTALL)
    # tpattern_matcher = re.compile(".*?((FROM\s*)(?P<gnames>(\<.*\>)+))?\s*WHERE\s*\{(?P<tpattern>.*)\}.*", flags=re.DOTALL)

    # WHERE is optional too!!
    tpattern_matcher = re.compile(".*?(FROM\s*(?P<gnames>\<.*\>+))?\s*(WHERE\s*)?\{(?P<tpattern>.*)\}.*",
                                  flags=re.DOTALL)

    glogger.debug(rq)
    tp_match = tpattern_matcher.match(rq)
    if tp_match:
        vtpattern = tp_match.group('tpattern')
        gnames = tp_match.group('gnames')
        glogger.debug("Detected graph names: {}".format(gnames))
        glogger.debug("Detected BGP: {}".format(vtpattern))
        glogger.debug("Matched triple pattern with parameter")
        if gnames:
            codes_subquery = re.sub("SELECT.*\{.*\}.*",
                                    "SELECT DISTINCT ?" + v + " FROM " + gnames + " WHERE { " + vtpattern + " }", rq,
                                    flags=re.DOTALL)
        else:
            codes_subquery = re.sub("SELECT.*\{.*\}.*",
                                    "SELECT DISTINCT ?" + v + " WHERE { " + vtpattern + " }", rq,
                                    flags=re.DOTALL)
        glogger.debug("Codes subquery: {}".format(codes_subquery))
        glogger.debug(endpoint)
        codes_json = requests.get(endpoint, params={'query': codes_subquery},
                                  headers={'Accept': static.mimetypes['json'],
                                           'Authorization': 'token {}'.format(static.ACCESS_TOKEN)}, auth=auth).json()
        for code in codes_json['results']['bindings']:
            vcodes.append(list(code.values())[0]["value"])
    else:
        glogger.debug("No match between variable name and query.")

    return vcodes 

def replace_in_multiline_string(pattern: str, substitute: str, text: str) -> str:
    return re.sub(pattern, substitute, text, flags=re.DOTALL) 

def test_anyall(self):
        self.assertEqual(re.match("a.b", "a\nb", re.DOTALL).group(0),
                         "a\nb")
        self.assertEqual(re.match("a.*b", "a\n\nb", re.DOTALL).group(0),
                         "a\n\nb") 

def test_constants(self):
        self.assertEqual(re.I, re.IGNORECASE)
        self.assertEqual(re.L, re.LOCALE)
        self.assertEqual(re.M, re.MULTILINE)
        self.assertEqual(re.S, re.DOTALL)
        self.assertEqual(re.X, re.VERBOSE) 

def testOtherCases(self):
        self.assertSequenceEqual(sre_yield.AllStrings('[aeiou]'), list('aeiou'))
        self.assertEqual(len(sre_yield.AllStrings('1.3', flags=re.DOTALL)), 256)
        v = sre_yield.AllStrings('[^-]3[._]1415', flags=re.DOTALL)
        print(list(v))
        self.assertEqual(len(v), 510)
        self.assertEqual(len(sre_yield.AllStrings('(.|5[6-9]|[6-9][0-9])[a-z].?',
                                               flags=re.DOTALL)),
                          300 * 26 * 257)
        self.assertEqual(len(sre_yield.AllStrings('..', charset='0123456789')),
                          100)
        self.assertEqual(len(sre_yield.AllStrings('0*')), 65536)
        # For really big lists, we can't use the len() function any more
        self.assertEqual(sre_yield.AllStrings('0*').__len__(), 65536)
        self.assertEqual(sre_yield.AllStrings('[01]*').__len__(), 2 ** 65536 - 1) 

def testDotallFlag(self):
        parsed = sre_yield.AllStrings('.', charset='abc\n')
        self.assertEqual(['a', 'b', 'c'], parsed[:])
        parsed = sre_yield.AllStrings('.', charset='abc\n', flags=re.DOTALL)
        self.assertEqual(['a', 'b', 'c', '\n'], parsed[:]) 

def testDotStarCase(self):
        test_size = sre_yield.AllStrings('.*', re.DOTALL).__len__()
        actual_size = 0
        for _ in range(65536):
            actual_size = actual_size * 256 + 1
        self.assertEqual(test_size, actual_size) 

def get(self, getme=None, fromEnd=False):
        if not getme:
            return self
        try:
            getme = int(getme)
            if getme < 0:
                return self[:(-1 * getme)]
            else:
                return [self[getme-1]]
        except IndexError:
            return []
        except ValueError:
            rangeResult = self.rangePattern.search(getme)
            if rangeResult:
                start = rangeResult.group('start') or None
                end = rangeResult.group('start') or None
                if start:
                    start = int(start) - 1
                if end:
                    end = int(end)
                return self[start:end]

            getme = getme.strip()

            if getme.startswith(r'/') and getme.endswith(r'/'):
                finder = re.compile(getme[1:-1], re.DOTALL | re.MULTILINE | re.IGNORECASE)
                def isin(hi):
                    return finder.search(hi)
            else:
                def isin(hi):
                    return (getme.lower() in hi.lowercase)
            return [itm for itm in self if isin(itm)]