Python re.DOTALL Examples

def ssh_edit_file(adress, user, passw, remotefile, regex, replace):
        client = paramiko.SSHClient()
        client.load_system_host_keys()
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        trans = paramiko.Transport((adress, 22))
        trans.connect(username=user, password=passw)
        sftp = paramiko.SFTPClient.from_transport(trans)
        f_in = sftp.file(remotefile, "r")
        c_in = f_in.read()
        pattern = re.compile(regex, re.MULTILINE | re.DOTALL)
        c_out = pattern.sub(replace, c_in)
        f_out = sftp.file(remotefile, "w")
        f_out.write(c_out)
        f_in.close()
        f_out.close()
        sftp.close()
        trans.close() 

def render_re(regex):
  """Renders a repr()-style value for a compiled regular expression."""
  actual_flags = []
  if regex.flags:
    flags = [
      (re.IGNORECASE, 'IGNORECASE'),
      (re.LOCALE, 'LOCALE'),
      (re.UNICODE, 'UNICODE'),
      (re.MULTILINE, 'MULTILINE'),
      (re.DOTALL, 'DOTALL'),
      (re.VERBOSE, 'VERBOSE'),
    ]
    for val, name in flags:
      if regex.flags & val:
        actual_flags.append(name)
  if actual_flags:
    return 're.compile(%r, %s)' % (regex.pattern, '|'.join(actual_flags))
  else:
    return 're.compile(%r)' % regex.pattern 

def show_system_resources(self):
        self.xapi.op(cmd="show system resources", cmd_xml=True)
        result = self.xapi.xml_root()
        regex = re.compile(r"load average: ([\d.]+).* ([\d.]+)%id.*Mem:.*?([\d.]+)k total.*?([\d]+)k free", re.DOTALL)
        match = regex.search(result)
        if match:
            """
            return cpu, mem_free, load
            """
            return {
                'load': Decimal(match.group(1)),
                'cpu': 100 - Decimal(match.group(2)),
                'mem_total': int(match.group(3)),
                'mem_free': int(match.group(4)),
            }
        else:
            raise err.PanDeviceError("Problem parsing show system resources",
                                     pan_device=self) 

def search_file(attribute, string):

    if attribute in ['gen', 'gencost', 'bus', 'branch']:
        pattern = r'mpc\.{}\s*=\s*\[[\n]?(?P<data>.*?)[\n]?\];'.format(attribute)
    elif attribute in ['version', 'baseMVA']:
        pattern = r'mpc\.{}\s*=\s*(?P<data>.*?);'.format(attribute)
    elif attribute == 'bus_name':
        pattern = r'mpc\.{}\s*=\s*\{{[\n]?(?P<data>.*?)[\n]?\}};'.format('bus_name')
    else:
        logger.warning('Unable to parse mpc.%s. Please contact the developer.', attribute)
        return None

    match = re.search(pattern, string, re.DOTALL)
    if match is not None:
        return match.groupdict().get('data', None)
    else:
        return match 

def search_nzb_url(self):
        """Search for NZB Download URL and return the URL
        :return bool, str: """
        try:
            self.header = self.header.replace('_', ' ')
            res = requests.get(self.search_url.format(quote(self.header, encoding='utf-8')),
                               timeout=REQUESTS_TIMEOUT, headers={'Cookie': 'agreed=true'}, verify=False)
        except requests.exceptions.Timeout:
            print(Col.WARN + ' Timeout' + Col.OFF, flush=True)
            return False, None
        except requests.exceptions.ConnectionError:
            print(Col.WARN + ' Connection Error' + Col.OFF, flush=True)
            return False, None

        m = re.search(self.regex, res.text, re.DOTALL)
        if m is None:
            print(Col.WARN + ' NOT FOUND' + Col.OFF, flush=True)
            return False, None

        self.nzb_url = self.download_url.format(**m.groupdict())

        return True, self.nzb_url 

def find_existing_bugs(version: str, api_key: str) -> Set[str]:
    search_query_params = {
        "whiteboard": BUG_WHITEBOARD_TAG,
        "include_fields": "description,summary",
    }
    response = requests.get(BUGZILLA_BUG_URL, params=search_query_params,
                            headers=bugzilla_request_header(api_key))
    response.raise_for_status()

    found_bugs = response.json()["bugs"]

    probes_with_bugs = set()
    for bug in found_bugs:
        if re.search(r"release: version (\d+)", bug["description"]).group(1) != version:
            continue
        probes_in_bug = re.search(r"```(.*)```", bug["description"], re.DOTALL).group(1).split()
        for probe_name in probes_in_bug:
            probes_with_bugs.add(probe_name)

    return probes_with_bugs 

def play_video(video_url = common.args.url):
	stored_size = 0
	video_data = connection.getURL(video_url)
	video_model = re.compile('model: *(\[.*\]),\s*videoPlayer: _player,', re.DOTALL).findall(video_data)[0]
	video_model = simplejson.loads(video_model)
	try:
		sbitrate = long(addon.getSetting('quality')) * 1000
	except Exception as e:
		print "Exception: ", e
	hbitrate = -1
	print sbitrate
	for item in video_model[0]['flavors']:
		if item['format'] == 'mp4' and item['security_profile'][0] == 'progressive':
			bitrate = item['bitrate']
			if bitrate > hbitrate and bitrate <= sbitrate:
				hbitrate = bitrate
				url = item['url']
	finalurl = url
	xbmcplugin.setResolvedUrl(pluginHandle, True, xbmcgui.ListItem(path = finalurl)) 

def parse(self, response):
        soup = BeautifulSoup(
            response.content.decode('utf-8', 'ignore'), 'lxml')
        image_divs = soup.find_all('script')
        for div in image_divs:
            txt = div.string
            if txt is None or not txt.startswith('AF_initDataCallback'):
                continue
            if 'ds:1' not in txt:
                continue
            txt = re.sub(r"^AF_initDataCallback\({.*key: 'ds:(\d)'.+data:function\(\){return (.+)}}\);?$",
                         "\\2", txt, 0, re.DOTALL)

            meta = json.loads(txt)
            data = meta[31][0][12][2]

            uris = [img[1][3][0] for img in data if img[0] == 1]
            return [{'file_url': uri} for uri in uris] 

def update_grub_cfg_file(uninstall_distro_dir_name):
    """
    Main function to remove uninstall distro name from the grub.cfg file.
    :return:
    """

    grub_cfg_file = os.path.join(config.usb_mount, "multibootusb",
                                 "grub", "grub.cfg")
    if not os.path.exists(grub_cfg_file):
        gen.log("grub.cfg file not found for updating changes.")
    else:
        gen.log("Updating grub.cfg file...")
        string = open(grub_cfg_file).read()
        string = re.sub(r'#start ' + re.escape(uninstall_distro_dir_name)
                        + '.*?' + '#end '
                        + re.escape(uninstall_distro_dir_name)
                        + r'\s*', '', string, flags=re.DOTALL)
        config_file = open(grub_cfg_file, "w")
        config_file.write(string)
        config_file.close() 

def seasons(season_url = common.args.url):
	seasons = []
	season_data = connection.getURL(season_url)
	try:
		playlist = re.compile('"playlists":\s*(\[.*?\])', re.DOTALL).findall(season_data)[0]
		season_menu = simplejson.loads(playlist)
		for season_item in season_menu:
			seasons.append((season_item['name'],  SITE, 'episodes', FEED % season_item['guid'], -1, -1))
	except:
		try:
			season_tree = BeautifulSoup(season_data, 'html.parser', parse_only = SoupStrainer('div'))
			season_source = season_tree.find('div', id = 'TPVideoPlaylistTaxonomyContainer')['source']
			playlist_url = PLAYLIST % season_source
			playlist_data = connection.getURL(playlist_url)
			playlist_data = playlist_data.replace('$pdk.NBCplayer.ShowPlayerTaxonomy.GetList(', '').replace(');', '')
			season_menu = simplejson.loads(playlist_data)
			for season_item in season_menu['playlistTaxonomy']:
				season_name =  season_item['reference']['name']
				season_url = FEED % season_item['reference']['feed']
				seasons.append((season_name, SITE, 'episodes', season_url, -1, -1))
		except Exception:
			pass
	return seasons 

def update_sys_cfg_file(uninstall_distro_dir_name):
    """
    Main function to remove uninstall distro specific operations.
    :return:
    """

    sys_cfg_file = os.path.join(config.usb_mount, "multibootusb", "syslinux.cfg")
    if not os.path.exists(sys_cfg_file):
        gen.log("syslinux.cfg file not found for updating changes.")
    else:
        gen.log("Updating syslinux.cfg file...")
        string = open(sys_cfg_file).read()
        string = re.sub(r'#start ' + re.escape(uninstall_distro_dir_name)
                        + '.*?' + '#end '
                        + re.escape(uninstall_distro_dir_name)
                        + r'\s*', '', string, flags=re.DOTALL)
        config_file = open(sys_cfg_file, "w")
        config_file.write(string)
        config_file.close() 

def get_sdp_by_name(self, name, media_ports, **params):
        """
        Fetch the SDP content from the database and add missing values.
        """
        logger.debug("Loading sdp with: params = %s, media_ports %s", pprint.pformat(params), pprint.pformat(media_ports))
        ret = self._cur.execute("SELECT sdp FROM sdp WHERE name='?'")
        data = ret.fetchone()

        if data is None:
            # try to fetch the default sdp from the db
            ret = self._cur.execute("SELECT sdp FROM sdp WHERE name='default'")
            data = ret.fetchone()

        if data is None:
            data = (DEFAULT_SDP,)

        sdp = data[0]
        for n,v in media_ports.items():
            if v is None:
                sdp = re.sub("\[" + n +"\].*\[\/" + n + "\]", "", sdp, 0, re.DOTALL)
            else:
                params[n] = v

        sdp = sdp.format(**params)
        return bytes(sdp, "utf-8") 

def add_decks(dt: datetime.datetime, competition_id: int, final: Dict[str, int], s: str) -> int:
    # The HTML of this page is so badly malformed that BeautifulSoup cannot really help us with this bit.
    rows = re.findall('<tr style=">(.*?)</tr>', s, re.MULTILINE | re.DOTALL)
    decks_added, ds = 0, []
    matches: List[bs4.element.Tag] = []
    for row in rows:
        cells = BeautifulSoup(row, 'html.parser').find_all('td')
        d = tournament_deck(cells, competition_id, dt, final)
        if d is not None:
            if d.get('id') is None or not match.load_matches_by_deck(d):
                decks_added += 1
                ds.append(d)
                matches += tournament_matches(d)
    add_ids(matches, ds)
    insert_matches_without_dupes(dt, matches)
    guess_archetypes(ds)
    return decks_added 

def medal_winners(s: str) -> Dict[str, int]:
    winners = {}
    # The HTML of this page is so badly malformed that BeautifulSoup cannot really help us with this bit.
    rows = re.findall('<tr style=">(.*?)</tr>', s, re.MULTILINE | re.DOTALL)
    for row in rows:
        player = BeautifulSoup(row, 'html.parser').find_all('td')[2]
        if player.find('img'):
            mtgo_username = aliased(player.a.contents[0])
            img = re.sub(r'styles/Chandra/images/(.*?)\.png', r'\1', player.img['src'])
            if img == WINNER:
                winners[mtgo_username] = 1
            elif img == SECOND:
                winners[mtgo_username] = 2
            elif img == TOP_4:
                winners[mtgo_username] = 3
            elif img == TOP_8:
                winners[mtgo_username] = 5
            elif img == 'verified':
                pass
            else:
                raise InvalidDataException('Unknown player image `{img}`'.format(img=img))
    return winners 

def parse(self, response):
        soup = BeautifulSoup(response.content.decode("utf-8", "ignore"), "lxml")
        # image_divs = soup.find_all('script')
        image_divs = soup.find_all(name="script")
        for div in image_divs:
            # txt = div.text
            txt = str(div)
            # if not txt.startswith('AF_initDataCallback'):
            if "AF_initDataCallback" not in txt:
                continue
            if "ds:0" in txt or "ds:1" not in txt:
                continue
            # txt = re.sub(r"^AF_initDataCallback\({.*key: 'ds:(\d)'.+data:function\(\){return (.+)}}\);?$",
            #             "\\2", txt, 0, re.DOTALL)
            # meta = json.loads(txt)
            # data = meta[31][0][12][2]
            # uris = [img[1][3][0] for img in data if img[0] == 1]

            uris = re.findall(r"http.*?\.(?:jpg|png|bmp)", txt)
            return [{"file_url": uri} for uri in uris] 

def scanner(cls):
        if not getattr(cls, '_scanner', None):
            def h(tpe):
                return lambda sc, tk: cls.Token(tpe, tk)

            cls._scanner = re.Scanner([
                (r"(--|//).*?$",               h(cls.LINE_COMMENT)),
                (r"\/\*.+?\*\/",               h(cls.BLOCK_COMMENT)),
                (r'"(?:[^"\\]|\\.)*"',         h(cls.STRING)),
                (r"'(?:[^'\\]|\\.)*'",         h(cls.STRING)),
                (r"\$\$(?:[^\$\\]|\\.)*\$\$",  h(cls.STRING)),
                (r";",                         h(cls.SEMICOLON)),
                (r"\s+",                       h(cls.WHITESPACE)),
                (r".",                         h(cls.OTHER))
            ], re.MULTILINE | re.DOTALL)
        return cls._scanner 

def str_flags_to_int(str_flags):
    flags = 0
    if "i" in str_flags:
        flags |= re.IGNORECASE
    if "l" in str_flags:
        flags |= re.LOCALE
    if "m" in str_flags:
        flags |= re.MULTILINE
    if "s" in str_flags:
        flags |= re.DOTALL
    if "u" in str_flags:
        flags |= re.UNICODE
    if "x" in str_flags:
        flags |= re.VERBOSE

    return flags 

def __init__(self, exception, regexp=None):
        """
        Initialize an _AssertRaisesContextManager instance.

        Parameters
        ----------
        exception : class
            The expected Exception class.
        regexp : str, default None
            The regex to compare against the Exception message.
        """

        self.exception = exception

        if regexp is not None and not hasattr(regexp, "search"):
            regexp = re.compile(regexp, re.DOTALL)

        self.regexp = regexp 

def convert(html):
	"""converts an html string to markdown while preserving unsupported markup."""
	bs = BeautifulSoup(html, 'html.parser')
	_markdownify(bs)
	ret = unicode(bs).replace(u'\xa0', ' ')
	ret = re.sub(r'\n{3,}', r'\n\n', ret)
	# ! FIXME: hack
	ret = re.sub(r'&lt;&lt;&lt;FLOATING LINK: (.+)&gt;&gt;&gt;', r'<\1>', ret)
	# ! FIXME: hack
	sp = re.split(r'(&lt;&lt;&lt;BLOCKQUOTE: .*?&gt;&gt;&gt;)', ret, flags=re.DOTALL)
	for i,e in enumerate(sp):
		if e[:len('&lt;&lt;&lt;BLOCKQUOTE:')] == '&lt;&lt;&lt;BLOCKQUOTE:':
			sp[i] = '> ' + e[len('&lt;&lt;&lt;BLOCKQUOTE:') : -len('&gt;&gt;&gt;')]
			sp[i] = sp[i].replace('\n', '\n> ')
	ret = ''.join(sp)
	return ret.strip('\n') 

def _signed_content(descriptor: Union['stem.descriptor.server_descriptor.RelayDescriptor', 'stem.descriptor.hidden_service.HiddenServiceDescriptorV3']) -> bytes:
    """
    Provides this descriptor's signing constant, appended with the portion of
    the descriptor that's signed.
    """

    import stem.descriptor.server_descriptor

    if isinstance(descriptor, stem.descriptor.server_descriptor.RelayDescriptor):
      prefix = SIG_PREFIX_SERVER_DESC
      regex = b'(.+router-sig-ed25519 )'
    elif isinstance(descriptor, stem.descriptor.hidden_service.HiddenServiceDescriptorV3):
      prefix = SIG_PREFIX_HS_V3
      regex = b'(.+)signature '
    else:
      raise ValueError('BUG: %s type unexpected' % type(descriptor).__name__)

    match = re.search(regex, descriptor.get_bytes(), re.DOTALL)

    if not match:
      raise ValueError('Malformed descriptor missing signature line')

    return prefix + match.group(1) 

def helper(self, term_instance):
        """
        Called at the start of a WAV file capture.  Calculates the length of the
        file and modifies `self.re_capture` with laser precision.
        """
        data = term_instance.capture
        self.wav_header = struct.unpack(
            '4si4s4sihhiihh4si', self.re_wav_header.match(data).group())
        self.wav_length = self.wav_header[1] + 8
        if not self.sent_message:
            channels = "mono"
            if self.wav_header[6] == 2:
                channels = "stereo"
            if self.wav_length != self.wav_header[12] + 44:
                # Corrupt WAV file
                message = _("WAV File is corrupted: Header data mismatch.")
                term_instance.send_message(message)
                term_instance.cancel_capture = True
            message = _("WAV File: %skHz (%s)" % (self.wav_header[7], channels))
            term_instance.send_message(message)
            self.sent_message = True
        # Update the capture regex with laser precision:
        self.re_capture = re.compile(
            b'(RIFF....WAVE.{%s})' % (self.wav_length-12), re.DOTALL) 

def remove_module(v: str, module: str) -> str:
        """
        Remove the given module from the given Verilog source file, if it exists.

        :param v: Verilog source code
        :param module: Module to remove
        :return: Verilog with given module definition removed, if it exists
        """
        if not VerilogUtils.contains_module(v, module):
            # Don't risk touching the source if we don't think the module exists.
            return v

        regex = "(module\s+" + re.escape(module) + ".+?endmodule)"
        return re.sub(regex, "", v, flags=re.DOTALL) 

def find_attributes(string):
    pattern = 'mpc\.(?P<attribute>.*?)\s*=\s*'
    return re.findall(pattern, string, re.DOTALL) 

def replace_app_props(self, encoded_string):
        if encoded_string:
            decoded_test = self.decode_bytes(encoded_string)
            for k, v in self.get_config().items():
                if k.startswith('app.'):
                    re_variable = re.compile(r'#{(%s.*?)}' % k, flags=re.DOTALL)
                    decoded_test = re.sub(re_variable, str(v).strip(), decoded_test)
            return self.encode_string(decoded_test) 

def remove_links_missing_facts(links):
        """
        Remove any links that did not have facts encoded into command

        :param links:
        :return: updated list of links
        """
        links[:] = [l for l in links if
                    not re.findall(r'#{(.*?)}', b64decode(l.command).decode('utf-8'), flags=re.DOTALL)]
        return links 

def _build_single_test_variant(copy_test, combo, executor):
        """
        Replace all variables with facts from the combo to build a single test variant
        """
        score, used = 0, list()
        for var in combo:
            score += (score + var.score)
            used.append(var)
            re_variable = re.compile(r'#{(%s.*?)}' % var.trait, flags=re.DOTALL)
            copy_test = re.sub(re_variable, str(var.escaped(executor)).strip().encode('unicode-escape').decode('utf-8'), copy_test)
        return copy_test, score, used 

def load_json(path):
    content = codecs.open(path, 'r', 'utf-8').read()
    content = re.sub(re.compile("/\*.*?\*/", re.DOTALL) , "", content)
    content = re.sub(re.compile("[\s|\n]//.*?\n" ) , "", content)
    return json.loads(content) 

def __init__(self):
        super(CodeBlock, self).__init__('pre', ['{{{', '}}}'])
        self.regexp = re.compile(self.re_string(), re.DOTALL + re.MULTILINE)
        self.regexp2 = re.compile(self.re_string2(), re.MULTILINE) 

def __init__(self):
        self.log = self.add_service('learning_svc', self)
        self.model = set()
        self.parsers = self.add_parsers('app/learning')
        self.re_variable = re.compile(r'#{(.*?)}', flags=re.DOTALL)
        self.log.debug('Loaded %d parsers' % len(self.parsers)) 

def format(self, record):
        if record.name == "py.warnings":
            # trim the ugly warnings.warn message
            match = re.search(
                r"Warning:\s*(.*?)\s*warnings.warn\(", record.args[0], re.DOTALL)
            record.args = (match.group(1),)
            self._style = logging.PercentStyle("WARNING: %(message)s")
        else:
            if record.levelno == logging.WARNING:
                self._style = logging.PercentStyle("%(message)s")
            else:
                self._style = logging.PercentStyle("%(levelname)s: %(message)s")
        return super().format(record)