Python hashlib.md5() Examples

def hash(self, state: str) -> str:
        """Generate a hash representing the current sample state.

        :param state: string capturing the current system configuration state

        This function must be overridden to generate a meaningful hash for the current
        set of input samples."""

        newstate = io.BytesIO(state.encode('utf-8'))

        for k in ('input_patterns', 'samples_dir', 'val_dir', 'val_num', 'val_perc',
                  'test_dir', 'test_num', 'test_perc', 'random_seed'):
            newstate.write(str(getattr(self, k)).encode('utf-8'))

        md5 = hashlib.md5()
        md5.update(newstate.getvalue())
        return md5.hexdigest() 

def create_user(conn, username, password):
    try:
        cur = conn.cursor()
        cur.execute("SELECT passwd FROM pg_shadow WHERE usename = %s", [username])
        row = cur.fetchone()
        if row:
            m = hashlib.md5()
            m.update(password + username)
            encrypted_password = "md5" + m.hexdigest()
            if encrypted_password != row[0]:
                cur.execute("ALTER USER " + username + " ENCRYPTED PASSWORD %s SUPERUSER REPLICATION", [password])
        else:
            cur.execute("CREATE USER " + username + " WITH ENCRYPTED PASSWORD %s SUPERUSER REPLICATION", [password])
        conn.commit()
        return True
    except psycopg2.Error as e:
        print(e)
        conn.rollback()
        return False 

def get_file_md5_2(file_path):
    """
    分段读取，获取文件的md5值
    :param file_path:
    :return:
    """
    with open(file_path, 'rb') as file:
        md5_obj = hashlib.md5()
        while True:
            buffer = file.read(8096)
            if not buffer:
                break
            md5_obj.update(buffer)
        hash_code = md5_obj.hexdigest()
    md5 = str(hash_code).lower()
    return md5 

def ingest_data(self, **kwargs):
        """Parse and save design data.

        :param content: String of valid Deckhand YAML

        :returns: a tuple of a status response and a list of parsed objects from drydock_provisioner.objects
        """
        def local_parse():
            return self.parse_docs(kwargs.get('content'))

        if 'content' in kwargs:
            try:
                # Hash the input to use as the cache key. This is not a security
                # related hash, so use cheap and fast MD5
                hv = hashlib.md5(kwargs.get('content', b'')).hexdigest()
                local_cache = cache.get_cache('parsed_docs')
                results = local_cache.get(key=hv, createfunc=local_parse)
                parse_status, models = results
            except Exception as ex:
                self.logger.debug("Error parsing design - hash %s", hv, exc_info=ex)
                raise ex
        else:
            raise ValueError('Missing parameter "content"')

        return parse_status, models 

def add_account(self, account, nickname, password):
        """
        添加客服账号
        详情请参考
        http://mp.weixin.qq.com/wiki/1/70a29afed17f56d537c833f89be979c9.html

        :param account: 完整客服账号，格式为：账号前缀@公众号微信号
        :param nickname: 客服昵称，最长6个汉字或12个英文字符
        :param password: 客服账号登录密码
        :return: 返回的 JSON 数据包
        """
        password = to_binary(password)
        password = hashlib.md5(password).hexdigest()
        return self._post(
            "https://api.weixin.qq.com/customservice/kfaccount/add",
            data={"kf_account": account, "nickname": nickname, "password": password},
        ) 

def update_account(self, account, nickname, password):
        """
        更新客服账号
        详情请参考
        http://mp.weixin.qq.com/wiki/1/70a29afed17f56d537c833f89be979c9.html

        :param account: 完整客服账号，格式为：账号前缀@公众号微信号
        :param nickname: 客服昵称，最长6个汉字或12个英文字符
        :param password: 客服账号登录密码
        :return: 返回的 JSON 数据包
        """
        password = to_binary(password)
        password = hashlib.md5(password).hexdigest()
        return self._post(
            "https://api.weixin.qq.com/customservice/kfaccount/update",
            data={"kf_account": account, "nickname": nickname, "password": password},
        ) 

def verifyChecksumForDir(dirName, settings, type):
    """
        For each file listed in a list, verifies its checksum.
    """

    dirName = os.path.join(settings.getLocalDst(type), dirName)

    urlChecksumFile = settings.getRemoteSrc() + '/' + os.path.basename(dirName) + '.checksum'
    print("Verification of the decompressed directory '%s' started." % dirName)
    try:
        for line in urllib2.urlopen(urlChecksumFile):
            f, checksum = line.split('\t')
            f = os.path.join(dirName, f)
            # if checksum.strip() != hashlib.md5(open(f).read()).hexdigest():
            if checksum.strip() != getChecksumForFile(f):
                raise Exception("File '%s' is corrupted, it has a wrong checksum." % f)
    except Exception as e:
        print("Unable to verify directory: %s" % dirName)
        raise e

    print("Checksum verification completed successfully!") 

def send(self):
        pwhash = md5(self.conf['sms_http_password']).hexdigest()

        params = {
            'username'  : self.conf['sms_http_user'],
            'password'  : pwhash,
            'message'   : self.body,
            'from'      : self.sender,
            'to'        : self.recipient,
        }

        if self.msg:
            dlruri = settings.SERVO_URL + '/api/messages/?id={0}'.format(self.msg.code)
            params['notify_url'] = dlruri

        params = urllib.urlencode(params)
        r = urllib.urlopen(self.URL, params, context=_create_unverified_context()).read()

        if 'ERROR:' in r:
            raise ValueError(self.ERRORS.get(r, _('Unknown error (%s)') % r)) 

def _sign_fields(signable_fields):  # type: (SignableFields) -> str
    """
    Common signing code.
    """
    for (k, v) in signable_fields:
        assert isinstance(k, str), repr(k)
        assert isinstance(v, str), repr(v)

    if sys.version_info < (3,):
        # Python 2 doesn't do IRI encoding.
        text = urlencode([
            (k.encode('utf-8'), v.encode('utf-8'))
            for (k, v) in signable_fields
        ])
    else:
        text = urlencode(signable_fields, encoding='utf-8', errors='strict')

    return md5(text.encode('ascii')).hexdigest()


#: The checkout signature should ignore these leading and trailing whitespace characters.
#:
#: This list is an educated guess based on the PHP trim() function.
#: 

def __init__(self, Nplanets=1e8, **specs):
        
        # bring in inherited Completeness prototype __init__ values
        Completeness.__init__(self, **specs)
        
        # Number of planets to sample
        self.Nplanets = int(Nplanets)
       
        # get path to completeness interpolant stored in a pickled .comp file
        self.filename = self.PlanetPopulation.__class__.__name__ + self.PlanetPhysicalModel.__class__.__name__ + self.__class__.__name__

        # get path to dynamic completeness array in a pickled .dcomp file
        self.dfilename = self.PlanetPopulation.__class__.__name__ + \
                         self.PlanetPhysicalModel.__class__.__name__ +\
                         specs['modules']['OpticalSystem'] + \
                         specs['modules']['StarCatalog'] + \
                         specs['modules']['TargetList']
        atts = list(self.PlanetPopulation.__dict__)
        self.extstr = ''
        for att in sorted(atts, key=str.lower):
            if not callable(getattr(self.PlanetPopulation, att)) and att != 'PlanetPhysicalModel':
                self.extstr += '%s: ' % att + str(getattr(self.PlanetPopulation, att)) + ' '
        ext = hashlib.md5(self.extstr.encode("utf-8")).hexdigest()
        self.filename += ext
        self.filename.replace(" ","") #Remove spaces from string (in the case of prototype use) 

def get_file_meta_data(self, filepath, filename=None, getHash=False):
        """
        helper function to get meta information about a file to include it's path, date modified, size
        :param filepath: path to a file
        :param filename: filename
        :param getHash: whether or not the hash should be computed
        :return: a tuple of format (filename, filepath, filesize, filemodified, md5)
        """
        if filename is None:
            filename = os.path.split(filepath)[1]

        filemodified = time.ctime(os.path.getmtime(filepath))
        filesize = os.path.getsize(filepath)
        md5 = np.nan
        if getHash:
            md5 = self.get_file_hash(filepath)
        return (filename, filepath, filesize, filemodified, md5) 

def create_webinfo_db(self):
        try:
            cursor = self.conn.cursor()
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS webinfo (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                time varchar(255),
                domain varchar(255),
                waf varchar(255) DEFAULT '',
                title varchar(255) DEFAULT '',
                apps varchar(255) DEFAULT '',
                server varchar(255) DEFAULT '',
                address varchar(255) DEFAULT '',
                ipaddr varchar(255) DEFAULT '',
                os varchar(255) DEFAULT '',
                md5 varchar(40) UNIQUE
                )
                """)
        except sqlite3.OperationalError as e:
            pass 

def create_ports(self):
        try:
            cursor = self.conn.cursor()
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS ports (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                time varchar(255),
                ipaddr varchar(255),
                service varchar(255) DEFAULT '',
                port varchar(255) DEFAULT '',
                banner varchar(255) DEFAULT '',
                md5 varchar(40) UNIQUE
                )
                """)
        except Exception as e:
            pass 

def create_urls(self):
        try:
            cursor = self.conn.cursor()
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS urls (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                time varchar(255),
                domain varchar(255) DEFAULT '',
                title varchar(255) DEFAULT '',
                url varchar(255) DEFAULT '',
                contype varchar(255) DEFAULT '',
                rsp_len varchar(255) DEFAULT '',
                rsp_code varchar(255) DEFAULT '',
                md5 varchar(40) UNIQUE
                )
                """)
        except Exception as e:
            logging.exception(e) 

def get_ports(self, ipaddr, ports):
        self.create_ports()
        timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
        for i in ports:
            service = i.get('server')
            port = i.get('port')
            banner = i.get('banner')
            banner = re.sub('<', '', banner)
            banner = re.sub('>', '', banner)
            md5sum = hashlib.md5()
            strings = str(ipaddr) + str(service) + str(port)
            md5sum.update(strings.encode('utf-8'))
            md5 = md5sum.hexdigest()
            values = (timestamp, ipaddr, service, port, banner, md5)
            query = "INSERT OR IGNORE INTO ports (time, ipaddr, service, port, banner,md5) VALUES (?,?,?,?,?,?)"
            self.insert_ports(query, values) 

def content_md5(data):
    """计算data的MD5值，经过Base64编码并返回str类型。

    返回值可以直接作为HTTP Content-Type头部的值
    """
    if isinstance(data, str):
        data = hashlib.md5(data.encode('utf-8'))
    value = base64.b64encode(data.hexdigest().encode('utf-8'))
    return value.decode('utf-8') 

def private_key_file(self):
        if not self.key_dir:
            self.key_dir = '.'

        if not self.private_key:
            return None
        key_name = '.' + md5(self.private_key.encode('utf-8')).hexdigest()
        key_path = os.path.join(self.key_dir, key_name)
        if not os.path.exists(key_path):
            with open(key_path, 'w') as f:
                f.write(self.private_key)
            os.chmod(key_path, 0o400)
        return key_path 

def _gen_sign(self, api, timestamp_ms):
        data = self.app_secret + 'api=' + api + 'appkey=' + self.app_key + 'timestamp=' + \
               str(timestamp_ms) + self.app_secret
        return md5(data.encode('ascii')).hexdigest() 

def _md5(fname):
    hash_md5 = hashlib.md5()
    with open(fname, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_md5.update(chunk)
    return hash_md5.hexdigest() 

def calculate_signature(params, api_key):
    url = format_url(params, api_key)
    logger.debug("Calculate Signature URL: %s", url)
    return to_text(hashlib.md5(url).hexdigest().upper()) 

def __init__(self, key):
        self.key = to_binary(hashlib.md5(to_binary(key)).hexdigest())
        assert len(self.key) == 32 

def _build_event(card_name: str, times: Tuple[datetime, datetime], classroom: str, teacher: str, current_week: int,
                 week_string: str, cid: str) -> Event:
    """
    生成 `Event` 对象

    :param card_name: 课程名
    :param times: 开始和结束时间
    :param classroom: 课程地点
    :param teacher: 任课教师
    :return: `Event` 对象
    """

    event = Event()
    event.add('transp', 'TRANSPARENT')
    summary = card_name
    if classroom != 'None':
        summary = card_name + '@' + classroom
        event.add('location', classroom)

    description = week_string
    if teacher != 'None':
        description += '\n教师：' + teacher
    description += '\n由 EveryClass 每课 (https://everyclass.xyz) 导入'

    event.add('summary', summary)
    event.add('description', description)
    event.add('dtstart', times[0])
    event.add('dtend', times[1])
    event.add('last-modified', datetime.now())

    # 使用"cid-当前周"作为事件的超码
    event_sk = cid + '-' + str(current_week)
    event['uid'] = hashlib.md5(event_sk.encode('utf-8')).hexdigest() + '@everyclass.xyz'
    alarm = Alarm()
    alarm.add('action', 'none')
    alarm.add('trigger', datetime(1980, 1, 1, 3, 5, 0))
    event.add_component(alarm)
    return event 

def md5_hex(s):
    return md5(ntob(s, 'utf-8')).hexdigest() 

def render(self):
        """use JsonHuman library to render a dictionary"""
        jdata = json.dumps(self).replace("\\n", " ")
        m = hashlib.md5()
        m.update(jdata.encode("utf-8"))
        divid = m.hexdigest()
        html = f'<div id="{divid}" style="width:100%;"></div><script>'
        html += f'render_json({{divid: "{divid}", data: {jdata}}});</script>'
        return html 

def demo__custom_identity_verify(identity_dict):
    """
    For CC98 identity verify

    :type identity_dict: dict
    """
    import hashlib
    import requests
    import config

    if 'cc98_username' not in identity_dict or 'cc98_password' not in identity_dict:
        return False

    try:
        pass_md5 = hashlib.md5()
        pass_md5.update(identity_dict['cc98_password'].encode())
        pass_md5 = pass_md5.hexdigest()
        if config.is_use_proxy:
            proxy = config.requests_proxies
        else:
            proxy = None
        r = requests.post('http://www.cc98.org/sign.asp', data={
            'a': 'i',
            'u': identity_dict['cc98_username'],
            'p': pass_md5,
            'userhidden': 2
        }, proxies=proxy)
        if r.text == '9898':
            return True
        else:
            return False
    except:
        return False


# Demo for Twitter 

def get_md5sum(fname, chunk_size=1024):
    """
    Returns the MD5 checksum of a file.

    Args:
        fname (str): Filename
        chunk_size (Optional[int]): Size (in Bytes) of the chunks that should be
            read in at once. Increasing chunk size reduces the number of reads
            required, but increases the memory usage. Defaults to 1024.

    Returns:
        The MD5 checksum of the file, which is a string.
    """

    def iter_chunks(f):
        while True:
            chunk = f.read(chunk_size)
            if not chunk:
                break
            yield chunk

    sig = hashlib.md5()

    with open(fname, 'rb') as f:
        for chunk in iter_chunks(f):
            sig.update(chunk)

        # data = f.read()
        # return hashlib.md5(data).hexdigest()

    return sig.hexdigest() 

def put(self,txt,final=False):
        """Start parsing a job, return id which can be used to retrieve the result"""
        batch_id=hashlib.md5((str(random.random())+txt).encode("utf-8")).hexdigest()
        self.q_in.put((batch_id,txt)) #first job of 1 total
        self.job_counter+=1
        if final:
            self.q_in.put(("FINAL",""))
        return batch_id 

def get_prefix_len(query, min_prefix_len, min_suffix_len, n=0):
    hasher = hashlib.md5()
    hasher.update(query.encode('utf-8'))
    hasher.update(str(n).encode('utf-8'))
    prefix_len = min_prefix_len + int(hasher.hexdigest(), 16) % (len(query) - min_prefix_len - min_suffix_len + 1)
    # if query[prefix_len - 1] == ' ': prefix_len -= 1
    return prefix_len 

def dev_version():
    """
    Returns a hexdigest of all the python files in the module.
    """

    md5_hash = hashlib.md5()
    py_files = sorted(list_files(suffix=".py"))
    if not py_files:
        return ''
    for filename in py_files:
        with open(filename, 'rb') as fobj:
            content = fobj.read()
        md5_hash.update(content)
    return md5_hash.hexdigest() 

def get_md5(fpath):
    assert os.path.isfile(fpath)
    md5o = hashlib.md5()
    with open(fpath, 'rb') as f:
        # read in 1MB chunks
        for chunk in iter(lambda: f.read(1024 * 1024), b''):
            md5o.update(chunk)
    return md5o.hexdigest() 

def getChecksumForFile(f):
    fr = open(f)
    md5 = hashlib.md5()
    while True:
        data = fr.read(BLOCK_SIZE)
        if not data:
            break
        md5.update(data)
    return md5.hexdigest() 

def checkSumWalk(f, root, out):
    if os.path.isfile(f):
        assert root in f
        if not os.path.basename(f).startswith('.'):
            out.write(os.path.sep.join(f[f.find(root.split(os.path.sep)[-1]):].split(os.path.sep)[1:]) +
                      '\t' + getChecksumForFile(f) + '\n')
            # + '\t' + hashlib.md5(open(f).read()).hexdigest() + '\n')
    else:
        for child in os.listdir(f):
            checkSumWalk(os.path.join(f, child), root, out) 

def upload_ss(
    filename, wikifilename, http_host, filedesc,
    statuscallback, errorcallback
):
    """Prepare for server-side upload."""
    statuscallback('Preparing for server-side upload...', -1)

    # Get hash
    md5 = hashlib.md5()
    with open(filename, 'rb') as f:
        while True:
            data = f.read(65536)
            if not data:
                break
            md5.update(data)

    # file name check
    wikifilename = wikifilename.replace('/', '-').replace(' ', '_')
    wikifilename = wikifilename.replace('\r\n', '_')
    wikifilename = wikifilename.replace('\r', '_').replace('\n', '_')

    newfilename = '/srv/v2c/ssu/' + wikifilename
    shutil.move(filename, newfilename)

    with open(newfilename + '.txt', 'w') as filedescfile:
        filedesc = filedesc.replace(
            '[[Category:Uploaded with video2commons]]',
            '[[Category:Uploaded with video2commons/Server-side uploads]]'
        )
        filedescfile.write(filedesc.encode('utf-8'))

    fileurl = 'https://' + http_host + '/' + wikifilename

    raise NeedServerSideUpload(fileurl, md5.hexdigest()) 

def test_super_resolution_example():
    """Test the super resolution example in the example/onnx folder"""
    sys.path.insert(0, os.path.join(CURR_PATH, '../../../../example/onnx/'))
    import super_resolution

    sym, arg_params, aux_params = super_resolution.import_onnx()

    logging.info("Asserted the result of the onnx model conversion")
    output_img_dim = 672
    input_image, img_cb, img_cr = super_resolution.get_test_image()
    result_img = super_resolution.perform_inference(sym, arg_params, aux_params,
                                                    input_image, img_cb, img_cr)

    assert hashlib.md5(result_img.tobytes()).hexdigest() == '0d98393a49b1d9942106a2ed89d1e854'
    assert result_img.size == (output_img_dim, output_img_dim) 

def avatar(self):
        email = self.email.lower()
        hash = md5(email.encode()).hexdigest()
        default = f'https://api.adorable.io/avatars/256/{email}.png'
        params = urlencode({'d': default, 's': 512})
        return f'https://www.gravatar.com/avatar/{hash}?{params}' 

def test_avatar(self):
        email = self.contact.email.lower()
        hash = md5(email.encode()).hexdigest()
        params = urlencode({
            'd': f'https://api.adorable.io/avatars/256/{email}.png',
            's': 512
        })
        expected = f'https://www.gravatar.com/avatar/{hash}?{params}'
        self.assertEqual(expected, self.contact.avatar()) 

def got_data(self, ch, method, properties, body):
		logging.info("Got data")
		newFile_bytes = bytearray(body)
		if newFile_bytes: #only write data when body is not empty
			try:
				newFile = open("%s/%s.zip" % (self.current_alarm_dir, hashlib.md5(newFile_bytes).hexdigest()), "wb")
				newFile.write(newFile_bytes)
				logging.info("Data written")
			except IOError as ie: # File can't be written, e.g. permissions wrong, directory doesn't exist
				logging.exception("Wasn't able to write received data: %s" % ie)
		self.received_data_counter += 1

	# callback for log messages 

def hash(self, text, key="bildad"):
        ''' Return an MD5 hash of the combined text and key.
        '''
        if not text or not key:
            return ""
        m = hashlib.md5()
        m.update((str(text) + key).encode('utf-8'))
        return m.hexdigest() 

def name_style_num(self, text):
        m = hashlib.md5(text.encode('utf-8'))
        return "%d" % (int(m.hexdigest()[:8], 16) & 0x07) 

def md5(data):
    m = hashlib.md5()
    m.update(data)
    return m.hexdigest() 

def _get_file_hash(file_content):
    m = hashlib.md5()
    m.update(file_content)
    return m.digest().hex() 

def send(self):
        """
        Sends SMS through SMS Jazz Gateway
        """
        conf = Configuration.conf()

        if not conf.get('sms_http_sender'):
            raise ValueError(_("SMS sender name not configured"))

        body = self.note.body.encode('utf-8')
        sender = conf.get('sms_http_sender')
        pwhash = md5(conf['sms_http_password']).hexdigest()
        checksum = md5(body + self.recipient.encode('ascii') + pwhash).hexdigest()

        params = {
            'username'  : conf['sms_http_user'],
            'password'  : pwhash,
            'message'   : body,
            'sender'    : sender.encode('ascii', 'replace'),
            'receiver'  : self.recipient,
            'charset'   : 'UTF8',
            'checksum'  : checksum,
        }

        if self.msg:
            dlruri = '/api/messages/?id={0}&status=%status%'.format(self.msg.code)
            dlruri = settings.SERVO_URL + dlruri
            params['dlruri'] = dlruri

        params = urllib.urlencode(params)
        r = urllib.urlopen(self.URL, params, context=_create_unverified_context()).read()

        if not '1:OK' in r:
            raise ValueError(_('Failed to send message to %s') % self.recipient) 

def fingerprint(key):
    """
    Return the fingerprint for an SSH Public Key
    """
    key = base64.b64decode(key.strip().split()[1].encode('ascii'))
    fp_plain = hashlib.md5(key).hexdigest()
    return ':'.join(a + b for a, b in zip(fp_plain[::2], fp_plain[1::2])) 

def MD5(s):
    '''对字符串s进行md5加密，并返回'''
    m = hashlib.md5()
    m.update(s)
    return m.hexdigest() 

def signature(self):
        try:
            from hashlib import md5
        except ImportError:
            from md5 import md5
        try:
            sig = md5()
            if self.start:
                sig.update(self.start.encode('latin-1'))
            if self.prec:
                sig.update("".join(["".join(p) for p in self.prec]).encode('latin-1'))
            if self.tokens:
                sig.update(" ".join(self.tokens).encode('latin-1'))
            for f in self.pfuncs:
                if f[3]:
                    sig.update(f[3].encode('latin-1'))
        except (TypeError,ValueError):
            pass
        return sig.digest()

    # -----------------------------------------------------------------------------
    # validate_modules()
    #
    # This method checks to see if there are duplicated p_rulename() functions
    # in the parser module file.  Without this function, it is really easy for
    # users to make mistakes by cutting and pasting code fragments (and it's a real
    # bugger to try and figure out why the resulting parser doesn't work).  Therefore,
    # we just do a little regular expression pattern matching of def statements
    # to try and detect duplicates.
    # ----------------------------------------------------------------------------- 

def add_etag(self, overwrite: bool = False, weak: bool = False) -> None:
        if overwrite or "etag" not in self.headers:
            self.set_etag(md5((await self.get_data())).hexdigest(), weak) 

def get_file_md5(file_path):
    """
    获取文件的MD5值
    :param file_path:
    :return:
    """
    with open(file_path, 'rb') as file:
        temp_md5 = hashlib.md5()
        temp_md5.update(file.read())
        hash_code = str(temp_md5.hexdigest()).lower()
    return hash_code 

def modify_file_md5(file_path):
    """
    修改文件的md5值
    :param file_path:
    :return:
    """
    with open(file_path, 'a') as file:
        file.write("####&&&&") 

def _get_filename(self, key):
        hash = md5(key.encode("utf8")).hexdigest()
        return os.path.join(self._path, hash) 

def create_skill_conversation_id(  # pylint: disable=W0221
        self, conversation_reference: ConversationReference
    ) -> str:
        cr_json = json.dumps(conversation_reference.serialize())

        key = hashlib.md5(
            f"{conversation_reference.conversation.id}{conversation_reference.service_url}".encode()
        ).hexdigest()

        if key not in self._conversation_refs:
            self._conversation_refs[key] = cr_json

        return key 

def __init__(self):
        # generate files
        self._generate_addons_file()
        self._generate_md5_file()
        # notify user
        print('Finished updating addons xml and md5 files') 

def _generate_md5_file(self):
        # create a new md5 hash
        with open('addons.xml', 'r', encoding='UTF-8') as fo:
            m = hashlib.md5(fo.read().encode('UTF-8')).hexdigest()

        # save file
        try:
            self._save_file(m.encode('UTF-8'), file='addons.xml.md5')
        except:
            # oops
            print('An error occurred creating addons.xml.md5 file!')
            print_exc() 

def validate_file(file_obj: Any, hash_value: str, hash_type: str = "sha256") -> bool:
    """Validate a given file object with its hash.

    Args:
        file_obj: File object to read from.
        hash_value (str): Hash for url.
        hash_type (str, optional): Hash type, among "sha256" and "md5" (Default: ``"sha256"``).

    Returns:
        bool: return True if its a valid file, else False.
    """

    if hash_type == "sha256":
        hash_func = hashlib.sha256()
    elif hash_type == "md5":
        hash_func = hashlib.md5()
    else:
        raise ValueError

    while True:
        # Read by chunk to avoid filling memory
        chunk = file_obj.read(1024 ** 2)
        if not chunk:
            break
        hash_func.update(chunk)

    return hash_func.hexdigest() == hash_value 

def test_update_with_no_id(self):
        #creare new place, get the generated ID for it
        new_place = json.loads('{"relationships": [], "admin": [], "updated": "2006-01-15T01:00:00+01:00", \
        "name": "simple place name", "geometry": {"type": "Point", "coordinates": [-114.43359375, 44.033203125]}, \
        "alternate" : [{"lang":"en","type":"preferred","name":"Vokey alt"}], "is_primary": true, \
        "uris": ["example.com/place#newplace123"], "feature_code": "BLDG", "centroid": [-114.43359375, 44.033203125], \
        "timeframe": {"end_range": 0,"start": "1800-01-01","end": "1900-01-01","start_range": 0 }}')
        import hashlib
        new_place_id = hashlib.md5("example.com/place#newplace123").hexdigest()[:16]
        self.conn.index("gaz-test-index", "place", new_place, id=new_place_id, metadata={"user_created": "test program"})
        self.conn.refresh(["gaz-test-index"])
        
        place = Place.objects.get(new_place_id)
        self.assertEqual(place.name,"simple place name")
        self.assertEqual(place.feature_code,"BLDG")
        self.assertEqual(place.alternate[0]["name"], "Vokey alt")
        
        self.batch_import_update_no_id.start_import = True
        self.batch_import_update_no_id.save()

        self.conn.refresh(["gaz-test-index"])
        place = Place.objects.get(new_place_id)
        self.assertEqual(place.name,"changing name and fcode with no id")
        self.assertEqual(place.feature_code,"DAM")
        self.assertEqual(place.alternate, [])  #alternate names are cleared
        self.assertEqual(place.centroid, [-114.43359375, 44.033203125]) # centroid not overwritten 

def _id(uri):
    return hashlib.md5(uri).hexdigest()[:16] 

def get_trainer(name):
    """return the unique id for a trainer, determined by the md5 sum
    """
    name = name.lower()
    return int(hashlib.md5(name.encode("utf-8")).hexdigest(), 16) % 10 ** 8 

def getFileHashes(infile):
    t = tarfile.open(infile)
    files = list()
    links = list()
    for f in t.getmembers():
        if f.isfile():
            mime = get_type (t, f)
            # we use f.name[1:] to get rid of the . at the beginning of the path
            files.append((f.name[1:], hashlib.md5(t.extractfile(f).read()).hexdigest(),
                          f.uid, f.gid, f.mode, mime, get_priority (f, mime)))
        elif f.issym():
            links.append((f.name[1:], f.linkpath))
    shutil.rmtree ('/tmp/tar2db_evaluate')
    return (files, links) 

def io_md5(target):
        """
        Performs MD5 with a block size of 64kb.
        """
        blocksize = 65536
        hasher = hashlib.md5()

        with open(target, 'rb') as ifp:
            buf = ifp.read(blocksize)
            while buf:
                hasher.update(buf)
                buf = ifp.read(blocksize)
            return hasher.hexdigest() 

def send_command(self, command, result_callback = None, *callback_parameters):
        entropy = str(random()) + str(time())
        cookie = md5(entropy.encode()).hexdigest()
        next_retr = self.delay_flt.lastval * 4.0
        exp_time = 3.0
        if isinstance(command, Rtp_proxy_cmd):
            if command.type == 'I':
                exp_time = 10.0
            if command.type == 'G':
                exp_time = 1.0
            nretr = command.nretr
            command = str(command)
        else:
            if command.startswith('I'):
                exp_time = 10.0
            elif command.startswith('G'):
                exp_time = 1.0
            nretr = None
        if nretr == None:
            nretr = getnretrans(next_retr, exp_time)
        command = '%s %s' % (cookie, command)
        timer = Timeout(self.retransmit, next_retr, 1, cookie)
        preq = Rtp_proxy_pending_req(next_retr, nretr - 1, timer, command, \
          result_callback, callback_parameters)
        self.worker.send_to(command, self.address)
        self.pending_requests[cookie] = preq 

def file_md5_checksum(fname):
    hash_md5 = hashlib.md5()
    with open(fname, 'rb') as f:
        hash_md5.update(f.read())
    return hash_md5.hexdigest()