Python zlib.adler32() Examples

def adler32(file):
    """
    An Adler-32 checksum is obtained by calculating two 16-bit checksums A and B and concatenating their bits into a 32-bit integer. A is the sum of all bytes in the stream plus one, and B is the sum of the individual values of A from each step.

    :param file: file name
    :returns: Hexified string, padded to 8 values.
    """

    # adler starting value is _not_ 0
    adler = 1

    try:
        with open(file, 'rb') as openFile:
            for line in openFile:
                adler = zlib.adler32(line, adler)
    except Exception as e:
        raise Exception('FATAL - could not get Adler32 checksum of file %s - %s' % (file, e))

    # backflip on 32bit
    if adler < 0:
        adler = adler + 2 ** 32

    return str('%08x' % adler) 

def __init__(self, documents=None, id_range=32000, myhash=zlib.adler32, debug=True):
        """
        By default, keep track of debug statistics and mappings. If you find yourself
        running out of memory (or are sure you don't need the debug info), set
        `debug=False`.
        """
        self.myhash = myhash # hash fnc: string->integer
        self.id_range = id_range # hash range: id = myhash(key) % id_range
        self.debug = debug

        # the following (potentially massive!) dictionaries are only formed if `debug` is True
        self.token2id = {}
        self.id2token = {} # reverse mapping int->set(words)
        self.dfs = {} # token_id -> how many documents this token_id appeared in
        self.dfs_debug = {} # token_string->how many documents this word appeared in

        self.num_docs = 0 # number of documents processed
        self.num_pos = 0 # total number of corpus positions
        self.num_nnz = 0 # total number of non-zeroes in the BOW matrix
        self.allow_update = True

        if documents is not None:
            self.add_documents(documents) 

def __hashlib_init(self):
        """
        Gives the finger print to the corresponding hash library.
        """
        finger = None
        if self.hash_library.lower() == 'sha512':
            finger = hashlib.sha512()
        elif self.hash_library.lower() == 'md5':
            finger = hashlib.md5()
        elif self.hash_library.lower() == 'sha256':
            finger = hashlib.sha256()
        elif self.hash_library.lower() == 'crc32':
            finger = _crc32()
        elif self.hash_library.lower() == 'adler32':
            finger = _adler32()
        else:
            print >> sys.stderr,"ERROR: Unknown hash library!"
            self.exit_flag = False
            sys.exit(1)
        return finger

    ###
    ###  __BUILD_PATHS
    ### 

def __init__(self, documents=None, id_range=32000, myhash=zlib.adler32, debug=True):
        """
        By default, keep track of debug statistics and mappings. If you find yourself
        running out of memory (or are sure you don't need the debug info), set
        `debug=False`.
        """
        self.myhash = myhash # hash fnc: string->integer
        self.id_range = id_range # hash range: id = myhash(key) % id_range
        self.debug = debug

        # the following (potentially massive!) dictionaries are only formed if `debug` is True
        self.token2id = {}
        self.id2token = {} # reverse mapping int->set(words)
        self.dfs = {} # token_id -> how many documents this token_id appeared in
        self.dfs_debug = {} # token_string->how many documents this word appeared in

        self.num_docs = 0 # number of documents processed
        self.num_pos = 0 # total number of corpus positions
        self.num_nnz = 0 # total number of non-zeroes in the BOW matrix
        self.allow_update = True

        if documents is not None:
            self.add_documents(documents) 

def string_to_int_code(string: str) -> int:
    """
    Converts the string to an integer code representation.

    Can be used for a random seed generation from a string

    Parameters
    ----------
    string
        string to convert

    Returns
    -------
    int_code
        integer representation of the string
    """
    int_code = zlib.adler32(bytes(string, 'utf-8')) & 0xffffffff
    return int_code 

def send_rpc_request(sock, req_obj, unknown):
    marsh = marshal.dumps(req_obj)  # python object

    # build out the header
    header =  "\x78\x01\x01" + struct.pack('<h', len(marsh))
    header += chr(unknown) # not sure exactly what this is
    header += "\xff"

    # add the ADLER32 checksum
    checksum = struct.pack('>i', zlib.adler32(marsh))

    post_data = header + marsh + checksum
    message = build_post(post_data)
    try:
        sock.send(message)
        #print("Sent request.")

        resp = sock.recv(1024)

        if resp is None:
            print("Did not receive a response from server.")
    except Exception as e:
        print("Error with request:")
        print(e) 

def send_rpc_request(sock, req_obj, unknown):
    marsh = marshal.dumps(req_obj)  # python object

    # build out the header
    header =  "\x78\x01\x01" + struct.pack('<h', len(marsh))
    header += chr(unknown) # not sure exactly what this is
    header += "\xff"

    # add the ADLER32 checksum
    checksum = struct.pack('>i', zlib.adler32(marsh))

    post_data = header + marsh + checksum
    message = build_post(post_data)
    try:
        sock.send(message)

        resp = sock.recv(1024)

        if resp is None:
            print("Did not receive a response from server.")
    except Exception as e:
        print("Error with request:")
        print(e) 

def build_post(data):
    message = "POST /api HTTP/1.1" + "\r\n"
    message += "Host: 127.0.0.1" + "\r\n"
    message += "Content-Length: " + str(len(data)) + "\r\n"
    message += "X-Drv-Encoding: 1" + "\r\n"
    message += "\r\n"
    message = message + data
    return message

# message format
# header 7 bytes
#   magic bytes? (3 bytes)
#   length of python object (1 byte)
#   no idea (3 bytes)
# python object
# adler32 checksum (4 bytes)

# leaving sock open on purpose 

def testBuild(self):
        d = HashDictionary(self.texts, myhash=zlib.adler32)
        expected =  {5232: 2,
                     5798: 3,
                     10608: 2,
                     12466: 2,
                     12736: 3,
                     15001: 2,
                     18451: 3,
                     23844: 3,
                     28591: 2,
                     29104: 2,
                     31002: 2,
                     31049: 2}

        self.assertEqual(d.dfs, expected)
        expected = {'minors': 15001, 'graph': 18451, 'system': 5798, 'trees': 23844, 'eps': 31049, 'computer': 10608, 'survey': 28591, 'user': 12736, 'human': 31002, 'time': 29104, 'interface': 12466, 'response': 5232}

        for ex in expected:
            self.assertEqual(d.token2id[ex], expected[ex]) 

def __init__(self, documents=None, id_range=32000, myhash=zlib.adler32, debug=True):
        """
        By default, keep track of debug statistics and mappings. If you find yourself
        running out of memory (or are sure you don't need the debug info), set
        `debug=False`.
        """
        self.myhash = myhash # hash fnc: string->integer
        self.id_range = id_range # hash range: id = myhash(key) % id_range
        self.debug = debug

        # the following (potentially massive!) dictionaries are only formed if `debug` is True
        self.token2id = {}
        self.id2token = {} # reverse mapping int->set(words)
        self.dfs = {} # token_id -> how many documents this token_id appeared in
        self.dfs_debug = {} # token_string->how many documents this word appeared in

        self.num_docs = 0 # number of documents processed
        self.num_pos = 0 # total number of corpus positions
        self.num_nnz = 0 # total number of non-zeroes in the BOW matrix
        self.allow_update = True

        if documents is not None:
            self.add_documents(documents) 

def testBuild(self):
        d = HashDictionary(self.texts, myhash=zlib.adler32)
        expected =  {5232: 2,
                     5798: 3,
                     10608: 2,
                     12466: 2,
                     12736: 3,
                     15001: 2,
                     18451: 3,
                     23844: 3,
                     28591: 2,
                     29104: 2,
                     31002: 2,
                     31049: 2}

        self.assertEqual(d.dfs, expected)
        expected = {'minors': 15001, 'graph': 18451, 'system': 5798, 'trees': 23844, 'eps': 31049, 'computer': 10608, 'survey': 28591, 'user': 12736, 'human': 31002, 'time': 29104, 'interface': 12466, 'response': 5232}

        for ex in expected:
            self.assertEqual(d.token2id[ex], expected[ex]) 

def __init__(self, proto: pb.Computation, name: Optional[str] = None):
    """Creates a representation of a fully constructed computation.

    Args:
      proto: An instance of pb.Computation with the computation logic.
      name: An optional string name to associate with this computation, used
        only for debugging purposes. If the name is not specified (None), it is
        autogenerated as a hexadecimal string from the hash of the proto.

    Raises:
      TypeError: if the arguments are of the wrong types.
    """
    py_typecheck.check_type(proto, pb.Computation)
    if name is not None:
      py_typecheck.check_type(name, str)
    super().__init__(type_serialization.deserialize_type(proto.type))
    self._proto = proto
    if name is not None:
      self._name = name
    else:
      self._name = '{:x}'.format(zlib.adler32(self._proto.SerializeToString())) 

def __init__(self, seed):
        if not seed:
            seed = "%.1f" % time.time()

        if hasattr(seed, "encode"):
            seed = seed.encode('ascii')

        # A note on hashfunctions.
        # We don't need cryptographic quality, so we won't use hashlib -
        # that'd be way to slow. The zlib module contains two hash
        # functions. Adler32 is fast, but not very uniform for short
        # strings. Crc32 is slower, but has better bit distribution.
        # So, we use crc32 whenever the hash is converted into an
        # exportable number, but we use adler32 when we're producing
        # intermediate values.
        self.seed = zlib.adler32(seed)
        self.text_seed = seed

        # Default, typically overridden
        self.size = 1024 + 786j 

def verify_ip_hash_cookie(hash_cookie_value):
    """
    根据cookie中的hash判断是否允许用户访问
    在 human_ip_verification 功能中使用
    hash一共14位
    hash(前7位+salt) = 后7位 以此来进行验证
    :type hash_cookie_value: str
    :rtype: bool
    """
    try:
        input_key_hash = hash_cookie_value[:8]
        output_hash = hash_cookie_value[8:]
        calculated_hash = hex(zlib.adler32(
            (input_key_hash + human_ip_verification_answers_hash_str).encode(encoding='utf-8')
        ))[2:]
        if output_hash == calculated_hash:
            return True
        else:
            return False
    except:
        return False 

def generate_ip_verify_hash(input_dict):
    """
    生成一个标示用户身份的hash
    在 human_ip_verification 功能中使用
    hash一共14位
    hash(前7位+salt) = 后7位 以此来进行验证
    :rtype str
    """
    strbuff = human_ip_verification_answers_hash_str
    for key in input_dict:
        strbuff += key + input_dict[key] + str(random.randint(0, 9000000))
    input_key_hash = hex(zlib.adler32(strbuff.encode(encoding='utf-8')))[2:]
    while len(input_key_hash) < 7:
        input_key_hash += '0'
    output_hash = hex(zlib.adler32((input_key_hash + human_ip_verification_answers_hash_str).encode(encoding='utf-8')))[2:]
    while len(output_hash) < 7:
        output_hash += '0'
    return input_key_hash + output_hash 

def get_in_caches(self, s):
        try:
            return self.__caches[ self.ctype ][ zlib.adler32( s ) ]
        except KeyError:
            return self.new_zero() 

def client_post_decrypt(self, buf):
        if self.raw_trans:
            return buf
        self.recv_buf += buf
        out_buf = b''
        while len(self.recv_buf) > 2:
            length = struct.unpack('>H', self.recv_buf[:2])[0]
            if length >= 8192 or length < 7:
                self.raw_trans = True
                self.recv_buf = b''
                raise Exception('client_post_decrypt data error')
            if length > len(self.recv_buf):
                break

            if struct.pack('<I', zlib.adler32(self.recv_buf[:length - 4]) & 0xFFFFFFFF) != self.recv_buf[length - 4:length]:
                self.raw_trans = True
                self.recv_buf = b''
                raise Exception('client_post_decrypt data uncorrect checksum')

            pos = common.ord(self.recv_buf[2]) + 2
            out_buf += self.recv_buf[pos:length - 4]
            self.recv_buf = self.recv_buf[length:]

        if out_buf:
            self.decrypt_packet_num += 1
        return out_buf 

def pack_data(self, buf):
        rnd_data = os.urandom(common.ord(os.urandom(1)[0]) % 16)
        data = common.chr(len(rnd_data) + 1) + rnd_data + buf
        data = struct.pack('>H', len(data) + 6) + data
        adler32 = zlib.adler32(data) & 0xFFFFFFFF
        data += struct.pack('<I', adler32)
        return data 

def pack_data(self, buf):
        data = self.rnd_data(len(buf)) + buf
        data = struct.pack('>H', len(data) + 6) + data
        adler32 = zlib.adler32(data) & 0xFFFFFFFF
        data += struct.pack('<I', adler32)
        return data 

def pack_data(self, buf):
        data = self.rnd_data(len(buf)) + buf
        data_len = len(data) + 8
        crc = binascii.crc32(struct.pack('>H', data_len)) & 0xFFFF
        data = struct.pack('<H', crc) + data
        data = struct.pack('>H', data_len) + data
        adler32 = zlib.adler32(data) & 0xFFFFFFFF
        data += struct.pack('<I', adler32)
        return data 

def _mdx_compress(data, compression_type=2):
	header = (struct.pack(b"<L", compression_type) + 
	         struct.pack(b">L", zlib.adler32(data) & 0xffffffff)) #depending on python version, zlib.adler32 may return a signed number. 
	if compression_type == 0: #no compression
		return header + data
	elif compression_type == 2:
		return header + zlib.compress(data)
	elif compression_type == 1:
		if HAVE_LZO:
			return header + lzo.compress(data)[5:] #python-lzo adds a 5-byte header.
		else:
			raise NotImplementedError()
	else:
		raise ParameterError("Unknown compression type") 

def update(self, block):
        import zlib
        self.crc = zlib.adler32(block, self.crc) 

def client_post_decrypt(self, buf):
        if self.raw_trans:
            return buf
        self.recv_buf += buf
        out_buf = b''
        while len(self.recv_buf) > 4:
            crc = struct.pack('<H', binascii.crc32(self.recv_buf[:2]) & 0xFFFF)
            if crc != self.recv_buf[2:4]:
                raise Exception('client_post_decrypt data uncorrect crc')
            length = struct.unpack('>H', self.recv_buf[:2])[0]
            if length >= 8192 or length < 7:
                self.raw_trans = True
                self.recv_buf = b''
                raise Exception('client_post_decrypt data error')
            if length > len(self.recv_buf):
                break

            if struct.pack('<I', zlib.adler32(self.recv_buf[:length - 4]) & 0xFFFFFFFF) != self.recv_buf[length - 4:length]:
                self.raw_trans = True
                self.recv_buf = b''
                raise Exception('client_post_decrypt data uncorrect checksum')

            pos = common.ord(self.recv_buf[4])
            if pos < 255:
                pos += 4
            else:
                pos = struct.unpack('>H', self.recv_buf[5:7])[0] + 4
            out_buf += self.recv_buf[pos:length - 4]
            self.recv_buf = self.recv_buf[length:]

        if out_buf:
            self.decrypt_packet_num += 1
        return out_buf 

def pack_data(self, buf):
        data = self.rnd_data(len(buf)) + buf
        data_len = len(data) + 8
        crc = binascii.crc32(struct.pack('>H', data_len)) & 0xFFFF
        data = struct.pack('<H', crc) + data
        data = struct.pack('>H', data_len) + data
        adler32 = zlib.adler32(data) & 0xFFFFFFFF
        data += struct.pack('<I', adler32)
        return data 

def _generate_volume_serial_number(device, workspace_id):
    return adler32(f"{device.organization_id}-{device.device_id}-{workspace_id}".encode()) 

def fixChecksum(data):
	hashv = hashlib.sha1(data[32:]).digest()
	data = assign(data, 12, 32, hashv)

	adlrv = struct.pack('I', zlib.adler32(data[12:]) & 0xffffffff)
	return assign(data, 8, 12, adlrv) 

def _populate_data(step: 'projects.ProjectStep') -> STEP_DATA:
    """..."""
    step_data = create_data(step)
    report = step.report

    component = components.get(step)

    includes = step_data.includes.copy()
    includes.extend([include._asdict() for include in component.includes])

    file_writes = step_data.file_writes.copy()
    file_writes.extend(component.files)

    body = step.get_dom()
    checksum = zlib.adler32(body.encode())

    return step_data._replace(
        has_error=bool(step.error),
        body=body,
        body_checksum=checksum,
        data=(
            step_data.data
            .copy()
            .update(report.data.fetch(None))
        ),
        includes=includes,
        file_writes=file_writes
    ) 

def get_spectrogram_hash_op(spectrogram):
  """Calculate hash of the spectrogram."""
  def get_spectrogram_hash(spectrogram):
    # Compute a hash of the spectrogram, save it as an int64.
    # Uses adler because it's fast and will fit into an int (md5 is too large).
    spectrogram_serialized = io.BytesIO()
    np.save(spectrogram_serialized, spectrogram)
    spectrogram_hash = np.int64(zlib.adler32(spectrogram_serialized.getvalue()))
    spectrogram_serialized.close()
    return spectrogram_hash
  spectrogram_hash = tf.py_func(get_spectrogram_hash, [spectrogram], tf.int64,
                                name='get_spectrogram_hash')
  spectrogram_hash.set_shape([])
  return spectrogram_hash 

def _compute_class_hash(openmm_class):
        """Return a numeric hash for the OpenMM class.

        The hash will become part of the OpenMM object serialization,
        so it is important for it consistent across processes in case
        the integrator is sent to a remote worker. The hash() built-in
        function is seeded by the PYTHONHASHSEED environmental variable,
        so we can't use it here.

        We also need to convert to float because some digits may be
        lost in the conversion.
        """
        return float(zlib.adler32(openmm_class.__name__.encode())) 

def _checksum_Adler32(data, crc_init=-1, crc_poly=-1):
    return zlib.adler32(data)