Python hashlib.new() Examples

def seeded_range(seed, start, stop=None, step=1, extra=None):
    """
    A filter to produce deterministic random numbers.

    Produce a random item from range(start, stop[, step]), use the value and
    optional ``extra`` value to set the seed for the random number generator.

    Basic usage::

        ansible_fqdn|seeded_range(60)

        "hello"|seeded_range(1, 10, extra="world")
    """
    hashed_seed = new_hash('sha1')
    hashed_seed.update(seed)

    if extra is not None:
        hashed_seed.update(extra)

    hashed_seed = hashed_seed.digest()

    # We rely on randrange's interpretation of parameters
    return Random(hashed_seed).randrange(start, stop, step) 

def modifyIfStmt(self, node, *args, **kwargs):
        ''' Convert if-else if-else statements into just if-else statements by nesting them inside each other '''

        # modify the if condition and the statements in the if-body
        if_condition, stmts = node.condition_stmts_tuples[0]
        if_condition, stmts = (self.modify(if_condition, *args, **kwargs),
                               flatten([self.modify(s, *args, **kwargs) for s in stmts]))
        node.condition_stmts_tuples[0] = (if_condition, stmts)

        # if if-else statement (i.e. no else-if part)
        if len(node.condition_stmts_tuples) == 2 and node.condition_stmts_tuples[1][0] is None:
            stmts = node.condition_stmts_tuples[1][1]
            stmts = flatten([self.modify(s, *args, **kwargs) for s in stmts])
            node.condition_stmts_tuples[1] = (None, stmts)

        # else if there is any "else if" part
        elif len(node.condition_stmts_tuples) > 1 and node.condition_stmts_tuples[1][0] is not None:
            else_if_condition, else_if_stmts = node.condition_stmts_tuples[1]
            # create a new if-statement consisting of the that 'else if' and all the following 'else if'/'else' clauses, and modify this recursively
            new_if_stmt = self.modifyIfStmt(ksp_ast.IfStmt(node.condition_stmts_tuples[1][0].lexinfo, node.condition_stmts_tuples[1:]), *args, **kwargs)[0]
            # the new contents of the else part will be the if-statement just created
            node.condition_stmts_tuples = [node.condition_stmts_tuples[0],
                                           (None, [new_if_stmt])]

        return [node] 

def getHBootKey(self):
        LOG.debug('Calculating HashedBootKey from SAM')
        QWERTY = "!@#$%^&*()qwertyUIOPAzxcvbnmQQQQQQQQQQQQ)(*@&%\0"
        DIGITS = "0123456789012345678901234567890123456789\0"

        F = self.getValue(ntpath.join('SAM\Domains\Account','F'))[1]

        domainData = DOMAIN_ACCOUNT_F(F)

        rc4Key = self.MD5(domainData['Key0']['Salt'] + QWERTY + self.__bootKey + DIGITS)

        rc4 = ARC4.new(rc4Key)
        self.__hashedBootKey = rc4.encrypt(domainData['Key0']['Key']+domainData['Key0']['CheckSum'])

        # Verify key with checksum
        checkSum = self.MD5( self.__hashedBootKey[:16] + DIGITS + self.__hashedBootKey[:16] + QWERTY)

        if checkSum != self.__hashedBootKey[16:]:
            raise Exception('hashedBootKey CheckSum failed, Syskey startup password probably in use! :(') 

def digest(self, message_parts):
        if self.key_id == 0:
            assert self.algorithm == "null"
            return b''
        elif "hmac" in self.algorithm:
            assert self.algorithm in ALGORITHMS
            digestmod = ALGORITHM_TO_DIGESTMOD[self.algorithm]
            the_hmac = hmac.new(self.secret.encode(), digestmod=digestmod)
            for message_part in message_parts:
                if message_part is not None:
                    the_hmac.update(message_part)
            return the_hmac.digest()
        else:
            assert self.algorithm in ALGORITHMS
            digestmod = ALGORITHM_TO_DIGESTMOD[self.algorithm]
            the_hash = hashlib.new(name=digestmod)
            the_hash.update(self.secret.encode())
            for message_part in message_parts:
                if message_part is not None:
                    the_hash.update(message_part)
            return the_hash.digest() 

def check_against_chunks(self, chunks):
        """Check good hashes against ones built from iterable of chunks of
        data.

        Raise HashMismatch if none match.

        """
        gots = {}
        for hash_name in iterkeys(self._allowed):
            try:
                gots[hash_name] = hashlib.new(hash_name)
            except (ValueError, TypeError):
                raise InstallationError('Unknown hash name: %s' % hash_name)

        for chunk in chunks:
            for hash in itervalues(gots):
                hash.update(chunk)

        for hash_name, got in iteritems(gots):
            if got.hexdigest() in self._allowed[hash_name]:
                return
        self._raise(gots) 

def check_against_chunks(self, chunks):
        """Check good hashes against ones built from iterable of chunks of
        data.

        Raise HashMismatch if none match.

        """
        gots = {}
        for hash_name in iterkeys(self._allowed):
            try:
                gots[hash_name] = hashlib.new(hash_name)
            except (ValueError, TypeError):
                raise InstallationError('Unknown hash name: %s' % hash_name)

        for chunk in chunks:
            for hash in itervalues(gots):
                hash.update(chunk)

        for hash_name, got in iteritems(gots):
            if got.hexdigest() in self._allowed[hash_name]:
                return
        self._raise(gots) 

def checkalreadyscanned((filedir, filename, checksum)):
	resolved_path = os.path.join(filedir, filename)
	try:
		os.stat(resolved_path)
	except:
		print >>sys.stderr, "Can't find %s" % filename
		return None
	if checksum != None:
		filehash = checksum
	else:
		scanfile = open(resolved_path, 'r')
		h = hashlib.new('sha256')
		h.update(scanfile.read())
		scanfile.close()
		filehash = h.hexdigest()
	return (filename, filehash) 

def computehash((filedir, filename, extrahashes, sha256sum)):
	resolved_path = os.path.join(filedir, filename)
	filehashes = {}
	filehashes['sha256'] = sha256sum
	scanfile = open(resolved_path, 'r')
	data = scanfile.read()
	scanfile.close()
	for i in extrahashes:
		if i == 'crc32':
			filehashes[i] = zlib.crc32(data) & 0xffffffff
		elif i == 'tlsh':
			if os.stat(resolved_path).st_size >= 256:
				tlshhash = tlsh.hash(data)
				filehashes[i] = tlshhash
			else:
				filehashes[i] = None
		else:
			h = hashlib.new(i)
			h.update(data)
			filehashes[i] = h.hexdigest()
	filehashes['sha256'] = sha256sum
	return (filedir, filename, filehashes) 

def generate_password(input_seed):
    """
    Generate some hard-to-guess but deterministic password.
    (deterministic so we have some hope of password recovery if something gets lost)

    Note: settings.SECRET_KEY is different (and actually secret) in production.
    """
    # generate seed integer
    secret = settings.SECRET_KEY
    seed_str = '_'.join([secret, input_seed, PW_SERIES, secret])
    h = hashlib.new('sha512')
    h.update(seed_str.encode('utf8'))
    seed = int(h.hexdigest(), 16)

    # use seed to pick characters: one letter, one digit, one punctuation, length 6-10
    letter, seed = _choose_from(LETTERS, seed)
    digit, seed = _choose_from(DIGITS, seed)
    punc, seed = _choose_from(PUNCTUATION, seed)
    pw = letter + digit + punc
    for i in range(7):
        c, seed = _choose_from(ALL_CHARS, seed)
        pw += c

    return pw 

def get_file_hash(algorithm, path):
    '''
    Calculates a file's hash.

    .. WARNING::
        The hashing algorithm must be supported on your system, as documented at `hashlib documentation page <http://docs.python.org/3/library/hashlib.html>`_.
    
    :param algorithm: Hashing algorithm.
    :type algorithm: string
    :param path: The file path
    :type path: string
    :rtype: string
    '''
    hashAlg = hashlib.new(algorithm)
    block_sz = 1*1024**2  # 1 MB

    with open(path, 'rb') as f:
        data = f.read(block_sz)
        while data:
            hashAlg.update(data)
            data = f.read(block_sz)
    
    return hashAlg.hexdigest() 

def computehasharchive((filedir, checksums, version, filename)):
	resolved_path = os.path.join(filedir, filename)
	try:
		os.stat(resolved_path)
	except:
		print >>sys.stderr, "Can't find %s" % filename
		return None
	if checksums.has_key(filename):
		filehash = checksums[filename]
	else:
		scanfile = open(resolved_path, 'r')
		h = hashlib.new('sha256')
		h.update(scanfile.read())
		scanfile.close()
		filehash = h.hexdigest()
	return (filename, version, filehash)

## per package:
## 1. unpack archive
## 2. scan all files and compute hashes
## 3. lookup what is available in the database from the same package and origin
## 4. copy unique and unscanned files to new archive dir 
## 5. create text file with metadata
## 6. pack archive
## 7. pack archive + metadata into BAT archive 

def gethash(path):
	scanfile = open(path, 'r')
	h = hashlib.new('sha256')
	scanfile.seek(0)
	hashdata = scanfile.read(10000000)
	while hashdata != '':
		h.update(hashdata)
		hashdata = scanfile.read(10000000)
	scanfile.close()
	return h.hexdigest()

## helper function to condense version numbers and squash numbers. 

def verifyCertificate(filename, cursor, conn, tempdir=None, tags=[], offsets={}, scanenv={}, debug=False, unpacktempdir=None, filehashes=None):
	newtags = []

	## a list of known certificates observed in the wild in various
	## installers for Windows software.
	## These certificates are likely in DER format but it is
	## a bit hard to find out which flavour so they can be verified
	## with openssl
	knowncerts = ['042f81e050c384566c1d10dd329712013e1265181196d976b6c75eb244b7f334',
		      'b2ae0c8d9885670d40dae35edc286b6617f1836053e42ffb1d83281f8a6354e6',
		      'dde6c511b2798af5a89fdeeaf176204df3f2c562c79a843d80b68f32a0fbccae',
		      'fdb72f2b5e7cbc57f196e37a7c96f71529124e1c1a7477c63df8e28dc2910c8b',
		     ]

	if os.path.basename(filename) == 'CERTIFICATE':
		certfile = open(filename, 'rb')
		h = hashlib.new('sha256')
		h.update(certfile.read())
		certfile.close()
		if h.hexdigest() in knowncerts:
			newtags.append('certificate')
	return newtags

## check compiled terminfo files
## man 5 term
## does not check the ncurses extensions 

def MD5(self, data):
        md5 = hashlib.new('md5')
        md5.update(data)
        return md5.digest() 

def __restore(self):
        # First of all stop the service if it was originally stopped
        if self.__shouldStop is True:
            LOG.info('Stopping service %s' % self.__serviceName)
            scmr.hRControlService(self.__scmr, self.__serviceHandle, scmr.SERVICE_CONTROL_STOP)
        if self.__disabled is True:
            LOG.info('Restoring the disabled state for service %s' % self.__serviceName)
            scmr.hRChangeServiceConfigW(self.__scmr, self.__serviceHandle, dwStartType = 0x4)
        if self.__serviceDeleted is False:
            # Check again the service we created does not exist, starting a new connection
            # Why?.. Hitting CTRL+C might break the whole existing DCE connection
            try:
                rpc = transport.DCERPCTransportFactory(r'ncacn_np:%s[\pipe\svcctl]' % self.__smbConnection.getRemoteHost())
                if hasattr(rpc, 'set_credentials'):
                    # This method exists only for selected protocol sequences.
                    rpc.set_credentials(*self.__smbConnection.getCredentials())
                    rpc.set_kerberos(self.__doKerberos, self.__kdcHost)
                self.__scmr = rpc.get_dce_rpc()
                self.__scmr.connect()
                self.__scmr.bind(scmr.MSRPC_UUID_SCMR)
                # Open SC Manager
                ans = scmr.hROpenSCManagerW(self.__scmr)
                self.__scManagerHandle = ans['lpScHandle']
                # Now let's open the service
                resp = scmr.hROpenServiceW(self.__scmr, self.__scManagerHandle, self.__tmpServiceName)
                service = resp['lpServiceHandle']
                scmr.hRDeleteService(self.__scmr, service)
                scmr.hRControlService(self.__scmr, service, scmr.SERVICE_CONTROL_STOP)
                scmr.hRCloseServiceHandle(self.__scmr, service)
                scmr.hRCloseServiceHandle(self.__scmr, self.__serviceHandle)
                scmr.hRCloseServiceHandle(self.__scmr, self.__scManagerHandle)
                rpc.disconnect()
            except Exception, e:
                # If service is stopped it'll trigger an exception
                # If service does not exist it'll trigger an exception
                # So. we just wanna be sure we delete it, no need to 
                # show this exception message
                pass 

def evpKDF(passwd, salt, key_size=8, iv_size=4, iterations=1, hash_algorithm="md5"):
    target_key_size = key_size + iv_size
    derived_bytes = ""
    number_of_derived_words = 0
    block = None
    hasher = hashlib.new(hash_algorithm)
    while number_of_derived_words < target_key_size:
        if block is not None:
            hasher.update(block)

        hasher.update(passwd)
        hasher.update(salt)
        block = hasher.digest()
        hasher = hashlib.new(hash_algorithm)

        for i in range(1, iterations):
            hasher.update(block)
            block = hasher.digest()
            hasher = hashlib.new(hash_algorithm)

        derived_bytes += block[0: min(len(block), (target_key_size - number_of_derived_words) * 4)]

        number_of_derived_words += len(block)/4

    return {
        "key": derived_bytes[0: key_size * 4],
        "iv": derived_bytes[key_size * 4:]
    } 

def encode(plaintext,passphrase,saltsize=8):
    salt= os.urandom(saltsize)
    data = evpKDF(passphrase,salt)
    decryptor = new(data['key'], MODE_CBC, IV=data['iv'])
    plaintext = PKCS7Encoder().encode(plaintext)
    enctext= decryptor.encrypt(plaintext)
    return base64.b64encode("Salted__"+salt+enctext)

##''if salt is provided, it should be string
##ciphertext is base64 and passphrase is string 

def decode(ciphertext,passphrase,salt=None):
    ciphertext=base64.b64decode(ciphertext)
    if not salt:
        salt=ciphertext[8:16]
        ciphertext=ciphertext[16:]
    data = evpKDF(passphrase, salt)
    decryptor = new(data['key'], MODE_CBC, IV=data['iv'])
    d= decryptor.decrypt(ciphertext)
    return PKCS7Encoder().decode(d) 

def new(key, mode, IV=None):
    if mode == MODE_ECB:
        return ECBMode(AES(key))
    elif mode == MODE_CBC:
        if IV is None:
            raise ValueError, "CBC mode needs an IV value!"

        return CBCMode(AES(key), IV)
    else:
        raise NotImplementedError

#### AES cipher implementation 

def sourceWalk(scandir, dbpath):
	conn = sqlite3.connect(dbpath, check_same_thread = False)

	cursor = conn.cursor()
	osgen = os.walk(scandir)
	lenscandir = len(scandir)
	notfound = 0
	total = 0

	try:
		while True:
			i = osgen.next()
			for p in i[2]:
				if os.stat("%s/%s" % (i[0], p)).st_size == 0:
					continue
				p_nocase = p.lower()
				for extension in extensions.keys():
					if (p_nocase.endswith(extension)):
						total = total + 1
						scanfile = open("%s/%s" % (i[0], p), 'r')
						h = hashlib.new('sha256')
						h.update(scanfile.read())
						scanfile.close()
						filehash = h.hexdigest()
						cursor.execute('''select checksum from processed_file where checksum=? limit 1''', (filehash,))
						res = cursor.fetchall()
						## there is at least one hit, so ignore
						if len(res) != 0:
							continue
						## no hits, so this is an interesting file
						else:
							print "%s" % os.path.join(scandir, i[0][lenscandir:],p)
							notfound = notfound + 1
				pass
	except StopIteration:
		pass
	print "Total files: %d" % total
	print "Files not found in database: %d" % notfound 

def gethash(path):
	scanfile = open(path, 'r')
	h = hashlib.new('sha256')
	scanfile.seek(0)
	hashdata = scanfile.read(10000000)
	while hashdata != '':
		h.update(hashdata)
		hashdata = scanfile.read(10000000)
	scanfile.close()
	return h.hexdigest() 

def computehash((filedir, filename, extension, language, extrahashes)):
	filehashes = {}
	resolved_path = os.path.join(filedir, filename)
	scanfile = open(resolved_path, 'r')
	h = hashlib.new('sha256')
	h.update(scanfile.read())
	scanfile.close()
	filehashes['sha256'] = h.hexdigest()
	if len(extrahashes) != 0:
		scanfile = open(resolved_path, 'r')
		data = scanfile.read()
		scanfile.close()
		for i in extrahashes:
			if i == 'crc32':
				filehashes[i] = zlib.crc32(data) & 0xffffffff
			elif i == 'tlsh':
				if os.stat(resolved_path).st_size >= 256:
					tlshhash = tlsh.hash(data)
					filehashes[i] = tlshhash
				else:
					filehashes[i] = None
			else:
				h = hashlib.new(i)
				h.update(data)
				filehashes[i] = h.hexdigest()
		
	return (filedir, filename, filehashes, extension, language) 

def scanhashes(resolved_path, extrahashes):
	filehashes = {}
	scanfile = open(resolved_path, 'r')
	h = hashlib.new('sha256')
	data = scanfile.read()
	h.update(data)
	scanfile.close()
	filehashes['sha256'] = h.hexdigest()

	for i in extrahashes:
		if i == 'crc32':
			if os.stat(resolved_path).st_size > 2147483647:
				filehashes[i] = None
			else:
				filehashes[i] = zlib.crc32(data) & 0xffffffff
		elif i == 'tlsh':
			if os.stat(resolved_path).st_size >= 256:
				tlshhash = tlsh.hash(data)
				filehashes[i] = tlshhash
			else:
				filehashes[i] = None
		else:
			h = hashlib.new(i)
			h.update(data)
			filehashes[i] = h.hexdigest()
		scanfile.close()
	return filehashes

## scan each RPM file and see if there are any source code archives inside.
## This check is based on conventions on how source code archives are named and
## might miss things.
## TODO: store patches as well 

def Hash160(msg):
    """RIPEME160(SHA256(msg)) -> bytes"""
    h = hashlib.new('ripemd160')
    h.update(hashlib.sha256(msg).digest())
    return h.digest() 

def Hash160(msg):
    """RIPEME160(SHA256(msg)) -> bytes"""
    h = hashlib.new('ripemd160')
    h.update(hashlib.sha256(msg).digest())
    return h.digest() 

def gen_hash(password, salt):
    h = hashlib.new('sha1')
    h.update(password)
    h.update(salt)
    hash = h.hexdigest()
    
    return hash 

def rehash(path, algo='sha256', blocksize=1 << 20):
    """Return (hash, length) for path using hashlib.new(algo)"""
    h = hashlib.new(algo)
    length = 0
    with open(path, 'rb') as f:
        for block in read_chunks(f, size=blocksize):
            length += len(block)
            h.update(block)
    digest = 'sha256=' + urlsafe_b64encode(
        h.digest()
    ).decode('latin1').rstrip('=')
    return (digest, length) 

def _hash_of_file(path, algorithm):
    """Return the hash digest of a file."""
    with open(path, 'rb') as archive:
        hash = hashlib.new(algorithm)
        for chunk in read_chunks(archive):
            hash.update(chunk)
    return hash.hexdigest() 

def _create_mac(key, msg, method):
    if callable(method):
        return hmac.HMAC(key, msg, method)

    def hashfunc(d=b""):
        return hashlib.new(method, d)

    # Python 2.7 used ``hasattr(digestmod, '__call__')``
    # to detect if hashfunc is callable
    hashfunc.__call__ = hashfunc
    return hmac.HMAC(key, msg, hashfunc) 

def _hash_internal(method, salt, password):
    """Internal password hash helper.  Supports plaintext without salt,
    unsalted and salted passwords.  In case salted passwords are used
    hmac is used.
    """
    if method == "plain":
        return password, method

    if isinstance(password, text_type):
        password = password.encode("utf-8")

    if method.startswith("pbkdf2:"):
        args = method[7:].split(":")
        if len(args) not in (1, 2):
            raise ValueError("Invalid number of arguments for PBKDF2")
        method = args.pop(0)
        iterations = args and int(args[0] or 0) or DEFAULT_PBKDF2_ITERATIONS
        is_pbkdf2 = True
        actual_method = "pbkdf2:%s:%d" % (method, iterations)
    else:
        is_pbkdf2 = False
        actual_method = method

    if is_pbkdf2:
        if not salt:
            raise ValueError("Salt is required for PBKDF2")
        rv = pbkdf2_hex(password, salt, iterations, hashfunc=method)
    elif salt:
        if isinstance(salt, text_type):
            salt = salt.encode("utf-8")
        mac = _create_mac(salt, password, method)
        rv = mac.hexdigest()
    else:
        rv = hashlib.new(method, password).hexdigest()
    return rv, actual_method