Python numpy.save() Examples

The following are code examples for showing how to use numpy.save(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: Keras-Unet   Author: MLearing   File: data.py    GNU General Public License v2.0 7 votes vote down vote up
def create_test_data(self):

        # 测试集生成npy文件
        i = 0
        print('-' * 30)
        print('Creating training images...')
        print('-' * 30)
        imgs = glob.glob(self.test_path + "/*." + self.img_type)           # deform/train
        print(len(imgs))
        imgdatas = np.ndarray((len(imgs), self.out_rows, self.out_cols, 1), dtype=np.uint8)
        for imgname in imgs:
            midname = imgname[imgname.rindex("/") + 1:]   # 图像的名字
            img = load_img(self.test_path + "/" + midname, grayscale=True)   # 转换为灰度图
            img = img_to_array(img)
            imgdatas[i] = img
            if i % 100 == 0:
                print('Done: {0}/{1} images'.format(i, len(imgs)))
            i += 1
        print('loading done', imgdatas.shape)
        np.save(self.npy_path + '/imgs_test.npy', imgdatas)            # 将30张训练集和30张label生成npy数据
        print('Saving to .npy files done.') 
Example 2
Project: prediction-constrained-topic-models   Author: dtak   File: slda_utils__dataset_manager.py    MIT License 6 votes vote down vote up
def save_dataset(
        dataset=None,
        output_path=None,
        split_name='train',
        y_arr_prefix='Y',
        x_arr_prefix='X_csr',
        ):
    x_arr_path = os.path.join(
        output_path,
        '%s_%s.npz' % (x_arr_prefix, split_name))
    y_arr_path = os.path.join(
        output_path,
        '%s_%s.npy' % (y_arr_prefix, split_name))
    np.save(y_arr_path, dataset['y_DC'])
    save_csr_matrix(x_arr_path, dataset['x_csr_DV'])
    return x_arr_path, y_arr_path 
Example 3
Project: RandomFourierFeatures   Author: tiskw   File: download_and_convert_mnist.py    MIT License 6 votes vote down vote up
def convert_label_data(filepath_input, filepath_output):

    ### Skip the following procedure if the output file exists.
    if os.path.exists(filepath_output):
        print("File '%s' already exists. Skip it." % filepath_output)
        return

    print("Convert: %s -> %s" % (filepath_input, filepath_output))

    ### Unzip downloaded file.
    with gzip.open(filepath_input, "rb") as ifp:
        data = ifp.read()

    ### Parse header information.
    identifier = int.from_bytes(data[ 0: 4], BYTE_ORDER)
    num_images = int.from_bytes(data[ 4: 8], BYTE_ORDER)

    if identifier != 2049:
        print("Input file '%s' does not seems to be MNIST image file." % filepath)

    labels = np.array([int(b) for b in data[8:]]).reshape((num_images, ))

    np.save(filepath_output, labels) 
Example 4
Project: tensorflow_generate_headlines   Author: FanWan   File: data_utils.py    GNU General Public License v3.0 6 votes vote down vote up
def get_glove_embedding(word2ind, embedding_dim=300):
    """
       creates embedding matrix for each word in word2ind. if that words is in
       pretrained_embeddings, that vector is used. otherwise initialized
       randomly.
    """
    if os.path.exists(os.path.dirname(embedding_matrix_save_path)):
        return np.load(embedding_matrix_save_path)
    else:
        pretrained_embeddings = load_pretrained_embeddings(glove_embeddings_path)
        embedding_matrix = np.zeros((len(word2ind), embedding_dim), dtype=np.float32)
        for word, i in word2ind.items():
            if word in pretrained_embeddings.keys():
                embedding_matrix[i] = pretrained_embeddings[word]
            else:
                embedding = np.array(np.random.uniform(-1.0, 1.0, embedding_dim))
                embedding_matrix[i] = embedding
        os.makedirs(os.path.dirname(embedding_matrix_save_path))
        np.save(embedding_matrix_save_path, embedding_matrix)
        return np.array(embedding_matrix) 
Example 5
Project: AnisotropicMultiStreamCNN   Author: AnnekeMeyer   File: generateTrainingData.py    MIT License 6 votes vote down vote up
def createAnisotropicFoldArrays(data_directory, fold_dir, output_directory, nr_folds = 4):

    # patients = os.listdir(input_directory)
    for i in range(1, nr_folds+1):

        val_data = np.load(fold_dir+'/test_fold' + str(i) + '.npy')
        val_list = val_data.tolist()


        #img_arr_tra, img_arr_cor, img_arr_sag, gt_arr= createArraysFromPatientList(train_list, data_directory)
        val_img_arr_tra, val_img_arr_cor, val_img_arr_sag, val_gt_arr = createArraysFromPatientList(val_list, data_directory)


        #np.save(output_directory + 'fold' + str(i) + '_train_imgs_tra.npy', img_arr_tra)
        np.save(output_directory + '/fold' + str(i) + '_val_imgs_tra.npy', val_img_arr_tra)

        #np.save(output_directory + 'fold' + str(i) + '_train_imgs_cor.npy', img_arr_cor)
        np.save(output_directory + '/fold' + str(i) + '_val_imgs_cor.npy', val_img_arr_cor)

        #np.save(output_directory + 'fold' + str(i) + '_train_imgs_sag.npy', img_arr_sag)
        np.save(output_directory + '/fold' + str(i) + '_val_imgs_sag.npy', val_img_arr_sag)

        #np.save(output_directory + 'fold' + str(i) + '_train_GT.npy', gt_arr)
        np.save(output_directory + '/fold' + str(i) + '_val_GT.npy', val_gt_arr) 
Example 6
Project: Scene-Understanding   Author: foamliu   File: class_rebal.py    MIT License 6 votes vote down vote up
def compute_class_prior(do_plot=False):
    names = [f for f in os.listdir(seg_path) if f.lower().endswith('.png')]
    num_samples = len(names)
    prior_prob = np.zeros(num_classes)
    pb = ProgressBar(total=num_samples, prefix='Compute class prior', suffix='', decimals=3, length=50, fill='=')
    for i in range(num_samples):
        name = names[i]
        filename = os.path.join(seg_path, name)
        category = np.ravel(cv.imread(filename, 0))
        counts = np.bincount(category)
        idxs = np.nonzero(counts)[0]
        prior_prob[idxs] += counts[idxs]
        pb.print_progress_bar(i + 1)

    prior_prob = prior_prob / (1.0 * np.sum(prior_prob))

    # Save
    np.save(os.path.join(data_dir, "prior_prob.npy"), prior_prob)

    if do_plot:
        plt.hist(prior_prob, bins=100)
        plt.yscale("log")
        plt.show() 
Example 7
Project: Scene-Understanding   Author: foamliu   File: class_rebal.py    MIT License 6 votes vote down vote up
def compute_prior_factor(gamma=0.5, alpha=1, do_plot=False):
    file_name = os.path.join(data_dir, "prior_prob_smoothed.npy")
    prior_prob_smoothed = np.load(file_name)

    u = np.ones_like(prior_prob_smoothed)
    u = u / np.sum(1.0 * u)

    prior_factor = (1 - gamma) * prior_prob_smoothed + gamma * u
    prior_factor = np.power(prior_factor, -alpha)

    # renormalize
    prior_factor = prior_factor / (np.sum(prior_factor * prior_prob_smoothed))

    file_name = os.path.join(data_dir, "prior_factor.npy")
    np.save(file_name, prior_factor)

    if do_plot:
        plt.plot(prior_factor)
        plt.yscale("log")
        plt.show() 
Example 8
Project: DOTA_models   Author: ringringyi   File: input.py    Apache License 2.0 6 votes vote down vote up
def extract_mnist_data(filename, num_images, image_size, pixel_depth):
  """
  Extract the images into a 4D tensor [image index, y, x, channels].

  Values are rescaled from [0, 255] down to [-0.5, 0.5].
  """
  # if not os.path.exists(file):
  if not tf.gfile.Exists(filename+".npy"):
    with gzip.open(filename) as bytestream:
      bytestream.read(16)
      buf = bytestream.read(image_size * image_size * num_images)
      data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32)
      data = (data - (pixel_depth / 2.0)) / pixel_depth
      data = data.reshape(num_images, image_size, image_size, 1)
      np.save(filename, data)
      return data
  else:
    with tf.gfile.Open(filename+".npy", mode='r') as file_obj:
      return np.load(file_obj) 
Example 9
Project: polish-sentence-evaluation   Author: sdadas   File: utils.py    GNU General Public License v3.0 6 votes vote down vote up
def cached(batcher_func: Callable, cache: Path) -> Callable:
    def batcher(params, batch):
        task_name = params.current_task
        dataset = params.get("batcher_dataset")
        offset = params.get("batcher_offset")
        size = str(len(batch))
        if task_name and dataset and offset:
            file_name = f"{task_name}_{dataset}_{offset}_{size}.npy"
            cached_file = cache / file_name
            if cached_file.exists():
                logging.debug("Loading batch from cache %s", file_name)
                return np.load(str(cached_file.absolute()))
            else:
                embeddings = batcher_func(params, batch)
                logging.debug("Saving batch to cache %s", file_name)
                np.save(str(cached_file.absolute()), embeddings)
                return embeddings
        else:
            return batcher_func(params, batch)

    return batcher 
Example 10
Project: sumo   Author: ratan-lab   File: test_evaluate.py    MIT License 6 votes vote down vote up
def test_init(tmpdir):
    # incorrect parameters
    with pytest.raises(AttributeError):
        SumoEvaluate()

    fname = os.path.join(tmpdir, "indata.npz")
    labels_npy = os.path.join(tmpdir, "labels.npy")
    args = _get_args(fname, labels_npy)

    # no input file
    with pytest.raises(FileNotFoundError):
        SumoEvaluate(**args)

    samples = 10
    labels = [0] * int(samples / 2) + [1] * int(samples / 2)
    data = np.array([
        ['sample_{}'.format(i) for i in range(samples)], labels]).T
    save_arrays_to_npz({'clusters': data}, fname)

    # no labels file
    with pytest.raises(FileNotFoundError):
        SumoEvaluate(**args)

    np.save(labels_npy, np.array(labels), allow_pickle=True)
    SumoEvaluate(**args) 
Example 11
Project: Distributed-AutoDeepLab   Author: HankKung   File: calculate_weights.py    Apache License 2.0 6 votes vote down vote up
def calculate_weigths_labels(dataset, dataloader, num_classes):
    # Create an instance from the data loader
    z = np.zeros((num_classes,))
    # Initialize tqdm
    tqdm_batch = tqdm(dataloader)
    print('Calculating classes weights')
    for sample in tqdm_batch:
        y = sample['label']
        y = y.detach().cpu().numpy()
        mask = (y >= 0) & (y < num_classes)
        labels = y[mask].astype(np.uint8)
        count_l = np.bincount(labels, minlength=num_classes)
        z += count_l
    tqdm_batch.close()
    total_frequency = np.sum(z)
    class_weights = []
    for frequency in z:
        class_weight = 1 / (np.log(1.02 + (frequency / total_frequency)))
        class_weights.append(class_weight)
    ret = np.array(class_weights)
    classes_weights_path = os.path.join(Path.db_root_dir(dataset), dataset+'_classes_weights.npy')
    np.save(classes_weights_path, ret)

    return ret 
Example 12
Project: pytorch-mri-segmentation-3D   Author: Achilleas   File: PP.py    MIT License 6 votes vote down vote up
def extractMeanDataStats(size = [200, 200, 100], 
						postfix = '_200x200x100orig', 
						main_folder_path = '../../Data/MS2017b/', 
						):
	scan_folders = glob.glob(main_folder_path + 'scans/*')
	img_path = 'pre/FLAIR' + postfix + '.nii.gz'
	segm_path = 'wmh' + postfix + '.nii.gz'
	
	shape_ = [len(scan_folders), size[0], size[1], size[2]]
	arr = np.zeros(shape_)

	for i, sf in enumerate(scan_folders):
		arr[i, :,:,:] =  numpyFromScan(os.path.join(sf,img_path)).squeeze()

	arr /= len(scan_folders)

	means = np.mean(arr)
	stds = np.std(arr, axis = 0)

	np.save(main_folder_path + 'extra_data/std' + postfix, stds)
	np.save(main_folder_path + 'extra_data/mean' + postfix, means) 
Example 13
Project: pytorch-mri-segmentation-3D   Author: Achilleas   File: PP.py    MIT License 6 votes vote down vote up
def generateImgSlicesFolder(data_folder = '../Data/MS2017a/scans/'):
	scan_folders = glob.glob(data_folder + '*')

	for sf in scan_folders:
		slice_dir_path = os.path.join(sf, 'slices/')
		if not os.path.exists(slice_dir_path):
			print('Creating directory at:' , slice_dir_path)
			os.makedirs(slice_dir_path)

		img = nib.load(os.path.join(sf, 'pre/FLAIR.nii.gz'))
		img_np = img.get_data()
		img_affine = img.affine
		print(sf)
		print('The img shape', img_np.shape[2])
		for i in range(img_np.shape[2]):
			slice_img_np = img_np[:,:,i]
			nft_img = nib.Nifti1Image(slice_img_np, img_affine)
			nib.save(nft_img, slice_dir_path + 'FLAIR_' + str(i) + '.nii.gz')

			if os.path.basename(sf) == '0':
				slice_img = nib.load(slice_dir_path + 'FLAIR_' + str(i) + '.nii.gz').get_data() / 5
				print('DID I GET HERE?')
				print('Writing to', str(i) + '.jpg') 
Example 14
Project: pytorch-mri-segmentation-3D   Author: Achilleas   File: PP.py    MIT License 6 votes vote down vote up
def generateGTSlicesFolder(data_folder = '../Data/MS2017a/scans/'):
	scan_folders = glob.glob(data_folder + '*')

	for sf in scan_folders:
		slice_dir_path = os.path.join(sf, 'gt_slices/')
		if not os.path.exists(slice_dir_path):
			print('Creating directory at:' , slice_dir_path)
			os.makedirs(slice_dir_path)

		img = nib.load(os.path.join(sf, 'wmh.nii.gz'))
		img_np = img.get_data()
		img_affine = img.affine
		print(sf)
		print('The img shape', img_np.shape[2])
		for i in range(img_np.shape[2]):
			slice_img_np = img_np[:,:,i]
			nft_img = nib.Nifti1Image(slice_img_np, img_affine)
			nib.save(nft_img, slice_dir_path + 'wmh_' + str(i) + '.nii.gz')

			if os.path.basename(sf) == '0':
				slice_img = nib.load(slice_dir_path + 'wmh_' + str(i) + '.nii.gz').get_data() * 256
				#cv2.imwrite('temp/' + str(i) + '.jpg', slice_img) 
Example 15
Project: pytorch-mri-segmentation-3D   Author: Achilleas   File: PP.py    MIT License 6 votes vote down vote up
def generateTrainValFile_Slices(train_fraction, main_folder = '../Data/MS2017a/'):
	train_folders, val_folders = splitTrainVal_Slices(0.8)

	train_folder_names = [train_folders[i].split(main_folder)[1] for i in range(len(train_folders))]
	val_folder_names = [val_folders[i].split(main_folder)[1] for i in range(len(val_folders))]

	f_train = open(main_folder + 'train_slices.txt', 'w+')
	f_val = open(main_folder + 'val_slices.txt', 'w+')

	for fn in train_folder_names:
		f_train.write(fn + '\n')

	for fn in val_folder_names:
		f_val.write(fn + '\n')

	f_train.close()
	f_val.close()

#Use this to save the images quickly (for testing purposes) 
Example 16
Project: UROP-Adversarial-Feature-Matching-for-Text-Generation   Author: Jeff-HOU   File: data.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def next_batch(self, whichSet='train'):
		if whichSet == 'train':
			self.trainBatchCnt += 1
			assert self.trainBatchCnt < self.trainMaxBatch
			return self.train[self.trainBatchCnt * self.batch_size: (self.trainBatchCnt + 1) * self.batch_size]
		elif whichSet == 'validation':
			self.validationBatchCnt += 1
			assert self.validationBatchCnt < self.validationMaxBatch
			return self.validation[self.validationBatchCnt * self.batch_size: (self.validationBatchCnt + 1) * self.batch_size]
		elif whichSet == 'test':
			self.testBatchCnt += 1
			assert self.testBatchCnt < self.testMaxBatch
			return self.test[self.testBatchCnt * self.batch_size: (self.testBatchCnt + 1) * self.batch_size]
		else:
			msg = 'Wrong set name!\n'+ \
				  'Should be train / validation / test.'
			raise Exception(msg)
	# Following code copied here:
	# https://stackoverflow.com/questions/17219481/save-to-file-and-load-an-instance-of-a-python-class-with-its-attributes 
Example 17
Project: face-attendance-machine   Author: matiji66   File: encoding_images.py    Apache License 2.0 5 votes vote down vote up
def encoding_images(path):
    """
    对path路径下的子文件夹中的图片进行编码,
    TODO:
        对人脸数据进行历史库中的人脸向量进行欧式距离的比较,当距离小于某个阈值的时候提醒:
        如果相似的是本人,则跳过该条记录,并提醒已经存在,否则警告人脸过度相似问题,
    :param path:
    :return:
    """
    with open(name_and_encoding, 'w') as f:
        subdirs = [os.path.join(path, x) for x in os.listdir(path) if os.path.isdir(os.path.join(path, x))]
        for subdir in subdirs:
            print('process image name :', subdir)
            person_image_encoding = []
            for y in os.listdir(subdir):
                print("image name is ", y)
                _image = face_recognition.load_image_file(os.path.join(subdir, y))
                face_encodings = face_recognition.face_encodings(_image)
                name = os.path.split(subdir)[-1]
                if face_encodings and len(face_encodings) == 1:
                    if len(person_image_encoding) == 0:
                        person_image_encoding.append(face_encodings[0])
                        known_face_names.append(name)
                        continue
                    for i in range(len(person_image_encoding)):
                        distances = face_recognition.compare_faces(person_image_encoding, face_encodings[0], tolerance=image_thread)
                        if False in distances:
                            person_image_encoding.append(face_encodings[0])
                            known_face_names.append(name)
                            print(name, " new feature")
                            f.write(name + ":" + str(face_encodings[0]) + "\n")
                            break
                    # face_encoding = face_recognition.face_encodings(_image)[0]
                    # face_recognition.compare_faces()
            known_face_encodings.extend(person_image_encoding)
            bb = np.array(known_face_encodings)
            print("--------")
    np.save(KNOWN_FACE_ENCODINGS, known_face_encodings)
    np.save(KNOWN_FACE_NANE, known_face_names) 
Example 18
Project: DBC-FederatedLearning-Client-VNX   Author: DeepBrainChain   File: learning.py    Apache License 2.0 5 votes vote down vote up
def save_model(model, model_path="new_model.pt"):
    # torch.save() needs pickle.py location. We will add code in future in the case that default does not work.
    try:
        torch.save(model, model_path)
    except Exception as e: 
        print("Failed to save model.")
        print(e)





##### Note: each img can only have 1 label for the following code 
Example 19
Project: disentangling_conditional_gans   Author: zalandoresearch   File: dataset_tool.py    MIT License 5 votes vote down vote up
def add_labels(self, labels):
        if self.print_progress:
            print('%-40s\r' % 'Saving labels...', end='', flush=True)
        assert labels.shape[0] == self.cur_images
        with open(self.tfr_prefix + '-rxx.labels', 'wb') as f:
            np.save(f, labels.astype(np.float32)) 
Example 20
Project: disentangling_conditional_gans   Author: zalandoresearch   File: dataset_tool.py    MIT License 5 votes vote down vote up
def extract(tfrecord_dir, output_dir):
    print('Loading dataset "%s"' % tfrecord_dir)
    tfutil.init_tf({'gpu_options.allow_growth': True})
    dset = dataset.TFRecordDataset(tfrecord_dir, max_label_size=0, repeat=False, shuffle_mb=0)
    tfutil.init_uninited_vars()
    
    print('Extracting images to "%s"' % output_dir)
    if not os.path.isdir(output_dir):
        os.makedirs(output_dir)
    idx = 0
    while True:
        if idx % 10 == 0:
            print('%d\r' % idx, end='', flush=True)
        try:
            images, labels = dset.get_minibatch_np(1)
        except tf.errors.OutOfRangeError:
            break
        if images.shape[1] == 1:
            img = PIL.Image.fromarray(images[0][0], 'L')
        else:
            img = PIL.Image.fromarray(images[0].transpose(1, 2, 0), 'RGB')
        img.save(os.path.join(output_dir, 'img%08d.png' % idx))
        idx += 1
    print('Extracted %d images.' % idx)

#---------------------------------------------------------------------------- 
Example 21
Project: Tacotron   Author: ElwynWang   File: pre_process.py    GNU General Public License v3.0 5 votes vote down vote up
def load_audio(audio_path):
	fname, mel, mag = load_spectrograms(audio_path)
	sub_paths = audio_path.strip().split('/')
	prefix = '/'.join(sub_paths[:-2])
	
	np.save(prefix+"/mels/{}".format(sub_paths[-1].replace('wav', 'npy')), mel)
	np.save(prefix+"/mags/{}".format(sub_paths[-1].replace('wav', 'npy')), mag) 
Example 22
Project: RandomFourierFeatures   Author: tiskw   File: download_and_convert_mnist.py    MIT License 5 votes vote down vote up
def convert_image_data(filepath_input, filepath_output):

    ### Skip the following procedure if the output file exists.
    if os.path.exists(filepath_output):
        print("File '%s' already exists. Skip it." % filepath_output)
        return

    print("Convert: %s -> %s" % (filepath_input, filepath_output))

    ### Unzip downloaded file.
    with gzip.open(filepath_input, "rb") as ifp:
        data = ifp.read()

    ### Parse header information.
    identifier = int.from_bytes(data[ 0: 4], BYTE_ORDER)
    num_images = int.from_bytes(data[ 4: 8], BYTE_ORDER)
    image_rows = int.from_bytes(data[ 8:12], BYTE_ORDER)
    image_cols = int.from_bytes(data[12:16], BYTE_ORDER)
    image_data = data[16:]

    if identifier != 2051:
        print("Input file '%s' does not seems to be MNIST image file." % filepath)

    images = np.zeros((num_images, image_rows, image_cols))

    for n in range(num_images):
        index_b = image_rows * image_cols * n
        index_e = image_rows * image_cols * (n + 1)
        image   = [int(b) for b in image_data[index_b:index_e]]
        images[n, :, :] = np.array(image).reshape((image_rows, image_cols))

    np.save(filepath_output, images)


### Convert MNIST label data to npy format. 
Example 23
Project: snake   Author: valentinmace   File: neural_network.py    MIT License 5 votes vote down vote up
def save(self, name=None):
        """
        Saves network weights and biases into 2 separated files in current folder

        :param name: str, in case you want to name it
        :return: creates two files
        """
        if not name:
            np.save('saved_weights_'+str(self.score), self.weights)
            np.save('saved_biases_'+str(self.score), self.biases)
        else:
            np.save(name + '_weights', self.weights)
            np.save(name + '_biases', self.biases) 
Example 24
Project: Keras-Unet   Author: MLearing   File: data.py    GNU General Public License v2.0 5 votes vote down vote up
def Augmentation(self):
        # 读入3通道的train和label, 分别转换成矩阵, 然后将label的第一个通道放在train的第2个通处, 做数据增强
        print("运行 Augmentation")
        """
        Start augmentation.....
        """
        trains = self.train_imgs
        labels = self.label_imgs
        path_train = self.train_path
        path_label = self.label_path
        path_merge = self.merge_path
        imgtype = self.img_type
        path_aug_merge = self.aug_merge_path
        print(len(trains), len(labels))
        if len(trains) != len(labels) or len(trains) == 0 or len(trains) == 0:
            print("trains can't match labels")
            return 0
        for i in range(len(trains)):
            img_t = load_img(path_train + "/" + str(i) + "." + imgtype)  # 读入train
            img_l = load_img(path_label + "/" + str(i) + "." + imgtype)  # 读入label
            x_t = img_to_array(img_t)                                    # 转换成矩阵
            x_l = img_to_array(img_l)
            x_t[:, :, 2] = x_l[:, :, 0]                                  # 把label当做train的第三个通道
            img_tmp = array_to_img(x_t)
            img_tmp.save(path_merge + "/" + str(i) + "." + imgtype)      # 保存合并后的图像
            img = x_t
            img = img.reshape((1,) + img.shape)                          # 改变shape(1, 512, 512, 3)
            savedir = path_aug_merge + "/" + str(i)                      # 存储合并增强后的图像
            if not os.path.lexists(savedir):
                os.mkdir(savedir)
            self.doAugmentate(img, savedir, str(i))                      # 数据增强 
Example 25
Project: Keras-Unet   Author: MLearing   File: data.py    GNU General Public License v2.0 5 votes vote down vote up
def create_train_data(self):
        # 将增强之后的训练集生成npy文件         
        i = 0
        print('-' * 30)
        print('creating train image')
        print('-' * 30)
        count = 0
        for indir in os.listdir(self.aug_merge_path):
            path = os.path.join(self.aug_merge_path, indir)
            count += len(os.listdir(path))
        imgdatas = np.ndarray((count, self.out_rows, self.out_cols, 1), dtype=np.uint8)
        imglabels = np.ndarray((count, self.out_rows, self.out_cols, 1), dtype=np.uint8)
        for indir in os.listdir(self.aug_merge_path):
            trainPath = os.path.join(self.aug_train_path, indir)
            labelPath = os.path.join(self.aug_label_path, indir)
            print(trainPath, labelPath)
            imgs = glob.glob(trainPath + '/*' + '.tif')
            for imgname in imgs:
                trainmidname = imgname[imgname.rindex('/') + 1:]
                labelimgname = imgname[imgname.rindex('/') + 1:imgname.rindex('_')] + '_label.tif'
                print(trainmidname, labelimgname)
                img = load_img(trainPath + '/' + trainmidname, grayscale=True)
                label = load_img(labelPath + '/' + labelimgname, grayscale=True)
                img = img_to_array(img)
                label = img_to_array(label)
                imgdatas[i] = img
                imglabels[i] = label
                if i % 100 == 0:
                    print('Done: {0}/{1} images'.format(i, len(imgs)))
                i += 1
                print(i)
        print('loading done', imgdatas.shape)
        np.save(self.npy_path + '/augimgs_train.npy', imgdatas)            # 将30张训练集和30张label生成npy数据
        np.save(self.npy_path + '/augimgs_mask_train.npy', imglabels)
        print('Saving to .npy files done.') 
Example 26
Project: Deep_VoiceChanger   Author: pstuvwx   File: dataset.py    MIT License 5 votes vote down vote up
def save(path, bps, data):
    if data.dtype != np.int16:
        data = data.astype(np.int16)
    data = np.reshape(data, -1)
    wav.write(path, bps, data) 
Example 27
Project: Deep_VoiceChanger   Author: pstuvwx   File: dataset.py    MIT License 5 votes vote down vote up
def pre_encode():
    import tqdm

    path = input('enter wave path...')
    ds = WaveDataset(path, -1, True)
    num = ds.max // dif

    imgs = [ds.get_example(i) for i in tqdm.tqdm(range(num))]    
    dst = np.concatenate(imgs, axis=1)
    print(dst.shape)

    np.save(path[:-3]+'npy', dst)
    print('encoded file saved at', path[:-3]+'npy') 
Example 28
Project: tensorflow_generate_headlines   Author: FanWan   File: data_utils.py    GNU General Public License v3.0 5 votes vote down vote up
def build_dict(train=False, word2index_path=None):
    article_list, title_list = get_init_data(train)
    if not os.path.exists(os.path.dirname(word2index_path)):
        # get word2index dictionary
        words = list()
        for sent in article_list + title_list:
            for word in sent:
                words.append(word)

        word_counter = collections.Counter(words).most_common(n=50000)
        word_dict = dict()
        word_dict["<padding>"] = 0
        word_dict["<unk>"] = 1
        word_dict["<s>"] = 2
        word_dict["</s>"] = 3
        for word, _ in word_counter:
            word_dict[word] = len(word_dict)

        # save word2index dictionary
        os.makedirs(os.path.dirname(word2index_path))
        with open(word2index_path, "wb") as f:
            pickle.dump(word_dict, f)
    else:
        with open(word2index_path, "rb") as f:
            word_dict = pickle.load(f)

    reversed_dict = dict(zip(word_dict.values(), word_dict.keys()))
    return word_dict, reversed_dict, article_list, title_list 
Example 29
Project: deep-learning-note   Author: wdxtub   File: preprocessing.py    MIT License 5 votes vote down vote up
def main():
    start = datetime.datetime.now()
    print("start", start)
    with tf.Session() as sess:
        processed_data = create_image_lists(sess, TEST_PERCENTAGE, VALIDATION_PERCENTAGE)
        # 通过 numpy 格式保存处理后的数据
        np.save(OUTPUT_FILE, processed_data)
    end = datetime.datetime.now()
    print("done", end) 
Example 30
Project: DJFeet   Author: libre-man   File: pickers.py    MIT License 5 votes vote down vote up
def process_song_file(mfcc_amount, cache_dir, song_file):
        """Process the given ``song_file``.

        This is done by calculating its ``mfcc`` and its tempo and storing this
        information in the given cache_dir.

        :param str song_file: The wav file of the song to process.
        :param int mfcc_amount: The amount of mfcc's to calculate.
        :param str cache_dir: The directory to save the cached properties in.
        :return: A tuple of the mfcc and tempo in this order.
        :rtype: tuple
        """
        l.info("Loading MFCC and tempo variables from %s.", song_file)
        mfcc, tempo = NCAPicker.get_mfcc_and_tempo(song_file, mfcc_amount)
        l.debug("Loaded mfcc and tempo. Writing mfcc.")

        filename, _ = os.path.splitext(os.path.basename(song_file))
        cache_file = os.path.join(cache_dir, filename)
        numpy.save(cache_file + "_mfcc", mfcc)
        l.debug("Done writing mfcc, writing tempo.")
        numpy.save(cache_file + "_tempo", tempo)
        l.debug("Done writing tempo. Touching done file.")
        with open(cache_file + "_done", "w+"):
            pass
        l.info("Done with processing %s.", song_file)

        return mfcc, tempo 
Example 31
Project: AnisotropicMultiStreamCNN   Author: AnnekeMeyer   File: generateTrainingData.py    MIT License 5 votes vote down vote up
def generateFolds(directory, foldDir, nrSplits = 5):
    from sklearn.model_selection import KFold

    X = os.listdir(directory)
    X = [x for x in X if '_deformed' not in x]
    kf = KFold(n_splits=nrSplits, shuffle=True, random_state = 5)
    i=1
    for train, test in kf.split(X):
        train_data = np.array(X)[train]
        test_data = np.array(X)[test]
        print('train', train_data.shape, train_data)
        print('test', test_data.shape, test_data)
        np.save(foldDir+'/train_fold'+str(i), train_data)
        np.save(foldDir+'/test_fold' + str(i), test_data)
        i = i+1 
Example 32
Project: Scene-Understanding   Author: foamliu   File: class_rebal.py    MIT License 5 votes vote down vote up
def smooth_class_prior(sigma=5, do_plot=False):
    prior_prob = np.load(os.path.join(data_dir, "prior_prob.npy"))
    # add an epsilon to prior prob to avoid 0 vakues and possible NaN
    prior_prob += 1E-3 * np.min(prior_prob)
    # renormalize
    prior_prob = prior_prob / (1.0 * np.sum(prior_prob))

    # Smooth with gaussian
    f = interp1d(np.arange(prior_prob.shape[0]), prior_prob)
    xx = np.linspace(0, prior_prob.shape[0] - 1, 1000)
    yy = f(xx)
    window = gaussian(2000, sigma)  # 2000 pts in the window, sigma=5
    smoothed = convolve(yy, window / window.sum(), mode='same')
    fout = interp1d(xx, smoothed)
    prior_prob_smoothed = np.array([fout(i) for i in range(prior_prob.shape[0])])
    prior_prob_smoothed = prior_prob_smoothed / np.sum(prior_prob_smoothed)

    # Save
    file_name = os.path.join(data_dir, "prior_prob_smoothed.npy")
    np.save(file_name, prior_prob_smoothed)

    if do_plot:
        plt.plot(prior_prob)
        plt.plot(prior_prob_smoothed, "g--")
        plt.plot(xx, smoothed, "r-")
        plt.yscale("log")
        plt.show() 
Example 33
Project: keras-anomaly-detection   Author: chen0040   File: convolutional.py    MIT License 5 votes vote down vote up
def fit(self, dataset, model_dir_path, batch_size=8, epochs=100, validation_split=0.1, metric='mean_absolute_error',
            estimated_negative_sample_ratio=0.9):

        self.time_window_size = dataset.shape[1]
        self.metric = metric

        input_timeseries_dataset = np.expand_dims(dataset, axis=2)

        weight_file_path = self.get_weight_file(model_dir_path=model_dir_path)
        architecture_file_path = self.get_architecture_file(model_dir_path)
        checkpoint = ModelCheckpoint(weight_file_path)
        self.model = self.create_model(self.time_window_size, metric=self.metric)
        open(architecture_file_path, 'w').write(self.model.to_json())
        history = self.model.fit(x=input_timeseries_dataset, y=dataset,
                                 batch_size=batch_size, epochs=epochs,
                                 verbose=self.VERBOSE, validation_split=validation_split,
                                 callbacks=[checkpoint]).history
        self.model.save_weights(weight_file_path)

        scores = self.predict(dataset)
        scores.sort()
        cut_point = int(estimated_negative_sample_ratio * len(scores))
        self.threshold = scores[cut_point]

        print('estimated threshold is ' + str(self.threshold))

        self.config = dict()
        self.config['time_window_size'] = self.time_window_size
        self.config['metric'] = self.metric
        self.config['threshold'] = self.threshold
        config_file_path = self.get_config_file(model_dir_path=model_dir_path)
        np.save(config_file_path, self.config)

        return history 
Example 34
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_forward.py    Apache License 2.0 5 votes vote down vote up
def _dump_images(shape):
    import skimage.io
    import skimage.transform
    img_list = []
    for img in sorted(os.listdir('data/test_images/')):
        img = skimage.io.imread('data/test_images/'+img)
        short_egde = min(img.shape[:2])
        yy = int((img.shape[0] - short_egde) / 2)
        xx = int((img.shape[1] - short_egde) / 2)
        img = img[yy : yy + short_egde, xx : xx + short_egde]
        img = skimage.transform.resize(img, shape)
        img_list.append(img)
    imgs = np.asarray(img_list, dtype=np.float32).transpose((0, 3, 1, 2)) - 128
    np.save('data/test_images_%d_%d.npy'%shape, imgs) 
Example 35
Project: spacesense   Author: spacesense-ai   File: training_data.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def save_as_npy(dataset, save_folder=os.getcwd(), file_name='data'):
        path = os.path.join(save_folder, file_name)
        np.save(path, dataset)
        print('dataset saved in .npy format at this location:', save_folder) 
Example 36
Project: spacesense   Author: spacesense-ai   File: datasets.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def save_as_npy(self, save_folder=os.getcwd()):
        np.save(save_folder + '/data', self.data)
        print('dataset saved in .npy format at this location:', save_folder) 
Example 37
Project: spacesense   Author: spacesense-ai   File: datasets.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def save_as_npy(self, data, save_path=os.getcwd() + '/data'):
        np.save(save_path, self.data)
        print('dataset saved in .npy format at this location:', save_path) 
Example 38
Project: DBQA-KBQA   Author: Lucien-qiang   File: extract_embeddings.py    Apache License 2.0 5 votes vote down vote up
def main():
  np.random.seed(123)

  data_dirs = [
              'TRAIN',
              'TRAIN-ALL',
              ]

  for data_dir in data_dirs:
    fname_vocab = os.path.join(data_dir, 'vocab.pickle')
    alphabet = cPickle.load(open(fname_vocab))
    words = alphabet.keys()
    print "Vocab size", len(alphabet)

    for fname in [
                  'embeddings/aquaint+wiki.txt.gz.ndim=50.bin',
                  ]:
      word2vec = load_bin_vec(fname, words)


      ndim = len(word2vec[word2vec.keys()[0]])
      print 'ndim', ndim

      random_words_count = 0
      vocab_emb = np.zeros((len(alphabet) + 1, ndim))
      for word, idx in alphabet.iteritems():
        word_vec = word2vec.get(word, None)
        if word_vec is None:
          word_vec = np.random.uniform(-0.25, 0.25, ndim)

          random_words_count += 1
        vocab_emb[idx] = word_vec
      print "Using zero vector as random"
      print 'random_words_count', random_words_count
      print vocab_emb.shape
      outfile = os.path.join(data_dir, 'emb_{}.npy'.format(os.path.basename(fname)))
      print outfile
      np.save(outfile, vocab_emb) 
Example 39
Project: Multi-modal-learning   Author: vanya2v   File: infer.py    Apache License 2.0 5 votes vote down vote up
def infer(args):
    s = tf.Session()

    filenames = pd.read_csv(args.csv, dtype=str).as_matrix()

    inputs, outputs = ResNetFCN.load(args.model_path, s)

    r = reader.OrgansReader([tf.float32, tf.int32],[[None, None, None, 1], [None, None, None]]) #,name='val_queue')

    
    for f in filenames:

        x, y = r._read_sample([f], is_training=False)

        sw = SlidingWindow(x.shape[1:4], [64, 64, 64], striding=[64, 64, 64])

        # Allocate the prediction output and a counter for averaging probabilities
        y_prob = np.zeros(y.shape + (num_classes,))
        y_pred_count = np.zeros_like(y_prob)
        for slicer in sw:
            y_sw = s.run(outputs['y_prob'], feed_dict={inputs[0]: x[slicer]})
            y_prob[slicer] += y_sw
            y_pred_count[slicer] += 1

        y_prob /= y_pred_count
        
        y_ = np.argmax(y_prob, axis=-1)

        dscs = metrics.dice(y_, y, num_classes)
        
        print(f[0] + ';  mean DSC = {:.3f}\n\t'.format(np.mean(dscs[1:]))
              + ', '.join(['DSC {}: {:.3f}'.format(i, dsc) for i, dsc in enumerate(dscs)]))

        y_ = np.squeeze (y_, axis = 0)

        itk_prediction = sitk.GetImageFromArray(y_)
        ds = np.transpose(dscs)
        DSC_all.append(ds)

    np.save('DSC_MR.npy', DSC_all) 
Example 40
Project: DOTA_models   Author: ringringyi   File: vocabulary_expansion.py    Apache License 2.0 5 votes vote down vote up
def main(unused_argv):
  if not FLAGS.skip_thoughts_model:
    raise ValueError("--skip_thoughts_model is required.")
  if not FLAGS.skip_thoughts_vocab:
    raise ValueError("--skip_thoughts_vocab is required.")
  if not FLAGS.word2vec_model:
    raise ValueError("--word2vec_model is required.")
  if not FLAGS.output_dir:
    raise ValueError("--output_dir is required.")

  if not tf.gfile.IsDirectory(FLAGS.output_dir):
    tf.gfile.MakeDirs(FLAGS.output_dir)

  # Load the skip-thoughts embeddings and vocabulary.
  skip_thoughts_emb = _load_skip_thoughts_embeddings(FLAGS.skip_thoughts_model)
  skip_thoughts_vocab = _load_vocabulary(FLAGS.skip_thoughts_vocab)

  # Load the Word2Vec model.
  word2vec = gensim.models.Word2Vec.load_word2vec_format(
      FLAGS.word2vec_model, binary=True)

  # Run vocabulary expansion.
  embedding_map = _expand_vocabulary(skip_thoughts_emb, skip_thoughts_vocab,
                                     word2vec)

  # Save the output.
  vocab = embedding_map.keys()
  vocab_file = os.path.join(FLAGS.output_dir, "vocab.txt")
  with tf.gfile.GFile(vocab_file, "w") as f:
    f.write("\n".join(vocab))
  tf.logging.info("Wrote vocabulary file to %s", vocab_file)

  embeddings = np.array(embedding_map.values())
  embeddings_file = os.path.join(FLAGS.output_dir, "embeddings.npy")
  np.save(embeddings_file, embeddings)
  tf.logging.info("Wrote embeddings file to %s", embeddings_file) 
Example 41
Project: DOTA_models   Author: ringringyi   File: input.py    Apache License 2.0 5 votes vote down vote up
def extract_mnist_labels(filename, num_images):
  """
  Extract the labels into a vector of int64 label IDs.
  """
  # if not os.path.exists(file):
  if not tf.gfile.Exists(filename+".npy"):
    with gzip.open(filename) as bytestream:
      bytestream.read(8)
      buf = bytestream.read(1 * num_images)
      labels = np.frombuffer(buf, dtype=np.uint8).astype(np.int32)
      np.save(filename, labels)
    return labels
  else:
    with tf.gfile.Open(filename+".npy", mode='r') as file_obj:
      return np.load(file_obj) 
Example 42
Project: DOTA_models   Author: ringringyi   File: train_student.py    Apache License 2.0 5 votes vote down vote up
def train_student(dataset, nb_teachers):
  """
  This function trains a student using predictions made by an ensemble of
  teachers. The student and teacher models are trained using the same
  neural network architecture.
  :param dataset: string corresponding to mnist, cifar10, or svhn
  :param nb_teachers: number of teachers (in the ensemble) to learn from
  :return: True if student training went well
  """
  assert input.create_dir_if_needed(FLAGS.train_dir)

  # Call helper function to prepare student data using teacher predictions
  stdnt_dataset = prepare_student_data(dataset, nb_teachers, save=True)

  # Unpack the student dataset
  stdnt_data, stdnt_labels, stdnt_test_data, stdnt_test_labels = stdnt_dataset

  # Prepare checkpoint filename and path
  if FLAGS.deeper:
    ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + str(nb_teachers) + '_student_deeper.ckpt' #NOLINT(long-line)
  else:
    ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + str(nb_teachers) + '_student.ckpt'  # NOLINT(long-line)

  # Start student training
  assert deep_cnn.train(stdnt_data, stdnt_labels, ckpt_path)

  # Compute final checkpoint name for student (with max number of steps)
  ckpt_path_final = ckpt_path + '-' + str(FLAGS.max_steps - 1)

  # Compute student label predictions on remaining chunk of test set
  student_preds = deep_cnn.softmax_preds(stdnt_test_data, ckpt_path_final)

  # Compute teacher accuracy
  precision = metrics.accuracy(student_preds, stdnt_test_labels)
  print('Precision of student after training: ' + str(precision))

  return True 
Example 43
Project: DOTA_models   Author: ringringyi   File: lm_1b_eval.py    Apache License 2.0 5 votes vote down vote up
def _DumpSentenceEmbedding(sentence, vocab):
  """Predict next words using the given prefix words.

  Args:
    sentence: Sentence words.
    vocab: Vocabulary. Contains max word chard id length and converts between
        words and ids.
  """
  targets = np.zeros([BATCH_SIZE, NUM_TIMESTEPS], np.int32)
  weights = np.ones([BATCH_SIZE, NUM_TIMESTEPS], np.float32)

  sess, t = _LoadModel(FLAGS.pbtxt, FLAGS.ckpt)

  if sentence.find('<S>') != 0:
    sentence = '<S> ' + sentence

  word_ids = [vocab.word_to_id(w) for w in sentence.split()]
  char_ids = [vocab.word_to_char_ids(w) for w in sentence.split()]

  inputs = np.zeros([BATCH_SIZE, NUM_TIMESTEPS], np.int32)
  char_ids_inputs = np.zeros(
      [BATCH_SIZE, NUM_TIMESTEPS, vocab.max_word_length], np.int32)
  for i in xrange(len(word_ids)):
    inputs[0, 0] = word_ids[i]
    char_ids_inputs[0, 0, :] = char_ids[i]

    # Add 'lstm/lstm_0/control_dependency' if you want to dump previous layer
    # LSTM.
    lstm_emb = sess.run(t['lstm/lstm_1/control_dependency'],
                        feed_dict={t['char_inputs_in']: char_ids_inputs,
                                   t['inputs_in']: inputs,
                                   t['targets_in']: targets,
                                   t['target_weights_in']: weights})

    fname = os.path.join(FLAGS.save_dir, 'lstm_emb_step_%d.npy' % i)
    with tf.gfile.Open(fname, mode='w') as f:
      np.save(f, lstm_emb)
    sys.stderr.write('LSTM embedding step %d file saved\n' % i) 
Example 44
Project: Pytorch-Project-Template   Author: moemen95   File: generate_class_weights.py    MIT License 5 votes vote down vote up
def calculate_weigths_labels():
    class Config:
        mode = "train"
        num_classes = 21
        batch_size = 32
        max_epoch = 150

        validate_every = 2

        checkpoint_file = "checkpoint.pth.tar"

        data_loader = "VOCDataLoader"
        data_root = "../data/pascal_voc_seg/"
        data_loader_workers = 4
        pin_memory = True
        async_loading = True

    # Create an instance from the data loader
    from tqdm import tqdm
    data_loader = VOCDataLoader(Config)
    z = np.zeros((Config.num_classes,))
    # Initialize tqdm
    tqdm_batch = tqdm(data_loader.train_loader, total=data_loader.train_iterations)

    for _, y in tqdm_batch:
        labels = y.numpy().astype(np.uint8).ravel().tolist()
        z += np.bincount(labels, minlength=Config.num_classes)
    tqdm_batch.close()
    # ret = compute_class_weight(class_weight='balanced', classes=np.arange(21), y=np.asarray(labels, dtype=np.uint8))
    total_frequency = np.sum(z)
    print(z)
    print(total_frequency)
    class_weights = []
    for frequency in z:
        class_weight = 1 / (np.log(1.02 + (frequency / total_frequency)))
        class_weights.append(class_weight)
    ret = np.array(class_weights)
    np.save('../pretrained_weights/voc2012_256_class_weights', ret)
    print(ret) 
Example 45
Project: GCN-VAE-opinion   Author: zxj32   File: generate_distribution.py    MIT License 5 votes vote down vote up
def get_dc_dis_uncertain():
    b_all = np.load("/network/rit/lab/ceashpc/xujiang/project/GAE_TEST/gae/traffic_data/dc_belief_T38_0.8.npy")
    u_all = np.load("/network/rit/lab/ceashpc/xujiang/project/GAE_TEST/gae/traffic_data/dc_uncertain_T38_0.8.npy")
    dis = np.zeros([180, 39])
    for i in range(len(u_all)):
        ui = u_all[i]
        bi = b_all[i]
        obs = get_omega(bi, ui)
        cb = Counter(obs)
        for k in range(39):
            dis[i][k] = cb[k+2]
    np.save("/network/rit/lab/ceashpc/xujiang/traffic_data/DC_data/distribution_DC.npy", dis)
    return dis 
Example 46
Project: GCN-VAE-opinion   Author: zxj32   File: generate_distribution.py    MIT License 5 votes vote down vote up
def get_pa_dis_uncertain():
    b_all = np.load("/network/rit/lab/ceashpc/xujiang/project/GAE_TEST/gae/traffic_data/pa_belief_T38_0.8.npy")
    u_all = np.load("/network/rit/lab/ceashpc/xujiang/project/GAE_TEST/gae/traffic_data/pa_uncertain_T38_0.8.npy")
    dis = np.zeros([180, 39])
    for i in range(len(u_all)):
        ui = u_all[i]
        bi = b_all[i]
        obs = get_omega(bi, ui)
        cb = Counter(obs)
        for k in range(39):
            dis[i][k] = cb[k+2]
    np.save("/network/rit/lab/ceashpc/xujiang/traffic_data/PA_data/distribution_PA.npy", dis)
    return dis 
Example 47
Project: rasa-faq-bot   Author: nghuyong   File: actions.py    MIT License 5 votes vote down vote up
def encode_standard_question():
    bc = BertClient()
    data = json.load(open("./data/nlu/faq.json", "rt", encoding="utf-8"))
    standard_questions = [each['q'] for each in data]
    print("Standard question size", len(standard_questions))
    print("Start to calculate encoder....")
    standard_questions_encoder = bc.encode(standard_questions)
    np.save("./data/standard_questions", standard_questions_encoder)
    standard_questions_encoder_len = np.sqrt(np.sum(standard_questions_encoder * standard_questions_encoder, axis=1))
    np.save("./data/standard_questions_len", standard_questions_encoder_len) 
Example 48
Project: QCElemental   Author: MolSSI   File: molecule.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def to_file(self, filename: str, dtype: Optional[str] = None) -> None:
        """Writes the Molecule to a file.

        Parameters
        ----------
        filename : str
            The filename to write to
        dtype : Optional[str], optional
            The type of file to write, attempts to infer dtype from the filename if not provided.

        """
        ext = Path(filename).suffix

        if dtype is None:
            if ext in _extension_map:
                dtype = _extension_map[ext]
            else:
                raise KeyError(f"Could not infer dtype from filename: `{filename}`")

        flags = "w"
        if dtype in ["xyz", "xyz+", "psi4"]:
            stringified = self.to_string(dtype)
        elif dtype in ["json"]:
            stringified = self.serialize("json")
        elif dtype in ["msgpack", "msgpack-ext"]:
            stringified = self.serialize("msgpack-ext")
            flags = "wb"
        elif dtype in ["numpy"]:
            elements = np.array(self.atomic_numbers).reshape(-1, 1)
            npmol = np.hstack((elements, self.geometry * constants.conversion_factor("bohr", "angstroms")))
            np.save(filename, npmol)
            return

        else:
            raise KeyError(f"Dtype `{dtype}` is not valid")

        with open(filename, flags) as handle:
            handle.write(stringified)

    ### Non-Pydantic internal functions 
Example 49
Project: fine-lm   Author: akzaidi   File: video_metrics.py    MIT License 5 votes vote down vote up
def save_results(results, output_dir, problem_name):
  for name, array in six.iteritems(results):
    output_filename = "{}_{}.npy".format(problem_name, name)
    output_filename = os.path.join(output_dir, output_filename)
    with tf.gfile.Open(output_filename, "wb") as fname:
      np.save(fname, array) 
Example 50
Project: FakeReviewGenerator   Author: amitabhnag   File: utils.py    MIT License 5 votes vote down vote up
def preprocess(self, input_file, vocab_file, tensor_file, encoding):
        """Given an input text file, generate arrays of tensors.

        Args:
            input_file: input text file.
            vocab_file: where to store the vocab.pkl file.
            tensor_file: where to store the data.npy file.
            encoding: encoding config to read the text file if any.
        """
        with codecs.open(input_file, "r", encoding=encoding) as f:
            data = f.read()

        # Optional text cleaning or make them lower case, etc.
        # data = self.clean_str(data)
        x_text = data.split()

        self.vocab, self.words = self.build_vocab(x_text)
        self.vocab_size = len(self.words)

        with open(vocab_file, 'wb') as f:
            cPickle.dump(self.words, f)

        # The same operation like this [self.vocab[word] for word in x_text]
        # index of words as our basic data
        self.tensor = np.array(list(map(self.vocab.get, x_text)))
        # Save the data to data.npy
        np.save(tensor_file, self.tensor)