Python numpy.load() Examples
The following are 30 code examples for showing how to use numpy.load(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.
You may check out the related API usage on the sidebar.
You may also want to check out all available functions/classes of the module
numpy
, or try the search function
.
Example 1
Project: vergeml Author: mme File: cache.py License: MIT License | 10 votes |
def _deserialize(self, data, type_): if self.compress: # decompress the data if needed data = lz4.frame.decompress(data) if type_ == _NUMPY: # deserialize numpy arrays buf = io.BytesIO(data) data = np.load(buf) elif type_ == _PICKLE: # deserialize other python objects data = pickle.loads(data) else: # Otherwise we just return data as it is (bytes) pass return data
Example 2
Project: Caffe-Python-Data-Layer Author: liuxianming File: BasePythonDataLayer.py License: BSD 2-Clause "Simplified" License | 6 votes |
def setup(self, bottom, top): layer_params = yaml.load(self.param_str) self._layer_params = layer_params # default batch_size = 256 self._batch_size = int(layer_params.get('batch_size', 256)) self._resize = layer_params.get('resize', -1) self._mean_file = layer_params.get('mean_file', None) self._source_type = layer_params.get('source_type', 'CSV') self._shuffle = layer_params.get('shuffle', False) # read image_mean from file and preload all data into memory # will read either file or array into self._mean self.set_mean() self.preload_db() self._compressed = self._layer_params.get('compressed', True) if not self._compressed: self.decompress_data()
Example 3
Project: Caffe-Python-Data-Layer Author: liuxianming File: BasePythonDataLayer.py License: BSD 2-Clause "Simplified" License | 6 votes |
def set_mean(self): if self._mean_file: if type(self._mean_file) is str: # read image mean from file try: # if it is a pickle file self._mean = np.load(self._mean_file) except (IOError): blob = caffe_pb2.BlobProto() blob_str = open(self._mean_file, 'rb').read() blob.ParseFromString(blob_str) self._mean = np.array(caffe.io.blobproto_to_array(blob))[0] else: self._mean = self._mean_file self._mean = np.array(self._mean) else: self._mean = None
Example 4
Project: face-attendance-machine Author: matiji66 File: encoding_images.py License: Apache License 2.0 | 6 votes |
def load_encodings(): """ 加载保存的历史人脸向量,以及name向量,并返回 :return: """ known_face_encodings = np.load(KNOWN_FACE_ENCODINGS) known_face_names = np.load(KNOWN_FACE_NANE) if not os.path.exists(KNOWN_FACE_NANE) or not os.path.exists(KNOWN_FACE_ENCODINGS): encoding_images(data_path) aa = [file for file in os.listdir(data_path) if os.path.isfile(os.path.join(data_path, file)) and file.endswith("npy")] # ("known_face_encodings_") or file.startswith("known_face_name_")) for data in aa: if data.startswith('known_face_encodings_'): tmp_face_encodings = np.load(os.path.join(data_path,data)) known_face_encodings = np.concatenate((known_face_encodings, tmp_face_encodings), axis=0) print("load ", data) elif data.startswith('known_face_name_'): tmp_face_name = np.load(os.path.join(data_path, data)) known_face_names = np.concatenate((known_face_names, tmp_face_name), axis=0) print("load ", data) else: print('skip to load original ', data) return known_face_encodings,known_face_names
Example 5
Project: disentangling_conditional_gans Author: zalandoresearch File: dataset_tool.py License: MIT License | 6 votes |
def create_cifar100(tfrecord_dir, cifar100_dir): print('Loading CIFAR-100 from "%s"' % cifar100_dir) import pickle with open(os.path.join(cifar100_dir, 'train'), 'rb') as file: data = pickle.load(file, encoding='latin1') images = data['data'].reshape(-1, 3, 32, 32) labels = np.array(data['fine_labels']) assert images.shape == (50000, 3, 32, 32) and images.dtype == np.uint8 assert labels.shape == (50000,) and labels.dtype == np.int32 assert np.min(images) == 0 and np.max(images) == 255 assert np.min(labels) == 0 and np.max(labels) == 99 onehot = np.zeros((labels.size, np.max(labels) + 1), dtype=np.float32) onehot[np.arange(labels.size), labels] = 1.0 with TFRecordExporter(tfrecord_dir, images.shape[0]) as tfr: order = tfr.choose_shuffled_order() for idx in range(order.size): tfr.add_image(images[order[idx]]) tfr.add_labels(onehot[order]) #----------------------------------------------------------------------------
Example 6
Project: dustmaps Author: gregreen File: json_serializers.py License: GNU General Public License v2.0 | 6 votes |
def deserialize_ndarray_npy(d): """ Deserializes a JSONified :obj:`numpy.ndarray` that was created using numpy's :obj:`save` function. Args: d (:obj:`dict`): A dictionary representation of an :obj:`ndarray` object, created using :obj:`numpy.save`. Returns: An :obj:`ndarray` object. """ with io.BytesIO() as f: f.write(json.loads(d['npy']).encode('latin-1')) f.seek(0) return np.load(f)
Example 7
Project: VSE-C Author: ExplorerFreda File: data.py License: MIT License | 6 votes |
def __init__(self, data_path, data_split, vocab, cap_suffix='caps'): self.vocab = vocab loc = data_path + '/' # Captions self.captions = [] with open(loc+'%s_%s.txt' % (data_split, cap_suffix), 'rb') as f: for line in f: tmp = line.strip() if type(tmp) == bytes: tmp = bytes.decode(tmp) self.captions.append(tmp) # Image features self.images = np.load(loc+'%s_ims.npy' % data_split) self.length = len(self.captions) # rkiros data has redundancy in images, we divide by 5, 10crop doesn't if self.images.shape[0] != self.length: self.im_div = 5 else: self.im_div = 1 # the development set for coco is large and so validation would be slow if data_split == 'dev': self.length = 5000
Example 8
Project: dynamic-training-with-apache-mxnet-on-aws Author: awslabs File: data_loader.py License: Apache License 2.0 | 6 votes |
def load_mnist(training_num=50000): data_path = os.path.join(os.path.dirname(os.path.realpath('__file__')), 'mnist.npz') if not os.path.isfile(data_path): from six.moves import urllib origin = ( 'https://github.com/sxjscience/mxnet/raw/master/example/bayesian-methods/mnist.npz' ) print('Downloading data from %s to %s' % (origin, data_path)) ctx = ssl._create_unverified_context() with urllib.request.urlopen(origin, context=ctx) as u, open(data_path, 'wb') as f: f.write(u.read()) print('Done!') dat = numpy.load(data_path) X = (dat['X'][:training_num] / 126.0).astype('float32') Y = dat['Y'][:training_num] X_test = (dat['X_test'] / 126.0).astype('float32') Y_test = dat['Y_test'] Y = Y.reshape((Y.shape[0],)) Y_test = Y_test.reshape((Y_test.shape[0],)) return X, Y, X_test, Y_test
Example 9
Project: dynamic-training-with-apache-mxnet-on-aws Author: awslabs File: utils.py License: Apache License 2.0 | 6 votes |
def load_params(dir_path="", epoch=None, name=""): prefix = os.path.join(dir_path, name) _, param_loading_path, _ = get_saving_path(prefix, epoch) while not os.path.isfile(param_loading_path): logging.info("in load_param, %s Not Found!" % param_loading_path) time.sleep(60) save_dict = nd.load(param_loading_path) arg_params = {} aux_params = {} for k, v in save_dict.items(): tp, name = k.split(':', 1) if tp == 'arg': arg_params[name] = v if tp == 'aux': aux_params[name] = v return arg_params, aux_params, param_loading_path
Example 10
Project: dynamic-training-with-apache-mxnet-on-aws Author: awslabs File: test_forward.py License: Apache License 2.0 | 6 votes |
def test_consistency(dump=False): shape = (299, 299) _get_model() _get_data(shape) if dump: _dump_images(shape) gt = None else: gt = {n: mx.nd.array(a) for n, a in np.load('data/inception-v3-dump.npz').items()} data = np.load('data/test_images_%d_%d.npy'%shape) sym, arg_params, aux_params = mx.model.load_checkpoint('model/Inception-7', 1) arg_params['data'] = data arg_params['softmax_label'] = np.random.randint(low=1, high=1000, size=(data.shape[0],)) ctx_list = [{'ctx': mx.gpu(0), 'data': data.shape, 'type_dict': {'data': data.dtype}}, {'ctx': mx.cpu(0), 'data': data.shape, 'type_dict': {'data': data.dtype}}] gt = check_consistency(sym, ctx_list, arg_params=arg_params, aux_params=aux_params, tol=1e-3, grad_req='null', raise_on_err=False, ground_truth=gt) if dump: np.savez('data/inception-v3-dump.npz', **{n: a.asnumpy() for n, a in gt.items()})
Example 11
Project: DOTA_models Author: ringringyi File: input.py License: Apache License 2.0 | 6 votes |
def extract_mnist_data(filename, num_images, image_size, pixel_depth): """ Extract the images into a 4D tensor [image index, y, x, channels]. Values are rescaled from [0, 255] down to [-0.5, 0.5]. """ # if not os.path.exists(file): if not tf.gfile.Exists(filename+".npy"): with gzip.open(filename) as bytestream: bytestream.read(16) buf = bytestream.read(image_size * image_size * num_images) data = np.frombuffer(buf, dtype=np.uint8).astype(np.float32) data = (data - (pixel_depth / 2.0)) / pixel_depth data = data.reshape(num_images, image_size, image_size, 1) np.save(filename, data) return data else: with tf.gfile.Open(filename+".npy", mode='r') as file_obj: return np.load(file_obj)
Example 12
Project: cvpr2018-hnd Author: kibok90 File: preparation.py License: MIT License | 6 votes |
def is_image_file(id, dataset, dtype, filename): filename_lower = filename.lower() if any(filename_lower.endswith(ext) for ext in IMG_EXTENSIONS): if dtype == 'novel': try: default_loader(filename) return True except OSError: print('{filename} failed to load'.format(filename=filename)) with open('taxonomy/{dataset}/corrupted_{dtype}_{id:d}.txt' \ .format(dataset=dataset, dtype=dtype, id=id), 'a') as f: f.write(filename + '\n') return False else: return True else: return False
Example 13
Project: medicaldetectiontoolkit Author: MIC-DKFZ File: data_loader.py License: Apache License 2.0 | 6 votes |
def generate_train_batch(self): batch_data, batch_segs, batch_pids, batch_targets = [], [], [], [] class_targets_list = [v['class_target'] for (k, v) in self._data.items()] #samples patients towards equilibrium of foreground classes on a roi-level (after randomly sampling the ratio "batch_sample_slack). batch_ixs = dutils.get_class_balanced_patients( class_targets_list, self.batch_size, self.cf.head_classes - 1, slack_factor=self.cf.batch_sample_slack) patients = list(self._data.items()) for b in batch_ixs: patient = patients[b][1] all_data = np.load(patient['data'], mmap_mode='r') data = all_data[0] seg = all_data[1].astype('uint8') batch_pids.append(patient['pid']) batch_targets.append(patient['class_target']) batch_data.append(data[np.newaxis]) batch_segs.append(seg[np.newaxis]) data = np.array(batch_data) seg = np.array(batch_segs).astype(np.uint8) class_target = np.array(batch_targets) return {'data': data, 'seg': seg, 'pid': batch_pids, 'class_target': class_target}
Example 14
Project: Black-Box-Audio Author: rtaori File: tf_logits.py License: MIT License | 5 votes |
def compute_mfcc(audio, **kwargs): """ Compute the MFCC for a given audio waveform. This is identical to how DeepSpeech does it, but does it all in TensorFlow so that we can differentiate through it. """ batch_size, size = audio.get_shape().as_list() audio = tf.cast(audio, tf.float32) # 1. Pre-emphasizer, a high-pass filter audio = tf.concat((audio[:, :1], audio[:, 1:] - 0.97*audio[:, :-1], np.zeros((batch_size,1000),dtype=np.float32)), 1) # 2. windowing into frames of 320 samples, overlapping windowed = tf.stack([audio[:, i:i+400] for i in range(0,size-320,160)],1) # 3. Take the FFT to convert to frequency space ffted = tf.spectral.rfft(windowed, [512]) ffted = 1.0 / 512 * tf.square(tf.abs(ffted)) # 4. Compute the Mel windowing of the FFT energy = tf.reduce_sum(ffted,axis=2)+1e-30 filters = np.load("filterbanks.npy").T feat = tf.matmul(ffted, np.array([filters]*batch_size,dtype=np.float32))+1e-30 # 5. Take the DCT again, because why not feat = tf.log(feat) feat = tf.spectral.dct(feat, type=2, norm='ortho')[:,:,:26] # 6. Amplify high frequencies for some reason _,nframes,ncoeff = feat.get_shape().as_list() n = np.arange(ncoeff) lift = 1 + (22/2.)*np.sin(np.pi*n/22) feat = lift*feat width = feat.get_shape().as_list()[1] # 7. And now stick the energy next to the features feat = tf.concat((tf.reshape(tf.log(energy),(-1,width,1)), feat[:, :, 1:]), axis=2) return feat
Example 15
Project: vergeml Author: mme File: cache.py License: MIT License | 5 votes |
def read(self, file, path): """Read the content index from file. """ pos, = struct.unpack('<Q', file.read(8)) if pos == 0: raise VergeMLError("Invalid cache file: {}".format(path)) file.seek(pos) self.index, self.meta, self.info = pickle.load(file)
Example 16
Project: DDPAE-video-prediction Author: jthsieh File: bouncing_balls.py License: MIT License | 5 votes |
def make_dataset(root, is_train): if is_train: folder = 'balls_n4_t60_ex50000' else: folder = 'balls_n4_t60_ex2000' dataset = np.load(os.path.join(root, folder, 'dataset_info.npy')) return dataset
Example 17
Project: DDPAE-video-prediction Author: jthsieh File: moving_mnist.py License: MIT License | 5 votes |
def load_fixed_set(root, is_train): # Load the fixed dataset filename = 'mnist_test_seq.npy' path = os.path.join(root, filename) dataset = np.load(path) dataset = dataset[..., np.newaxis] return dataset
Example 18
Project: FRIDA Author: LCAV File: utils.py License: MIT License | 5 votes |
def load_dirac_param(file_name): """ load stored Diracs' parameters :param file_name: the file name that the parameters are stored :return: """ stored_param = np.load(file_name) alpha_ks = stored_param['alpha_ks'] phi_ks = stored_param['phi_ks'] time_stamp = stored_param['time_stamp'].tostring() return alpha_ks, phi_ks, time_stamp
Example 19
Project: FRIDA Author: LCAV File: utils.py License: MIT License | 5 votes |
def load_mic_array_param(file_name): """ load stored microphone array parameters :param file_name: file that stored these parameters :return: """ stored_param = np.load(file_name) pos_mic_x = stored_param['pos_mic_x'] pos_mic_y = stored_param['pos_mic_y'] layout_time_stamp = stored_param['layout_time_stamp'].tostring() return pos_mic_x, pos_mic_y, layout_time_stamp
Example 20
Project: gated-graph-transformer-network Author: hexahedria File: display_graph.py License: MIT License | 5 votes |
def main(visdir): results = [] has_answer = os.path.isfile("{}/result_{}.npy".format(visdir,4)) the_range = range(1,5) if has_answer else range(4) results = [np.load("{}/result_{}.npy".format(visdir,i)) for i in the_range] import importlib.machinery try: options_mod = importlib.machinery.SourceFileLoader('options',os.path.join(visdir,"options.py")).load_module() options = options_mod.options except FileNotFoundError: options = {} print(prep_graph_display(results,options))
Example 21
Project: image_to_numpy Author: ageitgey File: test_load_image_file.py License: MIT License | 5 votes |
def test_jpeg_rotation(self): # Make sure all Portrait test images are auto-rotated correctly for i in range(9): img_jpg = load_image_file(f"Portrait_{i}.jpg") ref_img = np.load(f"Portrait_{i}.jpg.npy") self.assertTrue(np.array_equal(ref_img, img_jpg)) # Make sure all Landscape test images are auto-rotated correctly for i in range(9): img_jpg = load_image_file(f"Landscape_{i}.jpg") ref_img = np.load(f"Landscape_{i}.jpg.npy") self.assertTrue(np.array_equal(ref_img, img_jpg))
Example 22
Project: image_to_numpy Author: ageitgey File: test_load_image_file.py License: MIT License | 5 votes |
def test_jpeg_no_exif(self): # Can we load a jpeg with no metadata without crashing? img_jpg = load_image_file("Portrait_no_exif.jpg") self.assertEqual(img_jpg.shape, (1200, 1800, 3))
Example 23
Project: image_to_numpy Author: ageitgey File: test_load_image_file.py License: MIT License | 5 votes |
def test_png(self): # Can we load a non-jpeg file with no metadata? img_png = load_image_file("Portrait_8.png") self.assertEqual(img_png.shape, (1800, 1200, 3))
Example 24
Project: disentangling_conditional_gans Author: zalandoresearch File: dataset_tool.py License: MIT License | 5 votes |
def create_cifar10(tfrecord_dir, cifar10_dir): print('Loading CIFAR-10 from "%s"' % cifar10_dir) import pickle images = [] labels = [] for batch in range(1, 6): with open(os.path.join(cifar10_dir, 'data_batch_%d' % batch), 'rb') as file: data = pickle.load(file, encoding='latin1') images.append(data['data'].reshape(-1, 3, 32, 32)) labels.append(data['labels']) images = np.concatenate(images) labels = np.concatenate(labels) assert images.shape == (50000, 3, 32, 32) and images.dtype == np.uint8 assert labels.shape == (50000,) and labels.dtype == np.int32 assert np.min(images) == 0 and np.max(images) == 255 assert np.min(labels) == 0 and np.max(labels) == 9 onehot = np.zeros((labels.size, np.max(labels) + 1), dtype=np.float32) onehot[np.arange(labels.size), labels] = 1.0 with TFRecordExporter(tfrecord_dir, images.shape[0]) as tfr: order = tfr.choose_shuffled_order() for idx in range(order.size): tfr.add_image(images[order[idx]]) tfr.add_labels(onehot[order]) #----------------------------------------------------------------------------
Example 25
Project: disentangling_conditional_gans Author: zalandoresearch File: dataset_tool.py License: MIT License | 5 votes |
def create_svhn(tfrecord_dir, svhn_dir): print('Loading SVHN from "%s"' % svhn_dir) import pickle images = [] labels = [] for batch in range(1, 4): with open(os.path.join(svhn_dir, 'train_%d.pkl' % batch), 'rb') as file: data = pickle.load(file, encoding='latin1') images.append(data[0]) labels.append(data[1]) images = np.concatenate(images) labels = np.concatenate(labels) assert images.shape == (73257, 3, 32, 32) and images.dtype == np.uint8 assert labels.shape == (73257,) and labels.dtype == np.uint8 assert np.min(images) == 0 and np.max(images) == 255 assert np.min(labels) == 0 and np.max(labels) == 9 onehot = np.zeros((labels.size, np.max(labels) + 1), dtype=np.float32) onehot[np.arange(labels.size), labels] = 1.0 with TFRecordExporter(tfrecord_dir, images.shape[0]) as tfr: order = tfr.choose_shuffled_order() for idx in range(order.size): tfr.add_image(images[order[idx]]) tfr.add_labels(onehot[order]) #----------------------------------------------------------------------------
Example 26
Project: disentangling_conditional_gans Author: zalandoresearch File: dataset_tool.py License: MIT License | 5 votes |
def create_from_images(tfrecord_dir, image_dir, label_dir, shuffle): print('Loading images from "%s"' % image_dir) image_filenames = sorted(glob.glob(os.path.join(image_dir, '*'))) if len(image_filenames) == 0: error('No input images found') img = np.asarray(PIL.Image.open(image_filenames[0])) resolution = img.shape[0] channels = img.shape[2] if img.ndim == 3 else 1 if img.shape[1] != resolution: error('Input images must have the same width and height') if resolution != 2 ** int(np.floor(np.log2(resolution))): error('Input image resolution must be a power-of-two') if channels not in [1, 3]: error('Input images must be stored as RGB or grayscale') try: with open(label_dir, 'rb') as file: labels = pickle.load(file) except: error('Label file was not found') with TFRecordExporter(tfrecord_dir, len(image_filenames)) as tfr: order = tfr.choose_shuffled_order() if shuffle else np.arange(len(image_filenames)) reordered_names = [] for idx in range(order.size): image_filename = image_filenames[order[idx]] img = np.asarray(PIL.Image.open(image_filename)) if channels == 1: img = img[np.newaxis, :, :] # HW => CHW else: img = img.transpose(2, 0, 1) # HWC => CHW tfr.add_image(img) reordered_names.append(os.path.basename(image_filename)) reordered_labels = [] for key in reordered_names: reordered_labels += [labels[key]] reordered_labels = np.stack(reordered_labels, 0) tfr.add_labels(reordered_labels) #----------------------------------------------------------------------------
Example 27
Project: disentangling_conditional_gans Author: zalandoresearch File: dataset_tool.py License: MIT License | 5 votes |
def create_from_hdf5(tfrecord_dir, hdf5_filename, shuffle): print('Loading HDF5 archive from "%s"' % hdf5_filename) import h5py # conda install h5py with h5py.File(hdf5_filename, 'r') as hdf5_file: hdf5_data = max([value for key, value in hdf5_file.items() if key.startswith('data')], key=lambda lod: lod.shape[3]) with TFRecordExporter(tfrecord_dir, hdf5_data.shape[0]) as tfr: order = tfr.choose_shuffled_order() if shuffle else np.arange(hdf5_data.shape[0]) for idx in range(order.size): tfr.add_image(hdf5_data[order[idx]]) npy_filename = os.path.splitext(hdf5_filename)[0] + '-labels.npy' if os.path.isfile(npy_filename): tfr.add_labels(np.load(npy_filename)[order]) #----------------------------------------------------------------------------
Example 28
Project: models Author: kipoi File: model.py License: MIT License | 5 votes |
def _load_model(self): w = np.load(self.model_file)['weights'] self.mer6_dict = dict(zip(self.make_mer_list(6), range(4**6))) self.w_mat = w
Example 29
Project: Deep_VoiceChanger Author: pstuvwx File: dataset.py License: MIT License | 5 votes |
def load(path): bps, data = wav.read(path) if len(data.shape) != 1: data = data[:,0] + data[:,1] return bps, data
Example 30
Project: Deep_VoiceChanger Author: pstuvwx File: dataset.py License: MIT License | 5 votes |
def __init__(self, wave, dataset_len, test): self.wave = np.array(load(wave)[1], dtype=float) self.max = len(self.wave)-dif-sride*(3+padding*2) self.length = dataset_len if dataset_len <= 0: self.length = self.max // dif self.window = np.hanning(254) self.test = test