Python numpy.savez_compressed() Examples

def process_training_data(num_clips):
    """
    Processes random training clips from the full training data. Saves to TRAIN_DIR_CLIPS by
    default.

    @param num_clips: The number of clips to process. Default = 5000000 (set in __main__).

    @warning: This can take a couple of hours to complete with large numbers of clips.
    """
    num_prev_clips = len(glob(c.TRAIN_DIR_CLIPS + '*'))

    for clip_num in xrange(num_prev_clips, num_clips + num_prev_clips):
        clip = process_clip()

        np.savez_compressed(c.TRAIN_DIR_CLIPS + str(clip_num), clip)

        if (clip_num + 1) % 100 == 0: print 'Processed %d clips' % (clip_num + 1) 

def write_cache_file(cache_filename, Y, image_IDs):
    """ Writes data to a cache file using np.savez_compressed

    Args:
        cache_filename (str): cache filename
        Y (np.ndarray): data to write to cache file
        image_IDs (iterable): list of image IDs corresponding to data in cache
            file. If not specified, function will not check for correspondence

    """
    np.savez_compressed(cache_filename, **{
        'Y': Y, _image_ID_str: image_IDs
    })


# Cache file updating 

def save_subvolume(labels, origins, output_path, **misc_items):
  """Saves an FFN subvolume.

  Args:
    labels: 3d zyx number array with the segment labels
    origins: dictionary mapping segment ID to origin information
    output_path: path at which to save the segmentation in the form
        of a .npz file
    **misc_items: (optional) additional values to save
        in the output file
  """
  seg = segmentation.reduce_id_bits(labels)
  gfile.MakeDirs(os.path.dirname(output_path))
  with atomic_file(output_path) as fd:
    np.savez_compressed(fd,
                        segmentation=seg,
                        origins=origins,
                        **misc_items) 

def save_checkpoint(self, path):
    """Saves a inference checkpoint to `path`."""
    self.log_info('Saving inference checkpoint to %s.', path)
    with timer_counter(self.counters, 'save_checkpoint'):
      gfile.MakeDirs(os.path.dirname(path))
      with storage.atomic_file(path) as fd:
        seed_policy_state = None
        if self.seed_policy is not None:
          seed_policy_state = self.seed_policy.get_state()

        np.savez_compressed(fd,
                            movement_policy=self.movement_policy.get_state(),
                            segmentation=self.segmentation,
                            seg_qprob=self.seg_prob,
                            seed=self.seed,
                            origins=self.origins,
                            overlaps=self.overlaps,
                            history=np.array(self.history),
                            history_deleted=np.array(self.history_deleted),
                            seed_policy_state=seed_policy_state,
                            counters=self.counters.dumps())
    self.log_info('Inference checkpoint saved.') 

def fill_mesh(mesh2fill, file: str, opt):
    load_path = get_mesh_path(file, opt.num_aug)
    if os.path.exists(load_path):
        mesh_data = np.load(load_path, encoding='latin1', allow_pickle=True)
    else:
        mesh_data = from_scratch(file, opt)
        np.savez_compressed(load_path, gemm_edges=mesh_data.gemm_edges, vs=mesh_data.vs, edges=mesh_data.edges,
                            edges_count=mesh_data.edges_count, ve=mesh_data.ve, v_mask=mesh_data.v_mask,
                            filename=mesh_data.filename, sides=mesh_data.sides,
                            edge_lengths=mesh_data.edge_lengths, edge_areas=mesh_data.edge_areas,
                            features=mesh_data.features)
    mesh2fill.vs = mesh_data['vs']
    mesh2fill.edges = mesh_data['edges']
    mesh2fill.gemm_edges = mesh_data['gemm_edges']
    mesh2fill.edges_count = int(mesh_data['edges_count'])
    mesh2fill.ve = mesh_data['ve']
    mesh2fill.v_mask = mesh_data['v_mask']
    mesh2fill.filename = str(mesh_data['filename'])
    mesh2fill.edge_lengths = mesh_data['edge_lengths']
    mesh2fill.edge_areas = mesh_data['edge_areas']
    mesh2fill.features = mesh_data['features']
    mesh2fill.sides = mesh_data['sides'] 

def combine_levels(directory):
    """
    Merge all files in a single directory.
    """
    files = sorted(glob.glob(os.path.join(directory, '*.npz')))
    all_data = []
    max_name_len = 0
    for file in files:
        with np.load(file) as data:
            name = os.path.split(file)[1]
            max_name_len = max(max_name_len, len(name))
            all_data.append(data.items() + [('name', name)])
    dtype = []
    for key, val in all_data[0][:-1]:
        dtype.append((key, val.dtype, val.shape))
    dtype.append(('name', str, max_name_len))
    combo_data = np.array([
        tuple([val for key, val in data]) for data in all_data
    ], dtype=dtype)
    np.savez_compressed(directory + '.npz', levels=combo_data) 

def save_twiss_file(self, twiss_list):
        if self.tws_file is None:
            tws_file_name = self.output_beam_file.replace("particles", "tws")
        else:
            tws_file_name = self.tws_file

        self.folder_check_create(tws_file_name)

        bx = np.array([tw.beta_x for tw in twiss_list])
        by = np.array([tw.beta_y for tw in twiss_list])
        ax = np.array([tw.alpha_x for tw in twiss_list])
        ay = np.array([tw.alpha_x for tw in twiss_list])
        s = np.array([tw.s for tw in twiss_list])
        E = np.array([tw.E for tw in twiss_list])

        emit_x = np.array([tw.emit_x for tw in twiss_list])
        emit_y = np.array([tw.emit_y for tw in twiss_list])

        np.savez_compressed(tws_file_name, beta_x=bx, beta_y=by, alpha_x=ax, alpha_y=ay, E=E, s=s,
                            emit_x=emit_x, emit_y=emit_y) 

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("src_path", type=str)
    parser.add_argument("dst_path", type=str)
    args = parser.parse_args()

    src_path = Path(args.src_path)
    dst_path = Path(args.dst_path)

    assert src_path.is_file()
    src_trajs = types.load(str(src_path))
    dst_trajs = convert_trajs_to_sb(src_trajs)
    os.makedirs(dst_path.parent, exist_ok=True)
    with open(dst_path, "wb") as f:
        np.savez_compressed(f, **dst_trajs)

    print(f"Dumped rollouts to {dst_path}") 

def export_trimmed_glove_vectors(vocab, glove_filename, trimmed_filename, dim):
    """Saves glove vectors in numpy array

    Args:
        vocab: dictionary vocab[word] = index
        glove_filename: a path to a glove file
        trimmed_filename: a path where to store a matrix in npy
        dim: (int) dimension of embeddings

    """
    embeddings = np.zeros([len(vocab), dim])
    with open(glove_filename, encoding="utf8") as f:
        for line in f:
            line = line.strip().split(' ')
            word = line[0]
            embedding = [float(x) for x in line[1:]]
            if word in vocab:
                word_idx = vocab[word]
                embeddings[word_idx] = np.asarray(embedding)

    np.savez_compressed(trimmed_filename, embeddings=embeddings) 

def average_model_chainer(ifiles, ofile):
    omodel = {}
    # get keys from the first file
    model = np.load(ifiles[0])
    for x in model:
        if 'model' in x:
            print(x)
    keys = [x.split('main/')[1] for x in model if 'model' in x]
    print(keys)
    for path in ifiles:
        model = np.load(path)
        for key in keys:
            val = model['updater/model:main/{}'.format(key)]
            if key not in omodel:
                omodel[key] = val
            else:
                omodel[key] += val
    for key in keys:
        omodel[key] /= len(ifiles)
    np.savez_compressed(ofile, **omodel) 

def main():
    env = gym.make('FetchPickAndPlace-v0')
    numItr = 100
    initStateSpace = "random"
    env.reset()
    print("Reset!")
    while len(actions) < numItr:
        obs = env.reset()
        print("ITERATION NUMBER ", len(actions))
        goToGoal(env, obs)
        

    fileName = "data_fetch"
    fileName += "_" + initStateSpace
    fileName += "_" + str(numItr)
    fileName += ".npz"
    
    np.savez_compressed(fileName, acs=actions, obs=observations, info=infos) # save the file 

def main():
    env = gym.make('FetchPickAndPlace-v0')
    numItr = 100
    initStateSpace = "random"
    env.reset()
    print("Reset!")
    while len(actions) < numItr:
        obs = env.reset()
        print("ITERATION NUMBER ", len(actions))
        goToGoal(env, obs)
        

    fileName = "data_fetch"
    fileName += "_" + initStateSpace
    fileName += "_" + str(numItr)
    fileName += ".npz"
    
    np.savez_compressed(fileName, acs=actions, obs=observations, info=infos) # save the file 

def backup_batches(batches_vol, batches_seg, path, case_id):
    # Create model directory of not existent
    if not os.path.exists(path):
        os.mkdir(path)
    # Create subdirectory for the case if not existent
    case_dir = os.path.join(path, "tmp.case_" + str(case_id).zfill(5))
    if not os.path.exists(case_dir):
        os.mkdir(case_dir)
    # Backup volume batches
    if batches_vol is not None:
        for i, batch in enumerate(batches_vol):
            out_path = os.path.join(case_dir,
                                    "batch_vol." + str(i))
            np.savez(out_path, data=batch)

    # Backup segmentation batches
    if batches_seg is not None:
        for i, batch in enumerate(batches_seg):
            out_path = os.path.join(case_dir,
                                    "batch_seg." + str(i))
            np.savez_compressed(out_path, data=batch)

# Load a MRI object from a npz for fast access 

def generate_large_matrix():
    rows, cols = 1000000, 500
    print 'Test serializing a %i x %i matrix ...' % (rows, cols)
    t = time.time()
    vecs = numpy.random.normal(0,1,(rows,cols))
    print 'Matrix constructed, spent %.2f s' % (time.time() - t)

    f1 = open('test_data1', 'wb')
    t = time.time()
    print 'saving as numpy npz format ...'
    numpy.savez_compressed(f1, vecs)
    print 'save done, spent %.2f s' % (time.time() - t)
    f1.close()

    f2 = open('test_data2', 'wb')
    t = time.time()
    print 'saving as self-defined format ...'
    for v in vecs:
        f2.write(pickle.dumps(v, -1))
    f2.close()
    print 'save done, spent %.2f s' % (time.time() - t)
    pass 

def export_trimmed_glove_vectors(vocab, glove_filename, trimmed_filename, dim):
    """Saves glove vectors in numpy array

    Args:
        vocab: dictionary vocab[word] = index
        glove_filename: a path to a glove file
        trimmed_filename: a path where to store a matrix in npy
        dim: (int) dimension of embeddings

    """
    embeddings = np.zeros([len(vocab), dim])
    with open(glove_filename) as f:
        for line in f:
            line = line.strip().split(' ')
            word = line[0]
            embedding = [float(x) for x in line[1:]]
            if word in vocab:
                word_idx = vocab[word]
                embeddings[word_idx] = np.asarray(embedding)

    np.savez_compressed(trimmed_filename, embeddings=embeddings) 

def encode(self, unischema_field, value):
        expected_dtype = unischema_field.numpy_dtype
        if isinstance(value, np.ndarray):
            if expected_dtype != value.dtype.type:
                raise ValueError('Unexpected type of {} feature. '
                                 'Expected {}. Got {}'.format(unischema_field.name, expected_dtype, value.dtype))

            expected_shape = unischema_field.shape
            if not _is_compliant_shape(value.shape, expected_shape):
                raise ValueError('Unexpected dimensions of {} feature. '
                                 'Expected {}. Got {}'.format(unischema_field.name, expected_shape, value.shape))
        else:
            raise ValueError('Unexpected type of {} feature. '
                             'Expected ndarray of {}. Got {}'.format(unischema_field.name, expected_dtype, type(value)))

        memfile = BytesIO()
        np.savez_compressed(memfile, arr=value)
        return bytearray(memfile.getvalue()) 

def setUp(self):
        self.data = numpy.random.uniform(-1, 1, (2, 3)).astype(numpy.float32)

        fd, path = tempfile.mkstemp()
        os.close(fd)
        self.temp_file_path = path
        with open(path, 'wb') as f:
            savez = numpy.savez_compressed if self.compress else numpy.savez
            savez(
                f, **{'x/': None, 'y': self.data, 'z': numpy.asarray(10),
                      'zf32': numpy.array(-2**60, dtype=numpy.float32),
                      'zi64': numpy.array(-2**60, dtype=numpy.int64),
                      'w': None})

        try:
            self.npzfile = numpy.load(path, allow_pickle=True)
        except TypeError:
            self.npzfile = numpy.load(path)
        self.deserializer = npz.NpzDeserializer(self.npzfile) 

def make_npz(path, urls):
    x_url, y_url = urls
    x_path = download.cached_download(x_url)
    y_path = download.cached_download(y_url)

    with gzip.open(x_path, 'rb') as fx, gzip.open(y_path, 'rb') as fy:
        fx.read(4)
        fy.read(4)
        N, = struct.unpack('>i', fx.read(4))
        if N != struct.unpack('>i', fy.read(4))[0]:
            raise RuntimeError('wrong pair of MNIST images and labels')
        fx.read(8)

        x = numpy.empty((N, 784), dtype=numpy.uint8)
        y = numpy.empty(N, dtype=numpy.uint8)

        for i in six.moves.range(N):
            y[i] = ord(fy.read(1))
            for j in six.moves.range(784):
                x[i, j] = ord(fx.read(1))

    numpy.savez_compressed(path, x=x, y=y)
    return {'x': x, 'y': y} 

def extract_data_and_split(train_set, data_label_id, label2id):
    """1、处理数据和标签, 抽取title_word和content_word，将label和data对应起来 保存到文件中
        2、划分数据集"""
    datas = []
    for line in train_set:
        data_id, _, title_word, _, content_word = line.replace('\n', '').split('\t')
        labels = ','.join([str(label) for label in data_label_id[data_id]])
        info = '{}\t{}\t{}'.format(labels, title_word, content_word)
        datas.append(info)

    train_data, val_data = train_test_split(datas, test_size=0.05, random_state=2019)
    print('label num: {} - data num:{}'.format(len(label2id), len(datas)))
    print("train num: {} - dev num: {}".format(len(train_data), len(val_data)))

    np.savez_compressed(conf.label2id_path, data_label_id=data_label_id, label2id=label2id)
    with open(conf.train_file, 'w', encoding='utf-8') as fw:
        fw.write('\n'.join(train_data))
    with open(conf.dev_file, 'w', encoding='utf-8') as fw:
        fw.write('\n'.join(val_data)) 

def GenerateSample(filename, code_shape, layer_depth):
  # {0, +1} binary codes.
  # No conversion since the output file is expected to store
  # codes using {0, +1} codes (and not {-1, +1}).
  code = synthetic_model.GenerateSingleCode(code_shape)
  code = np.round(code)

  # Reformat the code so as to be compatible with what is generated
  # by the image encoder.
  # The image encoder generates a tensor of size:
  # iteration_count x batch_size x height x width x iteration_depth.
  # Here: batch_size = 1
  if code_shape[-1] % layer_depth != 0:
    raise ValueError('Number of layers is not an integer')
  height = code_shape[0]
  width = code_shape[1]
  code = code.reshape([1, height, width, -1, layer_depth])
  code = np.transpose(code, [3, 0, 1, 2, 4])

  int_codes = code.astype(np.int8)
  exported_codes = np.packbits(int_codes.reshape(-1))

  output = io.BytesIO()
  np.savez_compressed(output, shape=int_codes.shape, codes=exported_codes)
  with tf.gfile.FastGFile(filename, 'wb') as code_file:
    code_file.write(output.getvalue()) 

def mp_pack(inputs):
    ix , f = inputs
    file_path, source_dir, target_dir = f
    print('packing file number: {}'.format(ix))
    if 'npy' in file_path:
        source_path = os.path.join(source_dir, file_path)
        target_path = os.path.join(target_dir, file_path.split('.')[0] + '.npz')
        arr = np.load(source_path, mmap_mode='r')
        np.savez_compressed(target_path, data=arr)
        print('target_path', target_path) 

def save(df, path):
        """
        Args:
            df (DataFlow): the DataFlow to serialize.
            path (str): output npz file.
        """
        buffer = []
        size = _reset_df_and_get_size(df)
        with get_tqdm(total=size) as pbar:
            for dp in df:
                buffer.append(dp)
                pbar.update()
        np.savez_compressed(path, buffer=np.asarray(buffer, dtype=np.object)) 

def test_update_cache_file_add_obs(cachefile, example_cache,
                                   example_timeseries):
    """ Grab a subset of test data and see if we get more data back """
    stack_images = example_timeseries['images']
    stack_image_IDs = example_timeseries['image_IDs']

    # Presort and subset for comparison
    sort_idx = np.argsort(example_cache['image_IDs'])
    test_Y = example_cache['Y'][:, sort_idx, :]
    test_IDs = example_cache['image_IDs'][sort_idx]

    size_1 = 100
    size_2 = 200

    sort_idx = np.argsort(stack_image_IDs)[:size_2]
    stack_images = stack_images[sort_idx]
    stack_IDs = stack_image_IDs[sort_idx]

    # Create reduced dataset to add to
    np.savez_compressed('test.npz',
                        Y=test_Y[:, :size_1, :],
                        image_IDs=test_IDs[:size_1])

    # Write update and read back
    cache.update_cache_file(stack_images, stack_IDs,
                            'test.npz', 'test_new.npz',
                            0, io.gdal_reader)
    updated = np.load('test_new.npz')

    # Test and clean update
    np.testing.assert_equal(test_Y[:, :size_2, :], updated['Y'])
    np.testing.assert_equal(
        test_IDs[:size_2].astype(updated['image_IDs'].dtype),
        updated['image_IDs']
    )

    os.remove('test.npz')
    os.remove('test_new.npz') 

def test_savez_compressed_load(self):
        # Test that pathlib.Path instances can be used with savez.
        with temppath(suffix='.npz') as path:
            path = Path(path)
            np.savez_compressed(path, lab='place holder')
            data = np.load(path)
            assert_array_equal(data['lab'], 'place holder')
            data.close() 

def test_compressed_roundtrip():
    arr = np.random.rand(200, 200)
    npz_file = os.path.join(tempdir, 'compressed.npz')
    np.savez_compressed(npz_file, arr=arr)
    arr1 = np.load(npz_file)['arr']
    assert_array_equal(arr, arr1)


# aligned 

def save_npz(self, path, M):
        """Save to compressed npz

        Arguments:

            path (pathlib.Path): Path to store to

            M (ndarray): Contents of what to store.
        """
        p = pathlib.Path(path)
        p.parent.mkdir(parents=True, exist_ok=True)
        numpy.savez_compressed(str(path), M) 

def make_sprites(n=50000, path='./data'):
    path = os.path.expanduser(path)
    cache_filename = 'sprites_{}_{}.npz'.format(n, 50)
    if os.path.exists(os.path.join(path, cache_filename)):
        data = np.load(os.path.join(path, cache_filename), allow_pickle=True)
        return (data['x_train'], data['count_train'], None),\
               (data['x_test'], data['count_test'], None)

    images = np.zeros((n, 50, 50, 3))
    counts = np.zeros((n,))
    for i in range(n):
        if i < 100:
            num_sprites = i % 3
        else:
            num_sprites = np.random.random_integers(0, 2)
        counts[i] = num_sprites
        for j in range(num_sprites):
            images[i] = add_sprite(images[i])

    np.clip(images, 0.0, 1.0)
    x_train, count_train = images[:4 * n // 5], counts[:4 * n // 5]
    x_test, count_test = images[4 * n // 5:], counts[4 * n // 5:]

    with open(os.path.join(path, cache_filename), 'wb') as f:
        np.savez_compressed(f, x_train=x_train, count_train=count_train,
                            x_test=x_test, count_test=count_test)

    return (x_train, count_train, None), (x_test, count_test, None) 

def main(em_file, em_result):
    '''
    embedding ->numpy
    '''
    em = word2vec.load(em_file)
    vec = (em.vectors)
    word2id = em.vocab_hash
    # d = dict(vector = vec, word2id = word2id)
    # t.save(d,em_result)
    np.savez_compressed(em_result,vector=vec,word2id=word2id) 

def test_savez_compressed_load(self):
        # Test that pathlib.Path instances can be used with savez.
        with temppath(suffix='.npz') as path:
            path = Path(path)
            np.savez_compressed(path, lab='place holder')
            data = np.load(path)
            assert_array_equal(data['lab'], 'place holder')
            data.close() 

def test_compressed_roundtrip():
    arr = np.random.rand(200, 200)
    npz_file = os.path.join(tempdir, 'compressed.npz')
    np.savez_compressed(npz_file, arr=arr)
    arr1 = np.load(npz_file)['arr']
    assert_array_equal(arr, arr1)