Python pandas.HDFStore() Examples

The following are code examples for showing how to use pandas.HDFStore(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: smp_graphs   Author: x75   File: utils_logging.py    BSD 3-Clause "New" or "Revised" License 7 votes vote down vote up
def log_pd_dump_config(h5store, storekey = None):
    assert h5store is not None
    assert storekey is not None
    store = pd.HDFStore(h5store)
    try:
        ret = store.get_storer(storekey).attrs.conf
    except AttributeError:
        print("key %s doesn't exist" % (storekey))
        ret = None
    store.close()
    return ret
    
# def log_pd_dump_log(h5store = None):
#     assert h5store is not None
#     store = pd.HDFStore(h5store)

#     print "initial config = %s" % store.get_storer('conf').attrs.conf 
Example 2
Project: smp_graphs   Author: x75   File: smp_dfgui.py    BSD 3-Clause "New" or "Revised" License 7 votes vote down vote up
def main_show_group(args):
    print(("show_group args = %s" % (args, )))
    h5f = load_hdf(args)
    
    # dfs['experiments'] = '/experiments'
    # dfs['blocks'] = '/blocks'
        
    if h5f is None: return 1

    print(("HDFStore.keys = %s" % (list(h5f.keys()), )))

    df = None
    if args.group in list(h5f.keys()):
        print(("Found group %s in h5f.keys" % (args.group, )))
        df = h5f[args.group]

    if df is None: return 1
        
    dfgui.show(df) 
Example 3
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_pytables.py    MIT License 7 votes vote down vote up
def ensure_clean_store(path, mode="a", complevel=None, complib=None, fletcher32=False):

    try:

        # put in the temporary path if we don't have one already
        if not len(os.path.dirname(path)):
            path = create_tempfile(path)

        store = HDFStore(
            path, mode=mode, complevel=complevel, complib=complib, fletcher32=False
        )
        yield store
    finally:
        safe_close(store)
        if mode == "w" or mode == "a":
            safe_remove(path) 
Example 4
Project: RNASEqTool   Author: armell   File: hdf_storage.py    MIT License 6 votes vote down vote up
def test():
    store_path = APP_CONFIG["application_files_location"]
    store_name = APP_CONFIG["application_store_name"]

    join_path = store_path + store_name

    store = pd.HDFStore(join_path, complevel=9, complib='blosc')

    test_frame = pd.read_csv(APP_CONFIG["demo_data_set_raw"], sep="\t", index_col=0)

    store["test"] = test_frame

    print store

    store.close()

    store = pd.HDFStore(join_path, complevel=9, complib='blosc')

    frame = store["test"]

    print frame.iloc[0, 0] 
Example 5
Project: remixt   Author: amcpherson   File: pipeline.py    MIT License 6 votes vote down vote up
def simulate_normal_data(read_data_filename, mixture_filename, germline_alleles_filename, params):

    with open(mixture_filename, 'r') as mixture_file:
        genome_mixture = pickle.load(mixture_file)

    germline_genome = genome_mixture.genome_collection.genomes[0]
    germline_alleles = pd.HDFStore(germline_alleles_filename, 'r')

    np.random.seed(params['random_seed'])

    remixt.simulations.seqread.simulate_mixture_read_data(
        read_data_filename,
        [germline_genome],
        [params['h_total']],
        germline_alleles,
        params) 
Example 6
Project: remixt   Author: amcpherson   File: pipeline.py    MIT License 6 votes vote down vote up
def resample_normal_data(read_data_filename, source_filename, mixture_filename, germline_alleles_filename, params):

    with open(mixture_filename, 'r') as mixture_file:
        genome_mixture = pickle.load(mixture_file)

    germline_genome = genome_mixture.genome_collection.genomes[0]
    germline_alleles = pd.HDFStore(germline_alleles_filename, 'r')

    np.random.seed(params['random_seed'])

    remixt.simulations.seqread.resample_mixture_read_data(
        read_data_filename,
        source_filename,
        [germline_genome],
        [params['h_total']],
        germline_alleles,
        params) 
Example 7
Project: remixt   Author: amcpherson   File: pipeline.py    MIT License 6 votes vote down vote up
def resample_tumour_data(read_data_filename, source_filename, mixture_filename, germline_alleles_filename, params):

    with open(mixture_filename, 'r') as mixture_file:
        gm = pickle.load(mixture_file)

    germline_alleles = pd.HDFStore(germline_alleles_filename, 'r')

    np.random.seed(params['random_seed'])

    remixt.simulations.seqread.resample_mixture_read_data(
        read_data_filename,
        source_filename,
        gm.genome_collection.genomes,
        gm.frac * params['h_total'],
        germline_alleles,
        params) 
Example 8
Project: remixt   Author: amcpherson   File: seqdataio.py    MIT License 6 votes vote down vote up
def read_chromosomes(seqdata_filename):
    """ Read chromosomes from a HDF seqdata file.

    Args:
        seqdata_filename (str): name of seqdata file

    Returns:
        list of chromsomes

    """

    with pd.HDFStore(seqdata_filename, 'r') as store:
        chromosomes = set()
        for key in store.keys():
            if 'chromosome_' in key:
                chromosomes.add(key[key.index('chromosome_') + len('chromosome_'):])

        return chromosomes 
Example 9
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_pytables.py    MIT License 6 votes vote down vote up
def test_context(self):
        path = create_tempfile(self.path)
        try:
            with HDFStore(path) as tbl:
                raise ValueError("blah")
        except ValueError:
            pass
        finally:
            safe_remove(path)

        try:
            with HDFStore(path) as tbl:
                tbl["a"] = tm.makeDataFrame()

            with HDFStore(path) as tbl:
                assert len(tbl) == 1
                assert type(tbl["a"]) == DataFrame
        finally:
            safe_remove(path) 
Example 10
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_pytables.py    MIT License 6 votes vote down vote up
def test_open_args(self):

        with ensure_clean_path(self.path) as path:

            df = tm.makeDataFrame()

            # create an in memory store
            store = HDFStore(
                path, mode="a", driver="H5FD_CORE", driver_core_backing_store=0
            )
            store["df"] = df
            store.append("df2", df)

            tm.assert_frame_equal(store["df"], df)
            tm.assert_frame_equal(store["df2"], df)

            store.close()

            # the file should not have actually been written
            assert not os.path.exists(path) 
Example 11
Project: recruit   Author: Frank-qlu   File: test_pytables.py    Apache License 2.0 6 votes vote down vote up
def ensure_clean_store(path, mode='a', complevel=None, complib=None,
                       fletcher32=False):

    try:

        # put in the temporary path if we don't have one already
        if not len(os.path.dirname(path)):
            path = create_tempfile(path)

        store = HDFStore(path, mode=mode, complevel=complevel,
                         complib=complib, fletcher32=False)
        yield store
    finally:
        safe_close(store)
        if mode == 'w' or mode == 'a':
            safe_remove(path) 
Example 12
Project: recruit   Author: Frank-qlu   File: test_pytables.py    Apache License 2.0 6 votes vote down vote up
def test_context(self):
        path = create_tempfile(self.path)
        try:
            with HDFStore(path) as tbl:
                raise ValueError('blah')
        except ValueError:
            pass
        finally:
            safe_remove(path)

        try:
            with HDFStore(path) as tbl:
                tbl['a'] = tm.makeDataFrame()

            with HDFStore(path) as tbl:
                assert len(tbl) == 1
                assert type(tbl['a']) == DataFrame
        finally:
            safe_remove(path) 
Example 13
Project: recruit   Author: Frank-qlu   File: test_pytables.py    Apache License 2.0 6 votes vote down vote up
def test_open_args(self):

        with ensure_clean_path(self.path) as path:

            df = tm.makeDataFrame()

            # create an in memory store
            store = HDFStore(path, mode='a', driver='H5FD_CORE',
                             driver_core_backing_store=0)
            store['df'] = df
            store.append('df2', df)

            tm.assert_frame_equal(store['df'], df)
            tm.assert_frame_equal(store['df2'], df)

            store.close()

            # the file should not have actually been written
            assert not os.path.exists(path) 
Example 14
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_pytables.py    MIT License 6 votes vote down vote up
def ensure_clean_store(path, mode='a', complevel=None, complib=None,
                       fletcher32=False):

    try:

        # put in the temporary path if we don't have one already
        if not len(os.path.dirname(path)):
            path = create_tempfile(path)

        store = HDFStore(path, mode=mode, complevel=complevel,
                         complib=complib, fletcher32=False)
        yield store
    finally:
        safe_close(store)
        if mode == 'w' or mode == 'a':
            safe_remove(path) 
Example 15
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_pytables.py    MIT License 6 votes vote down vote up
def test_context(self):
        path = create_tempfile(self.path)
        try:
            with HDFStore(path) as tbl:
                raise ValueError('blah')
        except ValueError:
            pass
        finally:
            safe_remove(path)

        try:
            with HDFStore(path) as tbl:
                tbl['a'] = tm.makeDataFrame()

            with HDFStore(path) as tbl:
                assert len(tbl) == 1
                assert type(tbl['a']) == DataFrame
        finally:
            safe_remove(path) 
Example 16
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_pytables.py    MIT License 6 votes vote down vote up
def test_read_hdf_open_store(self):
        # GH10330
        # No check for non-string path_or-buf, and no test of open store
        df = DataFrame(np.random.rand(4, 5),
                       index=list('abcd'),
                       columns=list('ABCDE'))
        df.index.name = 'letters'
        df = df.set_index(keys='E', append=True)

        with ensure_clean_path(self.path) as path:
            df.to_hdf(path, 'df', mode='w')
            direct = read_hdf(path, 'df')
            store = HDFStore(path, mode='r')
            indirect = read_hdf(store, 'df')
            tm.assert_frame_equal(direct, indirect)
            assert store.is_open
            store.close() 
Example 17
Project: qdata   Author: X0Leon   File: utils.py    MIT License 5 votes vote down vote up
def hdf_keys(filename, clean=False):
    """ 返回hdf文件的键值 """
    with pd.HDFStore(filename) as store:
       k = store.keys()
    if clean:
        return [s.split('/')[-1] for s in k]
    return k 
Example 18
Project: RNASEqTool   Author: armell   File: hdf_storage.py    MIT License 5 votes vote down vote up
def store_hdf(df, intern_identifier):
    store_path = APP_CONFIG["application_files_location"]
    store_name = APP_CONFIG["application_store_name"]

    join_path = store_path + store_name

    store = pd.HDFStore(join_path, complevel=9, complib='blosc')
    store[intern_identifier] = df
    store.close()

    return join_path 
Example 19
Project: RNASEqTool   Author: armell   File: hdf_storage.py    MIT License 5 votes vote down vote up
def retrieve_hdf(identifier, location):
    store = pd.HDFStore(location, complevel=9, complib='blosc')
    data = store[identifier]
    store.close()
    return data 
Example 20
Project: remixt   Author: amcpherson   File: write_results.py    MIT License 5 votes vote down vote up
def write_results_tables(**args):
    store = pd.HDFStore(args['results_filename'], 'r')

    stats = store['stats']

    # Filter high proportion subclonal
    stats = stats[stats['proportion_divergent'] <= args['max_proportion_divergent']]

    # Select highest elbo solution
    stats = stats.sort_values('elbo', ascending=False).iloc[0]
    solution = stats['init_id']

    cn = store['solutions/solution_{0}/cn'.format(solution)]
    brk_cn = store['solutions/solution_{0}/brk_cn'.format(solution)]
    h = store['solutions/solution_{0}/h'.format(solution)]
    mix = store['solutions/solution_{0}/mix'.format(solution)]

    cn.to_csv(args['cn_filename'], sep='\t', index=False)
    brk_cn.to_csv(args['brk_cn_filename'], sep='\t', index=False)

    metadata = dict()
    for key, value in stats.items():
        try:
            metadata[key] = np.asscalar(value)
        except AttributeError:
            metadata[key] = value
    metadata['h'] = list(h.tolist())
    metadata['mix'] = list(mix.tolist())

    with open(args['meta_filename'], 'w') as f:
        yaml.dump(metadata, f, default_flow_style=False)

    store.close() 
Example 21
Project: remixt   Author: amcpherson   File: tasks.py    MIT License 5 votes vote down vote up
def merge_files_by_line(in_filenames, out_filename):
    with pd.HDFStore(out_filename, 'w') as store:
        for in_filename in in_filenames.values():
            data = pd.read_csv(
                in_filename, sep='\t', header=None,
                names=['chromosome', 'start', 'end', 'quality'],
                converters={'chromosome':str})
            for chromosome, chrom_data in data.groupby('chromosome'):
                store.append('chromosome_'+chromosome, chrom_data[['start', 'end', 'quality']], data_columns=True) 
Example 22
Project: remixt   Author: amcpherson   File: gcbias.py    MIT License 5 votes vote down vote up
def read_mappability_indicator(mappability_filename, chromosome, max_chromosome_length, map_qual_threshold):
    """ Read a mappability wig file into a mappability vector
    """
    with pd.HDFStore(mappability_filename, 'r') as store:
        mappability_table = store.select('chromosome_'+chromosome, 'quality >= map_qual_threshold')

    mappability = np.zeros(max_chromosome_length, dtype=np.uint8)

    for start, end in mappability_table[['start', 'end']].values:
        end = min(end, max_chromosome_length)
        mappability[start:end] = 1

    return mappability 
Example 23
Project: remixt   Author: amcpherson   File: pipeline.py    MIT License 5 votes vote down vote up
def collate(collate_filename, experiment_filename, init_results_filename, fit_results_filenames, config):

    # Extract the statistics for selecting solutions
    stats_table = list()
    for init_id, results_filename in fit_results_filenames.items():
        results = pickle.load(open(results_filename, 'rb'))
        stats = dict(results['stats'])
        stats['init_id'] = init_id
        stats_table.append(stats)
    stats_table = pd.DataFrame(stats_table)

    # Write out selected solutions
    with pd.HDFStore(collate_filename, 'w') as collated:
        collated['stats'] = stats_table

        with pd.HDFStore(init_results_filename, 'r') as results:
            for key, value in results.items():
                collated[key] = results[key]

        with open(experiment_filename, 'rb') as f:
            experiment = pickle.load(f)

        for init_id, results_filename in fit_results_filenames.items():
            results = pickle.load(open(results_filename, 'rb'))
            store_fit_results(collated, experiment, results, 'solutions/solution_{0}'.format(init_id))

        store_optimal_solution(stats_table, collated, config) 
Example 24
Project: remixt   Author: amcpherson   File: pipeline.py    MIT License 5 votes vote down vote up
def simulate_germline_alleles(germline_alleles_filename, params, config, ref_data_dir):

    np.random.seed(params['random_seed'])
    
    with pd.HDFStore(germline_alleles_filename, 'w', complevel=9, complib='zlib') as germline_alleles_store:
        for chromosome in params['chromosomes']:
            alleles_table = remixt.simulations.haplotype.create_sim_alleles(chromosome, config, ref_data_dir)
            germline_alleles_store.put('/chromosome_{}'.format(chromosome), alleles_table, format='table') 
Example 25
Project: remixt   Author: amcpherson   File: pipeline.py    MIT License 5 votes vote down vote up
def simulate_tumour_data(read_data_filename, mixture_filename, germline_alleles_filename, params):

    with open(mixture_filename, 'r') as mixture_file:
        gm = pickle.load(mixture_file)

    germline_alleles = pd.HDFStore(germline_alleles_filename, 'r')

    np.random.seed(params['random_seed'])

    remixt.simulations.seqread.simulate_mixture_read_data(
        read_data_filename,
        gm.genome_collection.genomes,
        gm.frac * params['h_total'],
        germline_alleles,
        params) 
Example 26
Project: remixt   Author: amcpherson   File: pipeline.py    MIT License 5 votes vote down vote up
def merge_evaluations(merged_filename, sim_defs, evaluation_filenames, key_names):
    """ Merge multiple evaluations of prediction results.

    Args:
        merged_filename (str): output hdf filename of merged evaluations
        sim_defs (dict): simulation definitions per simulation
        evaluation_filenames (dict of str): hdf filename of evaluation per simulation per tool

    """
    
    merged_store = pd.HDFStore(merged_filename, 'w')

    sim_defs_table = pd.DataFrame(
        sim_defs.values(),
        index=pd.Index(sim_defs.keys(), name='sim_id'),
    ).reset_index()

    merged_store['/simulations'] = sim_defs_table

    tables = collections.defaultdict(list)
    for key, evaluation_filename in evaluation_filenames.items():
        store = pd.HDFStore(evaluation_filename, 'r')

        if not isinstance(key, tuple):
            key = (key,)

        for table_name in ('/cn_evaluation', '/brk_cn_evaluation', '/mix_results', 'outlier_evaluation'):
            if table_name not in store:
                continue
            table = store[table_name]
            for value, name in zip(key, key_names):
                table[name] = value
            tables[table_name].append(table)

        merged_store['/brk_cn_table/' + '/'.join(key)] = store['/brk_cn_table']
    
    for table_name, table in tables.items():
        merged_store[table_name] = pd.DataFrame(table) 
Example 27
Project: remixt   Author: amcpherson   File: seqdataio.py    MIT License 5 votes vote down vote up
def merge_overlapping_seqdata(outfile, infiles, chromosomes):
    out_store = pd.HDFStore(outfile, 'w', complevel=9, complib='blosc')

    index_offsets = pd.Series(0, index=chromosomes, dtype=np.int64)

    for _id, infile in infiles.items():
        store = pd.HDFStore(infile)
        tables = store.keys()

        for chromosome in chromosomes:
            allele_table = '/alleles/chromosome_{}'.format(chromosome)
            fragment_table = '/fragments/chromosome_{}'.format(chromosome)

            if allele_table not in tables:
                print("missing table {}".format(allele_table))
                continue

            if fragment_table not in tables:
                print("missing table {}".format(fragment_table))
                continue

            alleles = store[allele_table]
            fragments = store[fragment_table]

            alleles['fragment_id'] = alleles['fragment_id'].astype(np.int64)
            fragments['fragment_id'] = fragments['fragment_id'].astype(np.int64)

            alleles['fragment_id'] += index_offsets[chromosome]
            fragments['fragment_id'] += index_offsets[chromosome]

            index_offsets[chromosome] = max(alleles['fragment_id'].max(), fragments['fragment_id'].max()) + 1

            out_store.append('/alleles/chromosome_{}'.format(chromosome), alleles)
            out_store.append('/fragments/chromosome_{}'.format(chromosome), fragments)

        store.close()
    out_store.close() 
Example 28
Project: remixt   Author: amcpherson   File: seqdataio.py    MIT License 5 votes vote down vote up
def create_chromosome_seqdata(seqdata_filename, bam_filename, snp_filename, chromosome, max_fragment_length, max_soft_clipped, check_proper_pair):
    """ Create seqdata from bam for one chromosome.

    Args:
        seqdata_filename(str): seqdata hdf store to write to
        bam_filename(str): bam from which to extract read information
        snp_filename(str): TSV chromosome, position file listing SNPs
        chromosome(str): chromosome to extract
        max_fragment_length(int): maximum length of fragments generating paired reads
        max_soft_clipped(int): maximum soft clipping for considering a read concordant
        check_proper_pair(boo): check proper pair flag

    """

    reader = remixt.bamreader.AlleleReader(
        bam_filename,
        snp_filename,
        chromosome,
        max_fragment_length,
        max_soft_clipped,
        check_proper_pair,
    )

    with pd.HDFStore(seqdata_filename, 'w', complevel=9, complib='zlib') as store:
        while reader.ReadAlignments(10000000):
            _unique_index_append(store, _get_key('fragments', chromosome), reader.GetFragmentTable())
            _unique_index_append(store, _get_key('alleles', chromosome), reader.GetAlleleTable()) 
Example 29
Project: remixt   Author: amcpherson   File: seqdataio.py    MIT License 5 votes vote down vote up
def merge_seqdata(out_filename, in_filenames):
    """ Merge seqdata files for non-overlapping sets of chromosomes

    Args:
        out_filename(str): seqdata hdf store to write to
        out_filename(dict): seqdata hdf store to read from

    """

    with pd.HDFStore(out_filename, 'w', complevel=9, complib='zlib') as out_store:
        for in_filename in in_filenames.values():
            with pd.HDFStore(in_filename, 'r') as in_store:
                for key in in_store.keys():
                    out_store.put(key, in_store[key], format='table') 
Example 30
Project: remixt   Author: amcpherson   File: seqdataio.py    MIT License 5 votes vote down vote up
def __init__(self, seqdata_filename):
        """ Streaming writer of seq data hdf5 files 

        Args:
            seqdata_filename (str): name of seqdata hdf5 file

        """

        self.store = pd.HDFStore(seqdata_filename, 'w', complevel=9, complib='zlib') 
Example 31
Project: smp_graphs   Author: x75   File: utils_logging.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def log_pd_init(config):
    """log_pd_init

    global init for logging to hdf5 tables via pandas

    Arguments:
        - config: the global configuration dictionary
    """
    global log_store
    log_store_filename = '%s_log_pd.h5' % (config['params']['datafile_expr'], )
    log_store = pd.HDFStore(log_store_filename)
    # experiment = "%s" % (config['params']['id'])
    # log_store = pd.HDFStore("data/%s_pd.h5" % (experiment))
    # print "logging.log_pd_init log_store.root = %s" % (log_store.root) 
Example 32
Project: smp_graphs   Author: x75   File: smp_dfgui.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def load_hdf(args):
    h5f = None
    if os.path.exists(args.file):
        h5f = pd.HDFStore(args.file)
    else:
        print(("Couldn't load file %s" % (args.file, )))
    return h5f 
Example 33
Project: CalibrationNN   Author: Andres-Hernandez   File: data_utils.py    GNU General Public License v3.0 5 votes vote down vote up
def store_hdf5(file_name, key, val):
    with pd.HDFStore(file_name) as store:
        store[key] = val
        store.close() 
Example 34
Project: CalibrationNN   Author: Andres-Hernandez   File: data_utils.py    GNU General Public License v3.0 5 votes vote down vote up
def from_hdf5(key, file_name=h5file):
    with pd.HDFStore(file_name) as store:
        data =  store[key]
        store.close()
    return data 
Example 35
Project: DRCOG_Urbansim   Author: apdjustino   File: dataset.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def __init__(self,filename):
      self.store = pd.HDFStore(filename)
      self.d = {} # keep track of in memory pandas data frames so as not to load multiple times form disk
      self.coeffs = pd.DataFrame() # keep all coefficients in memory
      self.attrs = {} # keep all computed outputs in memory
      self.savetbls = [] # names of tables to write to output hdf5 
Example 36
Project: DRCOG_Urbansim   Author: apdjustino   File: dataset.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def save_output(self,filename):
    outstore = pd.HDFStore(filename)
    for key in self.attrs.keys():
      df = self.attrs[key]
      df = misc.df64bitto32bit(df)
      print key+"\n", df.describe()
      outstore[key] = df
    
    for key in self.savetbls:
      df = self.fetch(key)
      df = misc.df64bitto32bit(df)
      print key+"\n", df.describe()
      outstore[key] = df

    outstore.close() 
Example 37
Project: DRCOG_Urbansim   Author: apdjustino   File: dataset.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def copy_coeffs(self,filename=None):
    if filename: coeffstore = pd.HDFStore(filename)
    else: coeffstore = self.store
    self.coeffs = coeffstore['coefficients'] 
    coeffstore.close() 
Example 38
Project: DRCOG_Urbansim   Author: apdjustino   File: dataset.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def save_coeffs(self,filename):
    outstore = pd.HDFStore(filename)
    print self.coeffs
    outstore['coefficients'] = self.coeffs 
    outstore.close() 
Example 39
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_pytables.py    MIT License 5 votes vote down vote up
def test_format_kwarg_in_constructor(self):
        # GH 13291
        with ensure_clean_path(self.path) as path:
            with pytest.raises(ValueError):
                HDFStore(path, format="table") 
Example 40
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_pytables.py    MIT License 5 votes vote down vote up
def test_path_pathlib_hdfstore(self):
        df = tm.makeDataFrame()

        def writer(path):
            with pd.HDFStore(path) as store:
                df.to_hdf(store, "df")

        def reader(path):
            with pd.HDFStore(path) as store:
                return pd.read_hdf(store, "df")

        result = tm.round_trip_pathlib(writer, reader)
        tm.assert_frame_equal(df, result) 
Example 41
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_pytables.py    MIT License 5 votes vote down vote up
def test_path_localpath_hdfstore(self):
        df = tm.makeDataFrame()

        def writer(path):
            with pd.HDFStore(path) as store:
                df.to_hdf(store, "df")

        def reader(path):
            with pd.HDFStore(path) as store:
                return pd.read_hdf(store, "df")

        result = tm.round_trip_localpath(writer, reader)
        tm.assert_frame_equal(df, result) 
Example 42
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_pytables.py    MIT License 5 votes vote down vote up
def test_read_hdf_errors(self):
        df = DataFrame(np.random.rand(4, 5), index=list("abcd"), columns=list("ABCDE"))

        with ensure_clean_path(self.path) as path:
            with pytest.raises(IOError):
                read_hdf(path, "key")

            df.to_hdf(path, "df")
            store = HDFStore(path, mode="r")
            store.close()

            with pytest.raises(IOError):
                read_hdf(store, "df") 
Example 43
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_pytables.py    MIT License 5 votes vote down vote up
def test_read_nokey_empty(self):
        with ensure_clean_path(self.path) as path:
            store = HDFStore(path)
            store.close()

            with pytest.raises(ValueError):
                read_hdf(path) 
Example 44
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_pytables.py    MIT License 5 votes vote down vote up
def test_fspath(self):
        with tm.ensure_clean("foo.h5") as path:
            with pd.HDFStore(path) as store:
                assert os.fspath(store) == str(path) 
Example 45
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_pytables.py    MIT License 5 votes vote down vote up
def test_select_empty_where(self, where):
        # GH26610

        # Using keyword `where` as '' or (), or [None], etc
        # while reading from HDF store raises
        # "SyntaxError: only a single expression is allowed"

        df = pd.DataFrame([1, 2, 3])
        with ensure_clean_path("empty_where.h5") as path:
            with pd.HDFStore(path) as store:
                store.put("df", df, "t")
                result = pd.read_hdf(store, "df", where=where)
                assert_frame_equal(result, df) 
Example 46
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_pytables.py    MIT License 5 votes vote down vote up
def test_read_with_where_tz_aware_index(self):
        # GH 11926
        periods = 10
        dts = pd.date_range("20151201", periods=periods, freq="D", tz="UTC")
        mi = pd.MultiIndex.from_arrays([dts, range(periods)], names=["DATE", "NO"])
        expected = pd.DataFrame({"MYCOL": 0}, index=mi)

        key = "mykey"
        with ensure_clean_path(self.path) as path:
            with pd.HDFStore(path) as store:
                store.append(key, expected, format="table", append=True)
            result = pd.read_hdf(path, key, where="DATE > 20151130")
            assert_frame_equal(result, expected) 
Example 47
Project: recruit   Author: Frank-qlu   File: test_pytables.py    Apache License 2.0 5 votes vote down vote up
def test_format_kwarg_in_constructor(self):
        # GH 13291
        with ensure_clean_path(self.path) as path:
            pytest.raises(ValueError, HDFStore, path, format='table') 
Example 48
Project: recruit   Author: Frank-qlu   File: test_pytables.py    Apache License 2.0 5 votes vote down vote up
def test_api_default_format(self):

        # default_format option
        with ensure_clean_store(self.path) as store:
            df = tm.makeDataFrame()

            pd.set_option('io.hdf.default_format', 'fixed')
            _maybe_remove(store, 'df')
            store.put('df', df)
            assert not store.get_storer('df').is_table
            pytest.raises(ValueError, store.append, 'df2', df)

            pd.set_option('io.hdf.default_format', 'table')
            _maybe_remove(store, 'df')
            store.put('df', df)
            assert store.get_storer('df').is_table
            _maybe_remove(store, 'df2')
            store.append('df2', df)
            assert store.get_storer('df').is_table

            pd.set_option('io.hdf.default_format', None)

        with ensure_clean_path(self.path) as path:

            df = tm.makeDataFrame()

            pd.set_option('io.hdf.default_format', 'fixed')
            df.to_hdf(path, 'df')
            with HDFStore(path) as store:
                assert not store.get_storer('df').is_table
            pytest.raises(ValueError, df.to_hdf, path, 'df2', append=True)

            pd.set_option('io.hdf.default_format', 'table')
            df.to_hdf(path, 'df3')
            with HDFStore(path) as store:
                assert store.get_storer('df3').is_table
            df.to_hdf(path, 'df4', append=True)
            with HDFStore(path) as store:
                assert store.get_storer('df4').is_table

            pd.set_option('io.hdf.default_format', None) 
Example 49
Project: recruit   Author: Frank-qlu   File: test_pytables.py    Apache License 2.0 5 votes vote down vote up
def test_path_pathlib_hdfstore(self):
        df = tm.makeDataFrame()

        def writer(path):
            with pd.HDFStore(path) as store:
                df.to_hdf(store, 'df')

        def reader(path):
            with pd.HDFStore(path) as store:
                return pd.read_hdf(store, 'df')

        result = tm.round_trip_pathlib(writer, reader)
        tm.assert_frame_equal(df, result) 
Example 50
Project: recruit   Author: Frank-qlu   File: test_pytables.py    Apache License 2.0 5 votes vote down vote up
def test_path_localpath_hdfstore(self):
        df = tm.makeDataFrame()

        def writer(path):
            with pd.HDFStore(path) as store:
                df.to_hdf(store, 'df')

        def reader(path):
            with pd.HDFStore(path) as store:
                return pd.read_hdf(store, 'df')

        result = tm.round_trip_localpath(writer, reader)
        tm.assert_frame_equal(df, result)