Python pandas.HDFStore() Examples

def log_pd_dump_config(h5store, storekey = None):
    assert h5store is not None
    assert storekey is not None
    store = pd.HDFStore(h5store)
    try:
        ret = store.get_storer(storekey).attrs.conf
    except AttributeError:
        print("key %s doesn't exist" % (storekey))
        ret = None
    store.close()
    return ret
    
# def log_pd_dump_log(h5store = None):
#     assert h5store is not None
#     store = pd.HDFStore(h5store)

#     print "initial config = %s" % store.get_storer('conf').attrs.conf 

def main_show_group(args):
    print(("show_group args = %s" % (args, )))
    h5f = load_hdf(args)
    
    # dfs['experiments'] = '/experiments'
    # dfs['blocks'] = '/blocks'
        
    if h5f is None: return 1

    print(("HDFStore.keys = %s" % (list(h5f.keys()), )))

    df = None
    if args.group in list(h5f.keys()):
        print(("Found group %s in h5f.keys" % (args.group, )))
        df = h5f[args.group]

    if df is None: return 1
        
    dfgui.show(df) 

def ensure_clean_store(path, mode="a", complevel=None, complib=None, fletcher32=False):

    try:

        # put in the temporary path if we don't have one already
        if not len(os.path.dirname(path)):
            path = create_tempfile(path)

        store = HDFStore(
            path, mode=mode, complevel=complevel, complib=complib, fletcher32=False
        )
        yield store
    finally:
        safe_close(store)
        if mode == "w" or mode == "a":
            safe_remove(path) 

def test():
    store_path = APP_CONFIG["application_files_location"]
    store_name = APP_CONFIG["application_store_name"]

    join_path = store_path + store_name

    store = pd.HDFStore(join_path, complevel=9, complib='blosc')

    test_frame = pd.read_csv(APP_CONFIG["demo_data_set_raw"], sep="\t", index_col=0)

    store["test"] = test_frame

    print store

    store.close()

    store = pd.HDFStore(join_path, complevel=9, complib='blosc')

    frame = store["test"]

    print frame.iloc[0, 0] 

def simulate_normal_data(read_data_filename, mixture_filename, germline_alleles_filename, params):

    with open(mixture_filename, 'r') as mixture_file:
        genome_mixture = pickle.load(mixture_file)

    germline_genome = genome_mixture.genome_collection.genomes[0]
    germline_alleles = pd.HDFStore(germline_alleles_filename, 'r')

    np.random.seed(params['random_seed'])

    remixt.simulations.seqread.simulate_mixture_read_data(
        read_data_filename,
        [germline_genome],
        [params['h_total']],
        germline_alleles,
        params) 

def resample_normal_data(read_data_filename, source_filename, mixture_filename, germline_alleles_filename, params):

    with open(mixture_filename, 'r') as mixture_file:
        genome_mixture = pickle.load(mixture_file)

    germline_genome = genome_mixture.genome_collection.genomes[0]
    germline_alleles = pd.HDFStore(germline_alleles_filename, 'r')

    np.random.seed(params['random_seed'])

    remixt.simulations.seqread.resample_mixture_read_data(
        read_data_filename,
        source_filename,
        [germline_genome],
        [params['h_total']],
        germline_alleles,
        params) 

def resample_tumour_data(read_data_filename, source_filename, mixture_filename, germline_alleles_filename, params):

    with open(mixture_filename, 'r') as mixture_file:
        gm = pickle.load(mixture_file)

    germline_alleles = pd.HDFStore(germline_alleles_filename, 'r')

    np.random.seed(params['random_seed'])

    remixt.simulations.seqread.resample_mixture_read_data(
        read_data_filename,
        source_filename,
        gm.genome_collection.genomes,
        gm.frac * params['h_total'],
        germline_alleles,
        params) 

def read_chromosomes(seqdata_filename):
    """ Read chromosomes from a HDF seqdata file.

    Args:
        seqdata_filename (str): name of seqdata file

    Returns:
        list of chromsomes

    """

    with pd.HDFStore(seqdata_filename, 'r') as store:
        chromosomes = set()
        for key in store.keys():
            if 'chromosome_' in key:
                chromosomes.add(key[key.index('chromosome_') + len('chromosome_'):])

        return chromosomes 

def test_context(self):
        path = create_tempfile(self.path)
        try:
            with HDFStore(path) as tbl:
                raise ValueError("blah")
        except ValueError:
            pass
        finally:
            safe_remove(path)

        try:
            with HDFStore(path) as tbl:
                tbl["a"] = tm.makeDataFrame()

            with HDFStore(path) as tbl:
                assert len(tbl) == 1
                assert type(tbl["a"]) == DataFrame
        finally:
            safe_remove(path) 

def test_open_args(self):

        with ensure_clean_path(self.path) as path:

            df = tm.makeDataFrame()

            # create an in memory store
            store = HDFStore(
                path, mode="a", driver="H5FD_CORE", driver_core_backing_store=0
            )
            store["df"] = df
            store.append("df2", df)

            tm.assert_frame_equal(store["df"], df)
            tm.assert_frame_equal(store["df2"], df)

            store.close()

            # the file should not have actually been written
            assert not os.path.exists(path) 

def ensure_clean_store(path, mode='a', complevel=None, complib=None,
                       fletcher32=False):

    try:

        # put in the temporary path if we don't have one already
        if not len(os.path.dirname(path)):
            path = create_tempfile(path)

        store = HDFStore(path, mode=mode, complevel=complevel,
                         complib=complib, fletcher32=False)
        yield store
    finally:
        safe_close(store)
        if mode == 'w' or mode == 'a':
            safe_remove(path) 

def test_context(self):
        path = create_tempfile(self.path)
        try:
            with HDFStore(path) as tbl:
                raise ValueError('blah')
        except ValueError:
            pass
        finally:
            safe_remove(path)

        try:
            with HDFStore(path) as tbl:
                tbl['a'] = tm.makeDataFrame()

            with HDFStore(path) as tbl:
                assert len(tbl) == 1
                assert type(tbl['a']) == DataFrame
        finally:
            safe_remove(path) 

def test_open_args(self):

        with ensure_clean_path(self.path) as path:

            df = tm.makeDataFrame()

            # create an in memory store
            store = HDFStore(path, mode='a', driver='H5FD_CORE',
                             driver_core_backing_store=0)
            store['df'] = df
            store.append('df2', df)

            tm.assert_frame_equal(store['df'], df)
            tm.assert_frame_equal(store['df2'], df)

            store.close()

            # the file should not have actually been written
            assert not os.path.exists(path) 

def ensure_clean_store(path, mode='a', complevel=None, complib=None,
                       fletcher32=False):

    try:

        # put in the temporary path if we don't have one already
        if not len(os.path.dirname(path)):
            path = create_tempfile(path)

        store = HDFStore(path, mode=mode, complevel=complevel,
                         complib=complib, fletcher32=False)
        yield store
    finally:
        safe_close(store)
        if mode == 'w' or mode == 'a':
            safe_remove(path) 

def test_context(self):
        path = create_tempfile(self.path)
        try:
            with HDFStore(path) as tbl:
                raise ValueError('blah')
        except ValueError:
            pass
        finally:
            safe_remove(path)

        try:
            with HDFStore(path) as tbl:
                tbl['a'] = tm.makeDataFrame()

            with HDFStore(path) as tbl:
                assert len(tbl) == 1
                assert type(tbl['a']) == DataFrame
        finally:
            safe_remove(path) 

def test_read_hdf_open_store(self):
        # GH10330
        # No check for non-string path_or-buf, and no test of open store
        df = DataFrame(np.random.rand(4, 5),
                       index=list('abcd'),
                       columns=list('ABCDE'))
        df.index.name = 'letters'
        df = df.set_index(keys='E', append=True)

        with ensure_clean_path(self.path) as path:
            df.to_hdf(path, 'df', mode='w')
            direct = read_hdf(path, 'df')
            store = HDFStore(path, mode='r')
            indirect = read_hdf(store, 'df')
            tm.assert_frame_equal(direct, indirect)
            assert store.is_open
            store.close() 

def hdf_keys(filename, clean=False):
    """ 返回hdf文件的键值 """
    with pd.HDFStore(filename) as store:
       k = store.keys()
    if clean:
        return [s.split('/')[-1] for s in k]
    return k 

def store_hdf(df, intern_identifier):
    store_path = APP_CONFIG["application_files_location"]
    store_name = APP_CONFIG["application_store_name"]

    join_path = store_path + store_name

    store = pd.HDFStore(join_path, complevel=9, complib='blosc')
    store[intern_identifier] = df
    store.close()

    return join_path 

def retrieve_hdf(identifier, location):
    store = pd.HDFStore(location, complevel=9, complib='blosc')
    data = store[identifier]
    store.close()
    return data 

def write_results_tables(**args):
    store = pd.HDFStore(args['results_filename'], 'r')

    stats = store['stats']

    # Filter high proportion subclonal
    stats = stats[stats['proportion_divergent'] <= args['max_proportion_divergent']]

    # Select highest elbo solution
    stats = stats.sort_values('elbo', ascending=False).iloc[0]
    solution = stats['init_id']

    cn = store['solutions/solution_{0}/cn'.format(solution)]
    brk_cn = store['solutions/solution_{0}/brk_cn'.format(solution)]
    h = store['solutions/solution_{0}/h'.format(solution)]
    mix = store['solutions/solution_{0}/mix'.format(solution)]

    cn.to_csv(args['cn_filename'], sep='\t', index=False)
    brk_cn.to_csv(args['brk_cn_filename'], sep='\t', index=False)

    metadata = dict()
    for key, value in stats.items():
        try:
            metadata[key] = np.asscalar(value)
        except AttributeError:
            metadata[key] = value
    metadata['h'] = list(h.tolist())
    metadata['mix'] = list(mix.tolist())

    with open(args['meta_filename'], 'w') as f:
        yaml.dump(metadata, f, default_flow_style=False)

    store.close() 

def merge_files_by_line(in_filenames, out_filename):
    with pd.HDFStore(out_filename, 'w') as store:
        for in_filename in in_filenames.values():
            data = pd.read_csv(
                in_filename, sep='\t', header=None,
                names=['chromosome', 'start', 'end', 'quality'],
                converters={'chromosome':str})
            for chromosome, chrom_data in data.groupby('chromosome'):
                store.append('chromosome_'+chromosome, chrom_data[['start', 'end', 'quality']], data_columns=True) 

def read_mappability_indicator(mappability_filename, chromosome, max_chromosome_length, map_qual_threshold):
    """ Read a mappability wig file into a mappability vector
    """
    with pd.HDFStore(mappability_filename, 'r') as store:
        mappability_table = store.select('chromosome_'+chromosome, 'quality >= map_qual_threshold')

    mappability = np.zeros(max_chromosome_length, dtype=np.uint8)

    for start, end in mappability_table[['start', 'end']].values:
        end = min(end, max_chromosome_length)
        mappability[start:end] = 1

    return mappability 

def collate(collate_filename, experiment_filename, init_results_filename, fit_results_filenames, config):

    # Extract the statistics for selecting solutions
    stats_table = list()
    for init_id, results_filename in fit_results_filenames.items():
        results = pickle.load(open(results_filename, 'rb'))
        stats = dict(results['stats'])
        stats['init_id'] = init_id
        stats_table.append(stats)
    stats_table = pd.DataFrame(stats_table)

    # Write out selected solutions
    with pd.HDFStore(collate_filename, 'w') as collated:
        collated['stats'] = stats_table

        with pd.HDFStore(init_results_filename, 'r') as results:
            for key, value in results.items():
                collated[key] = results[key]

        with open(experiment_filename, 'rb') as f:
            experiment = pickle.load(f)

        for init_id, results_filename in fit_results_filenames.items():
            results = pickle.load(open(results_filename, 'rb'))
            store_fit_results(collated, experiment, results, 'solutions/solution_{0}'.format(init_id))

        store_optimal_solution(stats_table, collated, config) 

def simulate_germline_alleles(germline_alleles_filename, params, config, ref_data_dir):

    np.random.seed(params['random_seed'])
    
    with pd.HDFStore(germline_alleles_filename, 'w', complevel=9, complib='zlib') as germline_alleles_store:
        for chromosome in params['chromosomes']:
            alleles_table = remixt.simulations.haplotype.create_sim_alleles(chromosome, config, ref_data_dir)
            germline_alleles_store.put('/chromosome_{}'.format(chromosome), alleles_table, format='table') 

def simulate_tumour_data(read_data_filename, mixture_filename, germline_alleles_filename, params):

    with open(mixture_filename, 'r') as mixture_file:
        gm = pickle.load(mixture_file)

    germline_alleles = pd.HDFStore(germline_alleles_filename, 'r')

    np.random.seed(params['random_seed'])

    remixt.simulations.seqread.simulate_mixture_read_data(
        read_data_filename,
        gm.genome_collection.genomes,
        gm.frac * params['h_total'],
        germline_alleles,
        params) 

def merge_evaluations(merged_filename, sim_defs, evaluation_filenames, key_names):
    """ Merge multiple evaluations of prediction results.

    Args:
        merged_filename (str): output hdf filename of merged evaluations
        sim_defs (dict): simulation definitions per simulation
        evaluation_filenames (dict of str): hdf filename of evaluation per simulation per tool

    """
    
    merged_store = pd.HDFStore(merged_filename, 'w')

    sim_defs_table = pd.DataFrame(
        sim_defs.values(),
        index=pd.Index(sim_defs.keys(), name='sim_id'),
    ).reset_index()

    merged_store['/simulations'] = sim_defs_table

    tables = collections.defaultdict(list)
    for key, evaluation_filename in evaluation_filenames.items():
        store = pd.HDFStore(evaluation_filename, 'r')

        if not isinstance(key, tuple):
            key = (key,)

        for table_name in ('/cn_evaluation', '/brk_cn_evaluation', '/mix_results', 'outlier_evaluation'):
            if table_name not in store:
                continue
            table = store[table_name]
            for value, name in zip(key, key_names):
                table[name] = value
            tables[table_name].append(table)

        merged_store['/brk_cn_table/' + '/'.join(key)] = store['/brk_cn_table']
    
    for table_name, table in tables.items():
        merged_store[table_name] = pd.DataFrame(table) 

def merge_overlapping_seqdata(outfile, infiles, chromosomes):
    out_store = pd.HDFStore(outfile, 'w', complevel=9, complib='blosc')

    index_offsets = pd.Series(0, index=chromosomes, dtype=np.int64)

    for _id, infile in infiles.items():
        store = pd.HDFStore(infile)
        tables = store.keys()

        for chromosome in chromosomes:
            allele_table = '/alleles/chromosome_{}'.format(chromosome)
            fragment_table = '/fragments/chromosome_{}'.format(chromosome)

            if allele_table not in tables:
                print("missing table {}".format(allele_table))
                continue

            if fragment_table not in tables:
                print("missing table {}".format(fragment_table))
                continue

            alleles = store[allele_table]
            fragments = store[fragment_table]

            alleles['fragment_id'] = alleles['fragment_id'].astype(np.int64)
            fragments['fragment_id'] = fragments['fragment_id'].astype(np.int64)

            alleles['fragment_id'] += index_offsets[chromosome]
            fragments['fragment_id'] += index_offsets[chromosome]

            index_offsets[chromosome] = max(alleles['fragment_id'].max(), fragments['fragment_id'].max()) + 1

            out_store.append('/alleles/chromosome_{}'.format(chromosome), alleles)
            out_store.append('/fragments/chromosome_{}'.format(chromosome), fragments)

        store.close()
    out_store.close() 

def create_chromosome_seqdata(seqdata_filename, bam_filename, snp_filename, chromosome, max_fragment_length, max_soft_clipped, check_proper_pair):
    """ Create seqdata from bam for one chromosome.

    Args:
        seqdata_filename(str): seqdata hdf store to write to
        bam_filename(str): bam from which to extract read information
        snp_filename(str): TSV chromosome, position file listing SNPs
        chromosome(str): chromosome to extract
        max_fragment_length(int): maximum length of fragments generating paired reads
        max_soft_clipped(int): maximum soft clipping for considering a read concordant
        check_proper_pair(boo): check proper pair flag

    """

    reader = remixt.bamreader.AlleleReader(
        bam_filename,
        snp_filename,
        chromosome,
        max_fragment_length,
        max_soft_clipped,
        check_proper_pair,
    )

    with pd.HDFStore(seqdata_filename, 'w', complevel=9, complib='zlib') as store:
        while reader.ReadAlignments(10000000):
            _unique_index_append(store, _get_key('fragments', chromosome), reader.GetFragmentTable())
            _unique_index_append(store, _get_key('alleles', chromosome), reader.GetAlleleTable()) 

def merge_seqdata(out_filename, in_filenames):
    """ Merge seqdata files for non-overlapping sets of chromosomes

    Args:
        out_filename(str): seqdata hdf store to write to
        out_filename(dict): seqdata hdf store to read from

    """

    with pd.HDFStore(out_filename, 'w', complevel=9, complib='zlib') as out_store:
        for in_filename in in_filenames.values():
            with pd.HDFStore(in_filename, 'r') as in_store:
                for key in in_store.keys():
                    out_store.put(key, in_store[key], format='table') 

def __init__(self, seqdata_filename):
        """ Streaming writer of seq data hdf5 files 

        Args:
            seqdata_filename (str): name of seqdata hdf5 file

        """

        self.store = pd.HDFStore(seqdata_filename, 'w', complevel=9, complib='zlib') 

def log_pd_init(config):
    """log_pd_init

    global init for logging to hdf5 tables via pandas

    Arguments:
        - config: the global configuration dictionary
    """
    global log_store
    log_store_filename = '%s_log_pd.h5' % (config['params']['datafile_expr'], )
    log_store = pd.HDFStore(log_store_filename)
    # experiment = "%s" % (config['params']['id'])
    # log_store = pd.HDFStore("data/%s_pd.h5" % (experiment))
    # print "logging.log_pd_init log_store.root = %s" % (log_store.root) 

def load_hdf(args):
    h5f = None
    if os.path.exists(args.file):
        h5f = pd.HDFStore(args.file)
    else:
        print(("Couldn't load file %s" % (args.file, )))
    return h5f 

def store_hdf5(file_name, key, val):
    with pd.HDFStore(file_name) as store:
        store[key] = val
        store.close() 

def from_hdf5(key, file_name=h5file):
    with pd.HDFStore(file_name) as store:
        data =  store[key]
        store.close()
    return data 

def __init__(self,filename):
      self.store = pd.HDFStore(filename)
      self.d = {} # keep track of in memory pandas data frames so as not to load multiple times form disk
      self.coeffs = pd.DataFrame() # keep all coefficients in memory
      self.attrs = {} # keep all computed outputs in memory
      self.savetbls = [] # names of tables to write to output hdf5 

def save_output(self,filename):
    outstore = pd.HDFStore(filename)
    for key in self.attrs.keys():
      df = self.attrs[key]
      df = misc.df64bitto32bit(df)
      print key+"\n", df.describe()
      outstore[key] = df
    
    for key in self.savetbls:
      df = self.fetch(key)
      df = misc.df64bitto32bit(df)
      print key+"\n", df.describe()
      outstore[key] = df

    outstore.close() 

def copy_coeffs(self,filename=None):
    if filename: coeffstore = pd.HDFStore(filename)
    else: coeffstore = self.store
    self.coeffs = coeffstore['coefficients'] 
    coeffstore.close() 

def save_coeffs(self,filename):
    outstore = pd.HDFStore(filename)
    print self.coeffs
    outstore['coefficients'] = self.coeffs 
    outstore.close() 

def test_format_kwarg_in_constructor(self):
        # GH 13291
        with ensure_clean_path(self.path) as path:
            with pytest.raises(ValueError):
                HDFStore(path, format="table") 

def test_path_pathlib_hdfstore(self):
        df = tm.makeDataFrame()

        def writer(path):
            with pd.HDFStore(path) as store:
                df.to_hdf(store, "df")

        def reader(path):
            with pd.HDFStore(path) as store:
                return pd.read_hdf(store, "df")

        result = tm.round_trip_pathlib(writer, reader)
        tm.assert_frame_equal(df, result) 

def test_path_localpath_hdfstore(self):
        df = tm.makeDataFrame()

        def writer(path):
            with pd.HDFStore(path) as store:
                df.to_hdf(store, "df")

        def reader(path):
            with pd.HDFStore(path) as store:
                return pd.read_hdf(store, "df")

        result = tm.round_trip_localpath(writer, reader)
        tm.assert_frame_equal(df, result) 

def test_read_hdf_errors(self):
        df = DataFrame(np.random.rand(4, 5), index=list("abcd"), columns=list("ABCDE"))

        with ensure_clean_path(self.path) as path:
            with pytest.raises(IOError):
                read_hdf(path, "key")

            df.to_hdf(path, "df")
            store = HDFStore(path, mode="r")
            store.close()

            with pytest.raises(IOError):
                read_hdf(store, "df") 

def test_read_nokey_empty(self):
        with ensure_clean_path(self.path) as path:
            store = HDFStore(path)
            store.close()

            with pytest.raises(ValueError):
                read_hdf(path) 

def test_fspath(self):
        with tm.ensure_clean("foo.h5") as path:
            with pd.HDFStore(path) as store:
                assert os.fspath(store) == str(path) 

def test_select_empty_where(self, where):
        # GH26610

        # Using keyword `where` as '' or (), or [None], etc
        # while reading from HDF store raises
        # "SyntaxError: only a single expression is allowed"

        df = pd.DataFrame([1, 2, 3])
        with ensure_clean_path("empty_where.h5") as path:
            with pd.HDFStore(path) as store:
                store.put("df", df, "t")
                result = pd.read_hdf(store, "df", where=where)
                assert_frame_equal(result, df) 

def test_read_with_where_tz_aware_index(self):
        # GH 11926
        periods = 10
        dts = pd.date_range("20151201", periods=periods, freq="D", tz="UTC")
        mi = pd.MultiIndex.from_arrays([dts, range(periods)], names=["DATE", "NO"])
        expected = pd.DataFrame({"MYCOL": 0}, index=mi)

        key = "mykey"
        with ensure_clean_path(self.path) as path:
            with pd.HDFStore(path) as store:
                store.append(key, expected, format="table", append=True)
            result = pd.read_hdf(path, key, where="DATE > 20151130")
            assert_frame_equal(result, expected) 

def test_format_kwarg_in_constructor(self):
        # GH 13291
        with ensure_clean_path(self.path) as path:
            pytest.raises(ValueError, HDFStore, path, format='table') 

def test_api_default_format(self):

        # default_format option
        with ensure_clean_store(self.path) as store:
            df = tm.makeDataFrame()

            pd.set_option('io.hdf.default_format', 'fixed')
            _maybe_remove(store, 'df')
            store.put('df', df)
            assert not store.get_storer('df').is_table
            pytest.raises(ValueError, store.append, 'df2', df)

            pd.set_option('io.hdf.default_format', 'table')
            _maybe_remove(store, 'df')
            store.put('df', df)
            assert store.get_storer('df').is_table
            _maybe_remove(store, 'df2')
            store.append('df2', df)
            assert store.get_storer('df').is_table

            pd.set_option('io.hdf.default_format', None)

        with ensure_clean_path(self.path) as path:

            df = tm.makeDataFrame()

            pd.set_option('io.hdf.default_format', 'fixed')
            df.to_hdf(path, 'df')
            with HDFStore(path) as store:
                assert not store.get_storer('df').is_table
            pytest.raises(ValueError, df.to_hdf, path, 'df2', append=True)

            pd.set_option('io.hdf.default_format', 'table')
            df.to_hdf(path, 'df3')
            with HDFStore(path) as store:
                assert store.get_storer('df3').is_table
            df.to_hdf(path, 'df4', append=True)
            with HDFStore(path) as store:
                assert store.get_storer('df4').is_table

            pd.set_option('io.hdf.default_format', None) 

def test_path_pathlib_hdfstore(self):
        df = tm.makeDataFrame()

        def writer(path):
            with pd.HDFStore(path) as store:
                df.to_hdf(store, 'df')

        def reader(path):
            with pd.HDFStore(path) as store:
                return pd.read_hdf(store, 'df')

        result = tm.round_trip_pathlib(writer, reader)
        tm.assert_frame_equal(df, result) 

def test_path_localpath_hdfstore(self):
        df = tm.makeDataFrame()

        def writer(path):
            with pd.HDFStore(path) as store:
                df.to_hdf(store, 'df')

        def reader(path):
            with pd.HDFStore(path) as store:
                return pd.read_hdf(store, 'df')

        result = tm.round_trip_localpath(writer, reader)
        tm.assert_frame_equal(df, result)