Python astropy.table.vstack() Examples

def test_stack_incompatible(self, operation_table_type):
        self._setup(operation_table_type)
        with pytest.raises(TableMergeError) as excinfo:
            table.vstack([self.t1, self.t3], join_type='inner')
        assert ("The 'b' columns have incompatible types: {}"
                .format([self.t1['b'].dtype.name, self.t3['b'].dtype.name])
                in str(excinfo.value))

        with pytest.raises(TableMergeError) as excinfo:
            table.vstack([self.t1, self.t3], join_type='outer')
        assert "The 'b' columns have incompatible types:" in str(excinfo.value)

        with pytest.raises(TableMergeError):
            table.vstack([self.t1, self.t2], join_type='exact')

        t1_reshape = self.t1.copy()
        t1_reshape['b'].shape = [2, 1]
        with pytest.raises(TableMergeError) as excinfo:
            table.vstack([self.t1, t1_reshape])
        assert "have different shape" in str(excinfo.value) 

def get_data(input_files):
    """read in data from one or more files. In the case that there are multiple files, the data from all files
    will be combined into a single CI vs FWHM dataset.

    Parameters
    ----------
    input_files : list
        list of CI vs. FWHM .csv files to process

    Returns
    -------
    data_table: : numpy.ndarray
        A 2 x n sized numpy array. Column 1: CI values; Column 2: FWHM values
    """
    data_table = Table()
    for filename in input_files:
        if filename.endswith(".csv"):
            data_table = vstack([data_table,Table.read(filename,format='ascii.csv')])
        if filename.endswith(".json"):
            json_data = read_json_file(filename)
            data_table = vstack([data_table, json_data['data']['CI_FWHM']])
    return data_table

# -~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~-~- 

def test_masking_required_exception():
    """
    Test that outer join, hstack and vstack fail for a mixin column which
    does not support masking.
    """
    col = [1, 2, 3, 4] * u.m
    t1 = table.QTable([[1, 2, 3, 4], col], names=['a', 'b'])
    t2 = table.QTable([[1, 2], col[:2]], names=['a', 'c'])

    with pytest.raises(NotImplementedError) as err:
        table.vstack([t1, t2], join_type='outer')
    assert 'vstack requires masking' in str(err.value)

    with pytest.raises(NotImplementedError) as err:
        table.hstack([t1, t2], join_type='outer')
    assert 'hstack requires masking' in str(err.value)

    with pytest.raises(NotImplementedError) as err:
        table.join(t1, t2, join_type='outer')
    assert 'join requires masking' in str(err.value) 

def stitch(self, corrector_func=lambda x:x.normalize()):
        """ Stitch all light curves in the collection into a single lk.LightCurve

        Any function passed to `corrector_func` will be applied to each light curve
        before stitching. For example, passing "lambda x: x.normalize().flatten()"
        will normalize and flatten each light curve before stitching.

        Parameters
        ----------
        corrector_func : function
            Function that accepts and returns a `~lightkurve.lightcurve.LightCurve`.
            This function is applied to each light curve in the collection
            prior to stitching. The default is to normalize each light curve.

        Returns
        -------
        lc : `~lightkurve.lightcurve.LightCurve`
            Stitched light curve.
        """
        if corrector_func is None:
            corrector_func = lambda x: x
        lcs = [corrector_func(lc) for lc in self]
        # Need `join_type='inner'` until AstroPy supports masked Quantities
        return vstack(lcs, join_type='inner', metadata_conflicts='silent') 

def append(self, others: Iterable[LightCurve], inplace=False) -> LightCurve:
        """Append one or more other `LightCurve` object(s) to this one.

        Parameters
        ----------
        others : `LightCurve`, or list of `LightCurve`
            Light curve(s) to be appended to the current one.
        inplace : bool
            If True, change the current `LightCurve` instance in place instead
            of creating and returning a new one. Defaults to False.

        Returns
        -------
        new_lc : `LightCurve`
            Light curve which has the other light curves appened to it.
        """
        if inplace:
            raise ValueError("the `inplace` parameter is no longer supported "
                             "as of Lightkurve v2.0")
        if not hasattr(others, '__iter__'):
            others = (others,)
        # Need `join_type='inner'` until AstroPy supports masked Quantities
        return vstack((self, *others), join_type='inner', metadata_conflicts='silent') 

def test_vstack_one_table(self, operation_table_type):
        self._setup(operation_table_type)
        """Regression test for issue #3313"""
        assert (self.t1 == table.vstack(self.t1)).all()
        assert (self.t1 == table.vstack([self.t1])).all() 

def test_read_chunks_formats(masked):
    """
    Test different supported formats for chunked reading.
    """
    t1 = simple_table(size=102, cols=10, kinds='fS', masked=masked)
    for i, name in enumerate(t1.colnames):
        t1.rename_column(name, 'col{}'.format(i + 1))

    # TO DO commented_header does not currently work due to the special-cased
    # implementation of header parsing.

    for format in 'tab', 'csv', 'no_header', 'rdb', 'basic':
        out = StringIO()
        ascii.write(t1, out, format=format)
        t_gen = ascii.read(out.getvalue(), format=format,
                           fast_reader={'chunk_size': 400, 'chunk_generator': True})
        ts = list(t_gen)
        for t in ts:
            for col, col1 in zip(t.columns.values(), t1.columns.values()):
                assert col.name == col1.name
                assert col.dtype.kind == col1.dtype.kind

        assert len(ts) > 4
        t2 = table.vstack(ts)
        assert np.all(t1 == t2)

        # Now read the full table in chunks
        t3 = ascii.read(out.getvalue(), format=format, fast_reader={'chunk_size': 400})
        assert np.all(t1 == t3) 

def test_read_chunks_input_types():
    """
    Test chunked reading for different input types: file path, file object,
    and string input.
    """
    fpath = 'data/test5.dat'
    t1 = ascii.read(fpath, header_start=1, data_start=3, )

    for fp in (fpath, open(fpath, 'r'), open(fpath, 'r').read()):
        t_gen = ascii.read(fp, header_start=1, data_start=3,
                           guess=False, format='fast_basic',
                           fast_reader={'chunk_size': 400, 'chunk_generator': True})
        ts = list(t_gen)
        for t in ts:
            for col, col1 in zip(t.columns.values(), t1.columns.values()):
                assert col.name == col1.name
                assert col.dtype.kind == col1.dtype.kind

        assert len(ts) == 4
        t2 = table.vstack(ts)
        assert np.all(t1 == t2)

    for fp in (fpath, open(fpath, 'r'), open(fpath, 'r').read()):
        # Now read the full table in chunks
        t3 = ascii.read(fp, header_start=1, data_start=3,
                        fast_reader={'chunk_size': 300})
        assert np.all(t1 == t3) 

def print_headers_as_table(args):
    """Prints FITS header(s) in a machine-readable table format.

    Parameters
    ----------
    args : argparse.Namespace
        Arguments passed from the command-line as defined below.
    """
    tables = []
    # Create a Table object for each file
    for filename in args.filename:  # Support wildcards
        formatter = None
        try:
            formatter = TableHeaderFormatter(filename)
            tbl = formatter.parse(args.extensions,
                                  args.keywords,
                                  args.compressed)
            if tbl:
                tables.append(tbl)
        except OSError as e:
            log.error(str(e))  # file not found or unreadable
        finally:
            if formatter:
                formatter.close()

    # Concatenate the tables
    if len(tables) == 0:
        return False
    elif len(tables) == 1:
        resulting_table = tables[0]
    else:
        from astropy import table
        resulting_table = table.vstack(tables)
    # Print the string representation of the concatenated table
    resulting_table.write(sys.stdout, format=args.table) 

def test_vstack_unicode():
    """
    Test for problem related to issue #5617 when vstack'ing *unicode*
    columns.  In this case the character size gets multiplied by 4.
    """
    t = table.Table([['a']], names=['a'])
    assert t['a'].itemsize == 4  # 4-byte / char for U dtype

    t2 = table.vstack([t, t])
    assert len(t2) == 2
    assert t2['a'].itemsize == 4 

def test_vstack_bytes(operation_table_type):
    """
    Test for issue #5617 when vstack'ing bytes columns in Py3.
    This is really an upsteam numpy issue numpy/numpy/#8403.
    """
    t = operation_table_type([[b'a']], names=['a'])
    assert t['a'].itemsize == 1

    t2 = table.vstack([t, t])
    assert len(t2) == 2
    assert t2['a'].itemsize == 1 

def test_mixin_functionality(self, mixin_cols):
        col = mixin_cols['m']
        len_col = len(col)
        t = table.QTable([col], names=['a'])
        cls_name = type(col).__name__

        # Vstack works for these classes:
        implemented_mixin_classes = ['Quantity', 'Angle', 'Time',
                                     'Latitude', 'Longitude',
                                     'EarthLocation']
        if cls_name in implemented_mixin_classes:
            out = table.vstack([t, t])
            assert len(out) == len_col * 2
            assert np.all(out['a'][:len_col] == col)
            assert np.all(out['a'][len_col:] == col)
        else:
            with pytest.raises(NotImplementedError) as err:
                table.vstack([t, t])
            assert ('vstack unavailable for mixin column type(s): {}'
                    .format(cls_name) in str(err.value))

        # Check for outer stack which requires masking.  Only Time supports
        # this currently.
        t2 = table.QTable([col], names=['b'])  # different from col name for t
        if cls_name == 'Time':
            out = table.vstack([t, t2], join_type='outer')
            assert len(out) == len_col * 2
            assert np.all(out['a'][:len_col] == col)
            assert np.all(out['b'][len_col:] == col)
            assert np.all(out['a'].mask == [False] * len_col + [True] * len_col)
            assert np.all(out['b'].mask == [True] * len_col + [False] * len_col)
            # check directly stacking mixin columns:
            out2 = table.vstack([t, t2['b']])
            assert np.all(out['a'] == out2['a'])
            assert np.all(out['b'] == out2['b'])
        else:
            with pytest.raises(NotImplementedError) as err:
                table.vstack([t, t2], join_type='outer')
            assert ('vstack requires masking' in str(err.value) or
                    'vstack unavailable' in str(err.value)) 

def test_required_after_stacking(self):
        # When stacking, we have to temporarily relax the checking of the
        # columns in the time series, but we need to make sure that the
        # checking works again afterwards
        ts = vstack([self.series, self.series])
        with pytest.raises(ValueError) as exc:
            ts.remove_columns(ts.colnames)
        assert 'TimeSeries object is invalid' in exc.value.args[0] 

def test_vstack_one_masked(self, operation_table_type):
        if operation_table_type is QTable:
            pytest.xfail('Quantity columns do not support masking.')
        self._setup(operation_table_type)
        t1 = self.t1
        t4 = self.t4
        t4['b'].mask[1] = True
        t14 = table.vstack([t1, t4])
        assert t14.masked is False
        assert t14.pformat() == [' a   b ',
                                 '--- ---',
                                 '0.0 foo',
                                 '1.0 bar',
                                 '0.0 foo',
                                 '1.0  --'] 

def test_stacking(self):
        ts = vstack([self.series, self.series])
        assert isinstance(ts, self.series.__class__) 

def test_stack_basic_outer(self, operation_table_type):
        if operation_table_type is QTable:
            pytest.xfail('Quantity columns do not support masking.')
        self._setup(operation_table_type)
        t1 = self.t1
        t2 = self.t2
        t4 = self.t4
        t12 = table.vstack([t1, t2], join_type='outer')
        assert t12.masked is False
        assert t12.pformat() == [' a   b   c ',
                                 '--- --- ---',
                                 '0.0 foo  --',
                                 '1.0 bar  --',
                                 '2.0 pez   4',
                                 '3.0 sez   5']

        t124 = table.vstack([t1, t2, t4], join_type='outer')
        assert t124.masked is False
        assert t124.pformat() == [' a   b   c ',
                                  '--- --- ---',
                                  '0.0 foo  --',
                                  '1.0 bar  --',
                                  '2.0 pez   4',
                                  '3.0 sez   5',
                                  '0.0 foo  --',
                                  '1.0 bar  --'] 

def test_stack_basic_inner(self, operation_table_type):
        self._setup(operation_table_type)
        t1 = self.t1
        t2 = self.t2
        t4 = self.t4

        t12 = table.vstack([t1, t2], join_type='inner')
        assert t12.masked is False
        assert type(t12) is operation_table_type
        assert type(t12['a']) is type(t1['a'])
        assert type(t12['b']) is type(t1['b'])
        assert t12.pformat() == [' a   b ',
                                 '--- ---',
                                 '0.0 foo',
                                 '1.0 bar',
                                 '2.0 pez',
                                 '3.0 sez']

        t124 = table.vstack([t1, t2, t4], join_type='inner')
        assert type(t124) is operation_table_type
        assert type(t12['a']) is type(t1['a'])
        assert type(t12['b']) is type(t1['b'])
        assert t124.pformat() == [' a   b ',
                                  '--- ---',
                                  '0.0 foo',
                                  '1.0 bar',
                                  '2.0 pez',
                                  '3.0 sez',
                                  '0.0 foo',
                                  '1.0 bar'] 

def test_bad_input_type(self, operation_table_type):
        self._setup(operation_table_type)
        with pytest.raises(ValueError):
            table.vstack([])
        with pytest.raises(TypeError):
            table.vstack(1)
        with pytest.raises(TypeError):
            table.vstack([self.t2, 1])
        with pytest.raises(ValueError):
            table.vstack([self.t1, self.t2], join_type='invalid join type') 

def test_table_meta_merge(self, operation_table_type):
        self._setup(operation_table_type)
        out = table.vstack([self.t1, self.t2, self.t4], join_type='inner')
        assert out.meta == self.meta_merge 

def test_stack_table_column(self, operation_table_type):
        self._setup(operation_table_type)
        t2 = self.t1.copy()
        t2.meta.clear()
        out = table.vstack([self.t1, t2['a']])
        assert out.masked is False
        assert out.pformat() == [' a   b ',
                                 '--- ---',
                                 '0.0 foo',
                                 '1.0 bar',
                                 '0.0  --',
                                 '1.0  --'] 

def test_stack_rows(self, operation_table_type):
        self._setup(operation_table_type)
        t2 = self.t1.copy()
        t2.meta.clear()
        out = table.vstack([self.t1, t2[1]])
        assert type(out['a']) is type(self.t1['a'])
        assert type(out['b']) is type(self.t1['b'])
        assert out.pformat() == [' a   b ',
                                 '--- ---',
                                 '0.0 foo',
                                 '1.0 bar',
                                 '1.0 bar'] 

def merge_tables(self, tables, direction='vert', **kwargs):
        ''' Merges a list of Astropy tables of results together

        Combines two Astropy tables using either the Astropy
        vstack or hstack method.  vstack refers to vertical stacking of table rows.
        hstack refers to horizonal stacking of table columns.  hstack assumes the rows in each
        table refer to the same object.  Buyer beware: stacking tables without proper understanding
        of your rows and columns may results in deleterious results.

        merge_tables also accepts all keyword arguments that Astropy vstack and hstack method do.
        See `vstack <http://docs.astropy.org/en/stable/table/operations.html#stack-vertically>`_
        See `hstack <http://docs.astropy.org/en/stable/table/operations.html#stack-horizontally>`_

        Parameters:
            tables (list):
                A list of Astropy Table objects.  Required.
            direction (str):
                The direction of the table stacking, either vertical ('vert') or horizontal ('hor').
                Default is 'vert'.  Direction string can be fuzzy.

        Returns:
            A new Astropy table that is the stacked combination of all input tables

        Example:
            >>> # query 1
            >>> q, r = doQuery(search_filter='nsa.z < 0.1', returnparams=['g_r', 'cube.ra', 'cube.dec'])
            >>> # query 2
            >>> q2, r2 = doQuery(search_filter='nsa.z < 0.1')
            >>>
            >>> # convert to tables
            >>> table_1 = r.toTable()
            >>> table_2 = r2.toTable()
            >>> tables = [table_1, table_2]
            >>>
            >>> # vertical (row) stacking
            >>> r.merge_tables(tables, direction='vert')
            >>> # horizontal (column) stacking
            >>> r.merge_tables(tables, direction='hor')

        '''
        choices = ['vertical', 'horizontal']
        stackdir, score = process.extractOne(direction, choices)
        if stackdir == 'vertical':
            return vstack(tables, **kwargs)
        elif stackdir == 'horizontal':
            return hstack(tables, **kwargs) 

def _read_in_chunks(table, **kwargs):
    """
    For fast_reader read the ``table`` in chunks and vstack to create
    a single table, OR return a generator of chunk tables.
    """
    fast_reader = kwargs['fast_reader']
    chunk_size = fast_reader.pop('chunk_size')
    chunk_generator = fast_reader.pop('chunk_generator', False)
    fast_reader['parallel'] = False  # No parallel with chunks

    tbl_chunks = _read_in_chunks_generator(table, chunk_size, **kwargs)
    if chunk_generator:
        return tbl_chunks

    tbl0 = next(tbl_chunks)
    masked = tbl0.masked

    # Numpy won't allow resizing the original so make a copy here.
    out_cols = {col.name: col.data.copy() for col in tbl0.itercols()}

    str_kinds = ('S', 'U')
    for tbl in tbl_chunks:
        masked |= tbl.masked
        for name, col in tbl.columns.items():
            # Concatenate current column data and new column data

            # If one of the inputs is string-like and the other is not, then
            # convert the non-string to a string.  In a perfect world this would
            # be handled by numpy, but as of numpy 1.13 this results in a string
            # dtype that is too long (https://github.com/numpy/numpy/issues/10062).

            col1, col2 = out_cols[name], col.data
            if col1.dtype.kind in str_kinds and col2.dtype.kind not in str_kinds:
                col2 = np.array(col2.tolist(), dtype=col1.dtype.kind)
            elif col2.dtype.kind in str_kinds and col1.dtype.kind not in str_kinds:
                col1 = np.array(col1.tolist(), dtype=col2.dtype.kind)

            # Choose either masked or normal concatenation
            concatenate = np.ma.concatenate if masked else np.concatenate

            out_cols[name] = concatenate([col1, col2])

    # Make final table from numpy arrays, converting dict to list
    out_cols = [out_cols[name] for name in tbl0.colnames]
    out = tbl0.__class__(out_cols, names=tbl0.colnames, meta=tbl0.meta,
                         copy=False)

    return out