Python pandas.compat.iteritems() Examples

def _reindex_columns(self, columns, method, copy, level, fill_value=None,
                         limit=None, takeable=False):
        if level is not None:
            raise TypeError('Reindex by level not supported for sparse')

        if notna(fill_value):
            raise NotImplementedError("'fill_value' argument is not supported")

        if limit:
            raise NotImplementedError("'limit' argument is not supported")

        if method is not None:
            raise NotImplementedError("'method' argument is not supported")

        # TODO: fill value handling
        sdict = {k: v for k, v in compat.iteritems(self) if k in columns}
        return self._constructor(
            sdict, index=self.index, columns=columns,
            default_fill_value=self._default_fill_value).__finalize__(self) 

def test_creating_and_reading_multiple_sheets(self, ext):
        # see gh-9450
        #
        # Test reading multiple sheets, from a runtime
        # created Excel file with multiple sheets.
        def tdf(col_sheet_name):
            d, i = [11, 22, 33], [1, 2, 3]
            return DataFrame(d, i, columns=[col_sheet_name])

        sheets = ["AAA", "BBB", "CCC"]

        dfs = [tdf(s) for s in sheets]
        dfs = dict(zip(sheets, dfs))

        with ensure_clean(ext) as pth:
            with ExcelWriter(pth) as ew:
                for sheetname, df in iteritems(dfs):
                    df.to_excel(ew, sheetname)

            dfs_returned = read_excel(pth, sheet_name=sheets, index_col=0)

            for s in sheets:
                tm.assert_frame_equal(dfs[s], dfs_returned[s]) 

def _unpickle_sparse_frame_compat(self, state):
        """ original pickle format """
        series, cols, idx, fv, kind = state

        if not isinstance(cols, Index):  # pragma: no cover
            from pandas.io.pickle import _unpickle_array
            columns = _unpickle_array(cols)
        else:
            columns = cols

        if not isinstance(idx, Index):  # pragma: no cover
            from pandas.io.pickle import _unpickle_array
            index = _unpickle_array(idx)
        else:
            index = idx

        series_dict = DataFrame()
        for col, (sp_index, sp_values) in compat.iteritems(series):
            series_dict[col] = SparseSeries(sp_values, sparse_index=sp_index,
                                            fill_value=fv)

        self._data = to_manager(series_dict, columns, index)
        self._default_fill_value = fv
        self._default_kind = kind 

def test_replace_input_formats_scalar(self):
        df = DataFrame({'A': [np.nan, 0, np.inf], 'B': [0, 2, 5],
                        'C': ['', 'asdf', 'fd']})

        # dict to scalar
        to_rep = {'A': np.nan, 'B': 0, 'C': ''}
        filled = df.replace(to_rep, 0)
        expected = {k: v.replace(to_rep[k], 0)
                    for k, v in compat.iteritems(df)}
        assert_frame_equal(filled, DataFrame(expected))

        pytest.raises(TypeError, df.replace, to_rep, [np.nan, 0, ''])

        # list to scalar
        to_rep = [np.nan, 0, '']
        result = df.replace(to_rep, -1)
        expected = df.copy()
        for i in range(len(to_rep)):
            expected.replace(to_rep[i], -1, inplace=True)
        assert_frame_equal(result, expected) 

def test_apply_nanoseconds(self):
        tests = []

        tests.append((BusinessHour(),
                      {Timestamp('2014-07-04 15:00') + Nano(5): Timestamp(
                          '2014-07-04 16:00') + Nano(5),
                       Timestamp('2014-07-04 16:00') + Nano(5): Timestamp(
                           '2014-07-07 09:00') + Nano(5),
                       Timestamp('2014-07-04 16:00') - Nano(5): Timestamp(
                           '2014-07-04 17:00') - Nano(5)}))

        tests.append((BusinessHour(-1),
                      {Timestamp('2014-07-04 15:00') + Nano(5): Timestamp(
                          '2014-07-04 14:00') + Nano(5),
                       Timestamp('2014-07-04 10:00') + Nano(5): Timestamp(
                           '2014-07-04 09:00') + Nano(5),
                       Timestamp('2014-07-04 10:00') - Nano(5): Timestamp(
                           '2014-07-03 17:00') - Nano(5), }))

        for offset, cases in tests:
            for base, expected in compat.iteritems(cases):
                assert_offset_equal(offset, base, expected) 

def test_equals(self):

        for name, idx in compat.iteritems(self.indices):
            assert idx.equals(idx)
            assert idx.equals(idx.copy())
            assert idx.equals(idx.astype(object))

            assert not idx.equals(list(idx))
            assert not idx.equals(np.array(idx))

            # Cannot pass in non-int64 dtype to RangeIndex
            if not isinstance(idx, RangeIndex):
                same_values = Index(idx, dtype=object)
                assert idx.equals(same_values)
                assert same_values.equals(idx)

            if idx.nlevels == 1:
                # do not test MultiIndex
                assert not idx.equals(pd.Series(idx)) 

def test_delete_base(self):

        for name, idx in compat.iteritems(self.indices):

            if not len(idx):
                continue

            if isinstance(idx, RangeIndex):
                # tested in class
                continue

            expected = idx[1:]
            result = idx.delete(0)
            assert result.equals(expected)
            assert result.name == expected.name

            expected = idx[:-1]
            result = idx.delete(-1)
            assert result.equals(expected)
            assert result.name == expected.name

            with pytest.raises((IndexError, ValueError)):
                # either depending on numpy version
                idx.delete(len(idx)) 

def _combine_match_index(self, other, func, level=None):
        new_data = {}

        if level is not None:
            raise NotImplementedError("'level' argument is not supported")

        this, other = self.align(other, join='outer', axis=0, level=level,
                                 copy=False)

        for col, series in compat.iteritems(this):
            new_data[col] = func(series.values, other.values)

        fill_value = self._get_op_result_fill_value(other, func)

        return self._constructor(
            new_data, index=this.index, columns=self.columns,
            default_fill_value=fill_value).__finalize__(self) 

def test_infer_freq_tz(self):

        freqs = {'AS-JAN':
                 ['2009-01-01', '2010-01-01', '2011-01-01', '2012-01-01'],
                 'Q-OCT':
                 ['2009-01-31', '2009-04-30', '2009-07-31', '2009-10-31'],
                 'M': ['2010-11-30', '2010-12-31', '2011-01-31', '2011-02-28'],
                 'W-SAT':
                 ['2010-12-25', '2011-01-01', '2011-01-08', '2011-01-15'],
                 'D': ['2011-01-01', '2011-01-02', '2011-01-03', '2011-01-04'],
                 'H': ['2011-12-31 22:00', '2011-12-31 23:00',
                       '2012-01-01 00:00', '2012-01-01 01:00']}

        # GH 7310
        for tz in [None, 'Australia/Sydney', 'Asia/Tokyo', 'Europe/Paris',
                   'US/Pacific', 'US/Eastern']:
            for expected, dates in compat.iteritems(freqs):
                idx = DatetimeIndex(dates, tz=tz)
                assert idx.inferred_freq == expected 

def test_freq_code(self):
        assert get_freq('A') == 1000
        assert get_freq('3A') == 1000
        assert get_freq('-1A') == 1000

        assert get_freq('Y') == 1000
        assert get_freq('3Y') == 1000
        assert get_freq('-1Y') == 1000

        assert get_freq('W') == 4000
        assert get_freq('W-MON') == 4001
        assert get_freq('W-FRI') == 4005

        for freqstr, code in compat.iteritems(_period_code_map):
            result = get_freq(freqstr)
            assert result == code

            result = resolution.get_freq_group(freqstr)
            assert result == code // 1000 * 1000

            result = resolution.get_freq_group(code)
            assert result == code // 1000 * 1000 

def test_memory_usage(self):
        for name, index in compat.iteritems(self.indices):
            result = index.memory_usage()
            if len(index):
                index.get_loc(index[0])
                result2 = index.memory_usage()
                result3 = index.memory_usage(deep=True)

                # RangeIndex, IntervalIndex
                # don't have engines
                if not isinstance(index, (RangeIndex, IntervalIndex)):
                    assert result2 > result

                if index.inferred_type == 'object':
                    assert result3 > result2

            else:

                # we report 0 for no-length
                assert result == 0 

def _iter_data(self, data=None, keep_index=False, fillna=None):
        if data is None:
            data = self.data
        if fillna is not None:
            data = data.fillna(fillna)

        # TODO: unused?
        # if self.sort_columns:
        #     columns = com.try_sort(data.columns)
        # else:
        #     columns = data.columns

        for col, values in data.iteritems():
            if keep_index is True:
                yield col, values
            else:
                yield col, values.values 

def _simple_new(cls, start, stop=None, step=None, name=None,
                    dtype=None, **kwargs):
        result = object.__new__(cls)

        # handle passed None, non-integers
        if start is None and stop is None:
            # empty
            start, stop, step = 0, 0, 1

        if start is None or not is_integer(start):
            try:

                return RangeIndex(start, stop, step, name=name, **kwargs)
            except TypeError:
                return Index(start, stop, step, name=name, **kwargs)

        result._start = start
        result._stop = stop or 0
        result._step = step or 1
        result.name = name
        for k, v in compat.iteritems(kwargs):
            setattr(result, k, v)

        result._reset_identity()
        return result

    # -------------------------------------------------------------------- 

def _reindex_with_indexers(self, reindexers, method=None, fill_value=None,
                               limit=None, copy=False, allow_dups=False):

        if method is not None or limit is not None:
            raise NotImplementedError("cannot reindex with a method or limit "
                                      "with sparse")

        if fill_value is None:
            fill_value = np.nan

        reindexers = {self._get_axis_number(a): val
                      for (a, val) in compat.iteritems(reindexers)}

        index, row_indexer = reindexers.get(0, (None, None))
        columns, col_indexer = reindexers.get(1, (None, None))

        if columns is None:
            columns = self.columns

        new_arrays = {}
        for col in columns:
            if col not in self:
                continue
            if row_indexer is not None:
                new_arrays[col] = algos.take_1d(self[col].get_values(),
                                                row_indexer,
                                                fill_value=fill_value)
            else:
                new_arrays[col] = self[col]

        return self._constructor(new_arrays, index=index,
                                 columns=columns).__finalize__(self) 

def stack_sparse_frame(frame):
    """
    Only makes sense when fill_value is NaN
    """
    lengths = [s.sp_index.npoints for _, s in compat.iteritems(frame)]
    nobs = sum(lengths)

    # this is pretty fast
    minor_codes = np.repeat(np.arange(len(frame.columns)), lengths)

    inds_to_concat = []
    vals_to_concat = []
    # TODO: Figure out whether this can be reached.
    # I think this currently can't be reached because you can't build a
    # SparseDataFrame with a non-np.NaN fill value (fails earlier).
    for _, series in compat.iteritems(frame):
        if not np.isnan(series.fill_value):
            raise TypeError('This routine assumes NaN fill value')

        int_index = series.sp_index.to_int_index()
        inds_to_concat.append(int_index.indices)
        vals_to_concat.append(series.sp_values)

    major_codes = np.concatenate(inds_to_concat)
    stacked_values = np.concatenate(vals_to_concat)
    index = MultiIndex(levels=[frame.index, frame.columns],
                       codes=[major_codes, minor_codes],
                       verify_integrity=False)

    lp = DataFrame(stacked_values.reshape((nobs, 1)), index=index,
                   columns=['foo'])
    return lp.sort_index(level=0) 

def __setstate__(self, state):
        """
        Necessary for making this object picklable.
        """

        if isinstance(state, dict):
            self._data = state.pop('data')
            for k, v in compat.iteritems(state):
                setattr(self, k, v)

        elif isinstance(state, tuple):

            if len(state) == 2:
                nd_state, own_state = state
                data = np.empty(nd_state[1], dtype=nd_state[2])
                np.ndarray.__setstate__(data, nd_state)
                self.name = own_state[0]

            else:  # pragma: no cover
                data = np.empty(state)
                np.ndarray.__setstate__(data, state)

            self._data = data
            self._reset_identity()
        else:
            raise Exception("invalid pickle state") 

def __setstate__(self, state):
        """Necessary for making this object picklable"""
        if not isinstance(state, dict):
            raise Exception('invalid pickle state')

        # Provide compatibility with pre-0.15.0 Categoricals.
        if '_categories' not in state and '_levels' in state:
            state['_categories'] = self.dtype.validate_categories(state.pop(
                '_levels'))
        if '_codes' not in state and 'labels' in state:
            state['_codes'] = coerce_indexer_dtype(
                state.pop('labels'), state['_categories'])

        # 0.16.0 ordered change
        if '_ordered' not in state:

            # >=15.0 < 0.16.0
            if 'ordered' in state:
                state['_ordered'] = state.pop('ordered')
            else:
                state['_ordered'] = False

        # 0.21.0 CategoricalDtype change
        if '_dtype' not in state:
            state['_dtype'] = CategoricalDtype(state['_categories'],
                                               state['_ordered'])

        for k, v in compat.iteritems(state):
            setattr(self, k, v) 

def _simple_new(cls, values, name=None, dtype=None, **kwargs):
        """
        We require that we have a dtype compat for the values. If we are passed
        a non-dtype compat, then coerce using the constructor.

        Must be careful not to recurse.
        """
        if not hasattr(values, 'dtype'):
            if (values is None or not len(values)) and dtype is not None:
                values = np.empty(0, dtype=dtype)
            else:
                values = np.array(values, copy=False)
                if is_object_dtype(values):
                    values = cls(values, name=name, dtype=dtype,
                                 **kwargs)._ndarray_values

        if isinstance(values, (ABCSeries, ABCIndexClass)):
            # Index._data must always be an ndarray.
            # This is no-copy for when _values is an ndarray,
            # which should be always at this point.
            values = np.asarray(values._values)

        result = object.__new__(cls)
        result._data = values
        # _index_data is a (temporary?) fix to ensure that the direct data
        # manipulation we do in `_libs/reduction.pyx` continues to work.
        # We need access to the actual ndarray, since we're messing with
        # data buffers and strides. We don't re-use `_ndarray_values`, since
        # we actually set this value too.
        result._index_data = values
        result.name = name
        for k, v in compat.iteritems(kwargs):
            setattr(result, k, v)
        return result._reset_identity() 

def _clean_dict(d):
    """
    Sanitize dictionary for JSON by converting all keys to strings.

    Parameters
    ----------
    d : dict
        The dictionary to convert.

    Returns
    -------
    cleaned_dict : dict
    """

    return {str(k): v for k, v in compat.iteritems(d)} 

def test_replace_input_formats_listlike(self):
        # both dicts
        to_rep = {'A': np.nan, 'B': 0, 'C': ''}
        values = {'A': 0, 'B': -1, 'C': 'missing'}
        df = DataFrame({'A': [np.nan, 0, np.inf], 'B': [0, 2, 5],
                        'C': ['', 'asdf', 'fd']})
        filled = df.replace(to_rep, values)
        expected = {k: v.replace(to_rep[k], values[k])
                    for k, v in compat.iteritems(df)}
        assert_frame_equal(filled, DataFrame(expected))

        result = df.replace([0, 2, 5], [5, 2, 0])
        expected = DataFrame({'A': [np.nan, 5, np.inf], 'B': [5, 2, 0],
                              'C': ['', 'asdf', 'fd']})
        assert_frame_equal(result, expected)

        # scalar to dict
        values = {'A': 0, 'B': -1, 'C': 'missing'}
        df = DataFrame({'A': [np.nan, 0, np.nan], 'B': [0, 2, 5],
                        'C': ['', 'asdf', 'fd']})
        filled = df.replace(np.nan, values)
        expected = {k: v.replace(np.nan, values[k])
                    for k, v in compat.iteritems(df)}
        assert_frame_equal(filled, DataFrame(expected))

        # list to list
        to_rep = [np.nan, 0, '']
        values = [-2, -1, 'missing']
        result = df.replace(to_rep, values)
        expected = df.copy()
        for i in range(len(to_rep)):
            expected.replace(to_rep[i], values[i], inplace=True)
        assert_frame_equal(result, expected)

        pytest.raises(ValueError, df.replace, to_rep, values[1:]) 

def int_frame():
    """
    Fixture for DataFrame of ints with index of unique strings

    Columns are ['A', 'B', 'C', 'D']
    """
    df = DataFrame({k: v.astype(int)
                   for k, v in compat.iteritems(tm.getSeriesData())})
    # force these all to int64 to avoid platform testing issues
    return DataFrame({c: s for c, s in compat.iteritems(df)}, dtype=np.int64) 

def test_rank_int(self):
        s = self.s.dropna().astype('i8')

        for method, res in compat.iteritems(self.results):
            result = s.rank(method=method)
            expected = Series(res).dropna()
            expected.index = result.index
            assert_series_equal(result, expected) 

def test_reindex(test_data):
    identity = test_data.series.reindex(test_data.series.index)

    # __array_interface__ is not defined for older numpies
    # and on some pythons
    try:
        assert np.may_share_memory(test_data.series.index, identity.index)
    except AttributeError:
        pass

    assert identity.index.is_(test_data.series.index)
    assert identity.index.identical(test_data.series.index)

    subIndex = test_data.series.index[10:20]
    subSeries = test_data.series.reindex(subIndex)

    for idx, val in compat.iteritems(subSeries):
        assert val == test_data.series[idx]

    subIndex2 = test_data.ts.index[10:20]
    subTS = test_data.ts.reindex(subIndex2)

    for idx, val in compat.iteritems(subTS):
        assert val == test_data.ts[idx]
    stuffSeries = test_data.ts.reindex(subIndex)

    assert np.isnan(stuffSeries).all()

    # This is extremely important for the Cython code to not screw up
    nonContigIndex = test_data.ts.index[::2]
    subNonContig = test_data.ts.reindex(nonContigIndex)
    for idx, val in compat.iteritems(subNonContig):
        assert val == test_data.ts[idx]

    # return a copy the same index here
    result = test_data.ts.reindex()
    assert not (result is test_data.ts) 

def test_append(self, datetime_series, string_series, object_series):
        appendedSeries = string_series.append(object_series)
        for idx, value in compat.iteritems(appendedSeries):
            if idx in string_series.index:
                assert value == string_series[idx]
            elif idx in object_series.index:
                assert value == object_series[idx]
            else:
                raise AssertionError("orphaned index!")

        msg = "Indexes have overlapping values:"
        with pytest.raises(ValueError, match=msg):
            datetime_series.append(datetime_series, verify_integrity=True) 

def test_constructor_subclass_dict(self):
        data = tm.TestSubDict((x, 10.0 * x) for x in range(10))
        series = self.series_klass(data)
        expected = self.series_klass(dict(compat.iteritems(data)))
        self._assert_series_equal(series, expected) 

def test_offset(self, case):
        offset, cases = case
        for base, expected in compat.iteritems(cases):
            assert_offset_equal(offset, base, expected) 

def test_offset(self, case):
        offset, cases = case
        for base, expected in compat.iteritems(cases):
            assert_offset_equal(offset, base, expected) 

def test_offset(self, case):
        offset, cases = case
        for base, expected in compat.iteritems(cases):
            assert_offset_equal(offset, base, expected) 

def test_offset(self, case):
        offset, cases = case
        for base, expected in compat.iteritems(cases):
            assert_offset_equal(offset, base, expected) 

def test_offset(self, case):
        offset, cases = case
        for base, expected in compat.iteritems(cases):
            assert_offset_equal(offset, base, expected)