Python pandas.core.common.isna() Examples

def test_where_none(self):
        # GH 4667
        # setting with None changes dtype
        df = DataFrame({'series': Series(range(10))}).astype(float)
        df[df > 7] = None
        expected = DataFrame(
            {'series': Series([0, 1, 2, 3, 4, 5, 6, 7, np.nan, np.nan])})
        assert_frame_equal(df, expected)

        # GH 7656
        df = DataFrame([{'A': 1, 'B': np.nan, 'C': 'Test'}, {
                       'A': np.nan, 'B': 'Test', 'C': np.nan}])
        expected = df.where(~isna(df), None)
        with tm.assert_raises_regex(TypeError, 'boolean setting '
                                    'on mixed-type'):
            df.where(~isna(df), None, inplace=True) 

def test_combine_generic(self):
        df1 = self.frame
        df2 = self.frame.loc[self.frame.index[:-5], ['A', 'B', 'C']]

        combined = df1.combine(df2, np.add)
        combined2 = df2.combine(df1, np.add)
        assert combined['D'].isna().all()
        assert combined2['D'].isna().all()

        chunk = combined.loc[combined.index[:-5], ['A', 'B', 'C']]
        chunk2 = combined2.loc[combined2.index[:-5], ['A', 'B', 'C']]

        exp = self.frame.loc[self.frame.index[:-5],
                             ['A', 'B', 'C']].reindex_like(chunk) * 2
        assert_frame_equal(chunk, exp)
        assert_frame_equal(chunk2, exp) 

def test_operators_none_as_na(self, op):
        df = DataFrame({"col1": [2, 5.0, 123, None],
                        "col2": [1, 2, 3, 4]}, dtype=object)

        # since filling converts dtypes from object, changed expected to be
        # object
        filled = df.fillna(np.nan)
        result = op(df, 3)
        expected = op(filled, 3).astype(object)
        expected[com.isna(expected)] = None
        assert_frame_equal(result, expected)

        result = op(df, df)
        expected = op(filled, filled).astype(object)
        expected[com.isna(expected)] = None
        assert_frame_equal(result, expected)

        result = op(df, df.fillna(7))
        assert_frame_equal(result, expected)

        result = op(df.fillna(7), df)
        assert_frame_equal(result, expected, check_dtype=False) 

def test_where_none(self):
        # GH 4667
        # setting with None changes dtype
        df = DataFrame({'series': Series(range(10))}).astype(float)
        df[df > 7] = None
        expected = DataFrame(
            {'series': Series([0, 1, 2, 3, 4, 5, 6, 7, np.nan, np.nan])})
        assert_frame_equal(df, expected)

        # GH 7656
        df = DataFrame([{'A': 1, 'B': np.nan, 'C': 'Test'}, {
                       'A': np.nan, 'B': 'Test', 'C': np.nan}])
        msg = 'boolean setting on mixed-type'

        with pytest.raises(TypeError, match=msg):
            df.where(~isna(df), None, inplace=True) 

def test_where_bug_transposition(self):
        # see gh-7506
        a = DataFrame({0: [1, 2], 1: [3, 4], 2: [5, 6]})
        b = DataFrame({0: [np.nan, 8], 1: [9, np.nan], 2: [np.nan, np.nan]})
        do_not_replace = b.isna() | (a > b)

        expected = a.copy()
        expected[~do_not_replace] = b

        result = a.where(do_not_replace, b)
        assert_frame_equal(result, expected)

        a = DataFrame({0: [4, 6], 1: [1, 0]})
        b = DataFrame({0: [np.nan, 3], 1: [3, np.nan]})
        do_not_replace = b.isna() | (a > b)

        expected = a.copy()
        expected[~do_not_replace] = b

        result = a.where(do_not_replace, b)
        assert_frame_equal(result, expected) 

def test_combine_generic(self):
        df1 = self.frame
        df2 = self.frame.loc[self.frame.index[:-5], ['A', 'B', 'C']]

        combined = df1.combine(df2, np.add)
        combined2 = df2.combine(df1, np.add)
        assert combined['D'].isna().all()
        assert combined2['D'].isna().all()

        chunk = combined.loc[combined.index[:-5], ['A', 'B', 'C']]
        chunk2 = combined2.loc[combined2.index[:-5], ['A', 'B', 'C']]

        exp = self.frame.loc[self.frame.index[:-5],
                             ['A', 'B', 'C']].reindex_like(chunk) * 2
        assert_frame_equal(chunk, exp)
        assert_frame_equal(chunk2, exp) 

def test_operators_none_as_na(self):
        df = DataFrame({"col1": [2, 5.0, 123, None],
                        "col2": [1, 2, 3, 4]}, dtype=object)

        ops = [operator.add, operator.sub, operator.mul, operator.truediv]

        # since filling converts dtypes from object, changed expected to be
        # object
        for op in ops:
            filled = df.fillna(np.nan)
            result = op(df, 3)
            expected = op(filled, 3).astype(object)
            expected[com.isna(expected)] = None
            assert_frame_equal(result, expected)

            result = op(df, df)
            expected = op(filled, filled).astype(object)
            expected[com.isna(expected)] = None
            assert_frame_equal(result, expected)

            result = op(df, df.fillna(7))
            assert_frame_equal(result, expected)

            result = op(df.fillna(7), df)
            assert_frame_equal(result, expected, check_dtype=False) 

def test_where_none(self):
        # GH 4667
        # setting with None changes dtype
        df = DataFrame({'series': Series(range(10))}).astype(float)
        df[df > 7] = None
        expected = DataFrame(
            {'series': Series([0, 1, 2, 3, 4, 5, 6, 7, np.nan, np.nan])})
        assert_frame_equal(df, expected)

        # GH 7656
        df = DataFrame([{'A': 1, 'B': np.nan, 'C': 'Test'}, {
                       'A': np.nan, 'B': 'Test', 'C': np.nan}])
        expected = df.where(~isna(df), None)
        with tm.assert_raises_regex(TypeError, 'boolean setting '
                                    'on mixed-type'):
            df.where(~isna(df), None, inplace=True) 

def test_operators_none_as_na(self):
        df = DataFrame({"col1": [2, 5.0, 123, None],
                        "col2": [1, 2, 3, 4]}, dtype=object)

        ops = [operator.add, operator.sub, operator.mul, operator.truediv]

        # since filling converts dtypes from object, changed expected to be
        # object
        for op in ops:
            filled = df.fillna(np.nan)
            result = op(df, 3)
            expected = op(filled, 3).astype(object)
            expected[com.isna(expected)] = None
            assert_frame_equal(result, expected)

            result = op(df, df)
            expected = op(filled, filled).astype(object)
            expected[com.isna(expected)] = None
            assert_frame_equal(result, expected)

            result = op(df, df.fillna(7))
            assert_frame_equal(result, expected)

            result = op(df.fillna(7), df)
            assert_frame_equal(result, expected, check_dtype=False) 

def test_combine_generic(self):
        df1 = self.frame
        df2 = self.frame.loc[self.frame.index[:-5], ['A', 'B', 'C']]

        combined = df1.combine(df2, np.add)
        combined2 = df2.combine(df1, np.add)
        assert combined['D'].isna().all()
        assert combined2['D'].isna().all()

        chunk = combined.loc[combined.index[:-5], ['A', 'B', 'C']]
        chunk2 = combined2.loc[combined2.index[:-5], ['A', 'B', 'C']]

        exp = self.frame.loc[self.frame.index[:-5],
                             ['A', 'B', 'C']].reindex_like(chunk) * 2
        assert_frame_equal(chunk, exp)
        assert_frame_equal(chunk2, exp) 

def test_operators_none_as_na(self):
        df = DataFrame({"col1": [2, 5.0, 123, None],
                        "col2": [1, 2, 3, 4]}, dtype=object)

        ops = [operator.add, operator.sub, operator.mul, operator.truediv]

        # since filling converts dtypes from object, changed expected to be
        # object
        for op in ops:
            filled = df.fillna(np.nan)
            result = op(df, 3)
            expected = op(filled, 3).astype(object)
            expected[com.isna(expected)] = None
            assert_frame_equal(result, expected)

            result = op(df, df)
            expected = op(filled, filled).astype(object)
            expected[com.isna(expected)] = None
            assert_frame_equal(result, expected)

            result = op(df, df.fillna(7))
            assert_frame_equal(result, expected)

            result = op(df.fillna(7), df)
            assert_frame_equal(result, expected, check_dtype=False) 

def test_where_none(self):
        # GH 4667
        # setting with None changes dtype
        df = DataFrame({'series': Series(range(10))}).astype(float)
        df[df > 7] = None
        expected = DataFrame(
            {'series': Series([0, 1, 2, 3, 4, 5, 6, 7, np.nan, np.nan])})
        assert_frame_equal(df, expected)

        # GH 7656
        df = DataFrame([{'A': 1, 'B': np.nan, 'C': 'Test'}, {
                       'A': np.nan, 'B': 'Test', 'C': np.nan}])
        expected = df.where(~isna(df), None)
        with tm.assert_raises_regex(TypeError, 'boolean setting '
                                    'on mixed-type'):
            df.where(~isna(df), None, inplace=True) 

def test_operators_none_as_na(self, op):
        df = DataFrame({"col1": [2, 5.0, 123, None],
                        "col2": [1, 2, 3, 4]}, dtype=object)

        # since filling converts dtypes from object, changed expected to be
        # object
        filled = df.fillna(np.nan)
        result = op(df, 3)
        expected = op(filled, 3).astype(object)
        expected[com.isna(expected)] = None
        assert_frame_equal(result, expected)

        result = op(df, df)
        expected = op(filled, filled).astype(object)
        expected[com.isna(expected)] = None
        assert_frame_equal(result, expected)

        result = op(df, df.fillna(7))
        assert_frame_equal(result, expected)

        result = op(df.fillna(7), df)
        assert_frame_equal(result, expected, check_dtype=False) 

def test_where_none(self):
        # GH 4667
        # setting with None changes dtype
        df = DataFrame({'series': Series(range(10))}).astype(float)
        df[df > 7] = None
        expected = DataFrame(
            {'series': Series([0, 1, 2, 3, 4, 5, 6, 7, np.nan, np.nan])})
        assert_frame_equal(df, expected)

        # GH 7656
        df = DataFrame([{'A': 1, 'B': np.nan, 'C': 'Test'}, {
                       'A': np.nan, 'B': 'Test', 'C': np.nan}])
        msg = 'boolean setting on mixed-type'

        with pytest.raises(TypeError, match=msg):
            df.where(~isna(df), None, inplace=True) 

def test_where_bug_transposition(self):
        # see gh-7506
        a = DataFrame({0: [1, 2], 1: [3, 4], 2: [5, 6]})
        b = DataFrame({0: [np.nan, 8], 1: [9, np.nan], 2: [np.nan, np.nan]})
        do_not_replace = b.isna() | (a > b)

        expected = a.copy()
        expected[~do_not_replace] = b

        result = a.where(do_not_replace, b)
        assert_frame_equal(result, expected)

        a = DataFrame({0: [4, 6], 1: [1, 0]})
        b = DataFrame({0: [np.nan, 3], 1: [3, np.nan]})
        do_not_replace = b.isna() | (a > b)

        expected = a.copy()
        expected[~do_not_replace] = b

        result = a.where(do_not_replace, b)
        assert_frame_equal(result, expected) 

def test_combine_generic(self):
        df1 = self.frame
        df2 = self.frame.loc[self.frame.index[:-5], ['A', 'B', 'C']]

        combined = df1.combine(df2, np.add)
        combined2 = df2.combine(df1, np.add)
        assert combined['D'].isna().all()
        assert combined2['D'].isna().all()

        chunk = combined.loc[combined.index[:-5], ['A', 'B', 'C']]
        chunk2 = combined2.loc[combined2.index[:-5], ['A', 'B', 'C']]

        exp = self.frame.loc[self.frame.index[:-5],
                             ['A', 'B', 'C']].reindex_like(chunk) * 2
        assert_frame_equal(chunk, exp)
        assert_frame_equal(chunk2, exp) 

def test_setitem_single_column_mixed_datetime(self):
        df = DataFrame(randn(5, 3), index=['a', 'b', 'c', 'd', 'e'],
                       columns=['foo', 'bar', 'baz'])

        df['timestamp'] = Timestamp('20010102')

        # check our dtypes
        result = df.get_dtype_counts()
        expected = Series({'float64': 3, 'datetime64[ns]': 1})
        assert_series_equal(result, expected)

        # set an allowable datetime64 type
        df.loc['b', 'timestamp'] = iNaT
        assert isna(df.loc['b', 'timestamp'])

        # allow this syntax
        df.loc['c', 'timestamp'] = nan
        assert isna(df.loc['c', 'timestamp'])

        # allow this syntax
        df.loc['d', :] = nan
        assert not isna(df.loc['c', :]).all()

        # as of GH 3216 this will now work!
        # try to set with a list like item
        # pytest.raises(
        #    Exception, df.loc.__setitem__, ('d', 'timestamp'), [nan]) 

def test_reindex_frame_add_nat(self):
        rng = date_range('1/1/2000 00:00:00', periods=10, freq='10s')
        df = DataFrame({'A': np.random.randn(len(rng)), 'B': rng})

        result = df.reindex(lrange(15))
        assert np.issubdtype(result['B'].dtype, np.dtype('M8[ns]'))

        mask = com.isna(result)['B']
        assert mask[-5:].all()
        assert not mask[:-5].any() 

def test_getitem_boolean_casting(self):

        # don't upcast if we don't need to
        df = self.tsframe.copy()
        df['E'] = 1
        df['E'] = df['E'].astype('int32')
        df['E1'] = df['E'].copy()
        df['F'] = 1
        df['F'] = df['F'].astype('int64')
        df['F1'] = df['F'].copy()

        casted = df[df > 0]
        result = casted.get_dtype_counts()
        expected = Series({'float64': 4, 'int32': 2, 'int64': 2})
        assert_series_equal(result, expected)

        # int block splitting
        df.loc[df.index[1:3], ['E1', 'F1']] = 0
        casted = df[df > 0]
        result = casted.get_dtype_counts()
        expected = Series({'float64': 6, 'int32': 1, 'int64': 1})
        assert_series_equal(result, expected)

        # where dtype conversions
        # GH 3733
        df = DataFrame(data=np.random.randn(100, 50))
        df = df.where(df > 0)  # create nans
        bools = df > 0
        mask = isna(df)
        expected = bools.astype(float).mask(mask)
        result = bools.mask(mask)
        assert_frame_equal(result, expected) 

def test_getitem_setitem_ix_negative_integers(self):
        with catch_warnings(record=True):
            result = self.frame.ix[:, -1]
        assert_series_equal(result, self.frame['D'])

        with catch_warnings(record=True):
            result = self.frame.ix[:, [-1]]
        assert_frame_equal(result, self.frame[['D']])

        with catch_warnings(record=True):
            result = self.frame.ix[:, [-1, -2]]
        assert_frame_equal(result, self.frame[['D', 'C']])

        with catch_warnings(record=True):
            self.frame.ix[:, [-1]] = 0
        assert (self.frame['D'] == 0).all()

        df = DataFrame(np.random.randn(8, 4))
        with catch_warnings(record=True):
            assert isna(df.ix[:, [-1]].values).all()

        # #1942
        a = DataFrame(randn(20, 2), index=[chr(x + 65) for x in range(20)])
        with catch_warnings(record=True):
            a.ix[-1] = a.ix[-2]

        with catch_warnings(record=True):
            assert_series_equal(a.ix[-1], a.ix[-2], check_names=False)
            assert a.ix[-1].name == 'T'
            assert a.ix[-2].name == 'S' 

def test_setitem_empty(self):
        # GH 9596
        df = pd.DataFrame({'a': ['1', '2', '3'],
                           'b': ['11', '22', '33'],
                           'c': ['111', '222', '333']})

        result = df.copy()
        result.loc[result.b.isna(), 'a'] = result.a
        assert_frame_equal(result, df) 

def test_getitem_fancy_slice_integers_step(self):
        df = DataFrame(np.random.randn(10, 5))

        # this is OK
        result = df.iloc[:8:2]  # noqa
        df.iloc[:8:2] = np.nan
        assert isna(df.iloc[:8:2]).values.all() 

def test_setitem_fancy_mixed_2d(self):

        with catch_warnings(record=True):
            self.mixed_frame.ix[:5, ['C', 'B', 'A']] = 5
            result = self.mixed_frame.ix[:5, ['C', 'B', 'A']]
            assert (result.values == 5).all()

            self.mixed_frame.ix[5] = np.nan
            assert isna(self.mixed_frame.ix[5]).all()

            self.mixed_frame.ix[5] = self.mixed_frame.ix[6]
            assert_series_equal(self.mixed_frame.ix[5], self.mixed_frame.ix[6],
                                check_names=False)

        # #1432
        with catch_warnings(record=True):
            df = DataFrame({1: [1., 2., 3.],
                            2: [3, 4, 5]})
            assert df._is_mixed_type

            df.ix[1] = [5, 10]

            expected = DataFrame({1: [1., 5., 3.],
                                  2: [3, 10, 5]})

            assert_frame_equal(df, expected) 

def test_set_value_resize(self):

        with tm.assert_produces_warning(FutureWarning,
                                        check_stacklevel=False):
            res = self.frame.set_value('foobar', 'B', 0)
        assert res is self.frame
        assert res.index[-1] == 'foobar'
        with tm.assert_produces_warning(FutureWarning,
                                        check_stacklevel=False):
            assert res.get_value('foobar', 'B') == 0

        self.frame.loc['foobar', 'qux'] = 0
        with tm.assert_produces_warning(FutureWarning,
                                        check_stacklevel=False):
            assert self.frame.get_value('foobar', 'qux') == 0

        res = self.frame.copy()
        with tm.assert_produces_warning(FutureWarning,
                                        check_stacklevel=False):
            res3 = res.set_value('foobar', 'baz', 'sam')
        assert res3['baz'].dtype == np.object_

        res = self.frame.copy()
        with tm.assert_produces_warning(FutureWarning,
                                        check_stacklevel=False):
            res3 = res.set_value('foobar', 'baz', True)
        assert res3['baz'].dtype == np.object_

        res = self.frame.copy()
        with tm.assert_produces_warning(FutureWarning,
                                        check_stacklevel=False):
            res3 = res.set_value('foobar', 'baz', 5)
        assert is_float_dtype(res3['baz'])
        assert isna(res3['baz'].drop(['foobar'])).all()
        with tm.assert_produces_warning(FutureWarning,
                                        check_stacklevel=False):
            pytest.raises(ValueError, res3.set_value, 'foobar', 'baz', 'sam') 

def test_reindex_frame_add_nat(self):
        rng = date_range('1/1/2000 00:00:00', periods=10, freq='10s')
        df = DataFrame({'A': np.random.randn(len(rng)), 'B': rng})

        result = df.reindex(lrange(15))
        assert np.issubdtype(result['B'].dtype, np.dtype('M8[ns]'))

        mask = com.isna(result)['B']
        assert mask[-5:].all()
        assert not mask[:-5].any() 

def test_reindex_frame_add_nat(self):
        rng = date_range('1/1/2000 00:00:00', periods=10, freq='10s')
        df = DataFrame({'A': np.random.randn(len(rng)), 'B': rng})

        result = df.reindex(lrange(15))
        assert np.issubdtype(result['B'].dtype, np.dtype('M8[ns]'))

        mask = com.isna(result)['B']
        assert mask[-5:].all()
        assert not mask[:-5].any() 

def test_getitem_boolean_casting(self):

        # don't upcast if we don't need to
        df = self.tsframe.copy()
        df['E'] = 1
        df['E'] = df['E'].astype('int32')
        df['E1'] = df['E'].copy()
        df['F'] = 1
        df['F'] = df['F'].astype('int64')
        df['F1'] = df['F'].copy()

        casted = df[df > 0]
        result = casted.get_dtype_counts()
        expected = Series({'float64': 4, 'int32': 2, 'int64': 2})
        assert_series_equal(result, expected)

        # int block splitting
        df.loc[df.index[1:3], ['E1', 'F1']] = 0
        casted = df[df > 0]
        result = casted.get_dtype_counts()
        expected = Series({'float64': 6, 'int32': 1, 'int64': 1})
        assert_series_equal(result, expected)

        # where dtype conversions
        # GH 3733
        df = DataFrame(data=np.random.randn(100, 50))
        df = df.where(df > 0)  # create nans
        bools = df > 0
        mask = isna(df)
        expected = bools.astype(float).mask(mask)
        result = bools.mask(mask)
        assert_frame_equal(result, expected) 

def test_setitem_empty(self):
        # GH 9596
        df = pd.DataFrame({'a': ['1', '2', '3'],
                           'b': ['11', '22', '33'],
                           'c': ['111', '222', '333']})

        result = df.copy()
        result.loc[result.b.isna(), 'a'] = result.a
        assert_frame_equal(result, df) 

def test_getitem_fancy_slice_integers_step(self):
        df = DataFrame(np.random.randn(10, 5))

        # this is OK
        result = df.iloc[:8:2]  # noqa
        df.iloc[:8:2] = np.nan
        assert isna(df.iloc[:8:2]).values.all() 

def test_setitem_fancy_mixed_2d(self):

        with catch_warnings(record=True):
            self.mixed_frame.ix[:5, ['C', 'B', 'A']] = 5
            result = self.mixed_frame.ix[:5, ['C', 'B', 'A']]
            assert (result.values == 5).all()

            self.mixed_frame.ix[5] = np.nan
            assert isna(self.mixed_frame.ix[5]).all()

            self.mixed_frame.ix[5] = self.mixed_frame.ix[6]
            assert_series_equal(self.mixed_frame.ix[5], self.mixed_frame.ix[6],
                                check_names=False)

        # #1432
        with catch_warnings(record=True):
            df = DataFrame({1: [1., 2., 3.],
                            2: [3, 4, 5]})
            assert df._is_mixed_type

            df.ix[1] = [5, 10]

            expected = DataFrame({1: [1., 5., 3.],
                                  2: [3, 10, 5]})

            assert_frame_equal(df, expected)