Python pandas.compat.PY3 Examples

def test_timestamp(self):
        # GH#17329
        # tz-naive --> treat it as if it were UTC for purposes of timestamp()
        ts = Timestamp.now()
        uts = ts.replace(tzinfo=utc)
        assert ts.timestamp() == uts.timestamp()

        tsc = Timestamp('2014-10-11 11:00:01.12345678', tz='US/Central')
        utsc = tsc.tz_convert('UTC')

        # utsc is a different representation of the same time
        assert tsc.timestamp() == utsc.timestamp()

        if PY3:
            # datetime.timestamp() converts in the local timezone
            with tm.set_timezone('UTC'):
                # should agree with datetime.timestamp method
                dt = ts.to_pydatetime()
                assert dt.timestamp() == ts.timestamp() 

def test_non_ascii_key(self):
        if compat.PY3:
            testjson = (
                b'[{"\xc3\x9cnic\xc3\xb8de":0,"sub":{"A":1, "B":2}},' +
                b'{"\xc3\x9cnic\xc3\xb8de":1,"sub":{"A":3, "B":4}}]'
            ).decode('utf8')
        else:
            testjson = ('[{"\xc3\x9cnic\xc3\xb8de":0,"sub":{"A":1, "B":2}},'
                        '{"\xc3\x9cnic\xc3\xb8de":1,"sub":{"A":3, "B":4}}]')

        testdata = {
            u'sub.A': [1, 3],
            u'sub.B': [2, 4],
            b"\xc3\x9cnic\xc3\xb8de".decode('utf8'): [0, 1]
        }
        expected = DataFrame(testdata)

        result = json_normalize(json.loads(testjson))
        tm.assert_frame_equal(result, expected) 

def test_numeric_compat(self):

        idx = self.create_index()
        tm.assert_raises_regex(TypeError, "cannot perform __mul__",
                               lambda: idx * 1)
        tm.assert_raises_regex(TypeError, "cannot perform __rmul__",
                               lambda: 1 * idx)

        div_err = "cannot perform __truediv__" if PY3 \
            else "cannot perform __div__"
        tm.assert_raises_regex(TypeError, div_err, lambda: idx / 1)
        div_err = div_err.replace(' __', ' __r')
        tm.assert_raises_regex(TypeError, div_err, lambda: 1 / idx)
        tm.assert_raises_regex(TypeError, "cannot perform __floordiv__",
                               lambda: idx // 1)
        tm.assert_raises_regex(TypeError, "cannot perform __rfloordiv__",
                               lambda: 1 // idx) 

def test_repr_roundtrip(self):

        ci = CategoricalIndex(['a', 'b'], categories=['a', 'b'], ordered=True)
        str(ci)
        tm.assert_index_equal(eval(repr(ci)), ci, exact=True)

        # formatting
        if PY3:
            str(ci)
        else:
            compat.text_type(ci)

        # long format
        # this is not reprable
        ci = CategoricalIndex(np.random.randint(0, 5, size=100))
        if PY3:
            str(ci)
        else:
            compat.text_type(ci) 

def test_legacy_pickle(self, datapath):
        if PY3:
            pytest.skip("testing for legacy pickles not "
                        "support on py3")

        path = datapath('indexes', 'data', 'multiindex_v1.pickle')
        obj = pd.read_pickle(path)

        obj2 = MultiIndex.from_tuples(obj.values)
        assert obj.equals(obj2)

        res = obj.get_indexer(obj)
        exp = np.arange(len(obj), dtype=np.intp)
        assert_almost_equal(res, exp)

        res = obj.get_indexer(obj2[::-1])
        exp = obj.get_indexer(obj[::-1])
        exp2 = obj2.get_indexer(obj2[::-1])
        assert_almost_equal(res, exp)
        assert_almost_equal(exp, exp2) 

def test_map_with_string_constructor(self):
        raw = [2005, 2007, 2009]
        index = PeriodIndex(raw, freq='A')
        types = str,

        if PY3:
            # unicode
            types += text_type,

        for t in types:
            expected = Index(lmap(t, raw))
            res = index.map(t)

            # should return an Index
            assert isinstance(res, Index)

            # preserve element types
            assert all(isinstance(resi, t) for resi in res)

            # lastly, values should compare equal
            tm.assert_index_equal(res, expected) 

def _add_numeric_methods_binary(cls):
        """
        Add in numeric methods.
        """
        cls.__add__ = _make_arithmetic_op(operator.add, cls)
        cls.__radd__ = _make_arithmetic_op(ops.radd, cls)
        cls.__sub__ = _make_arithmetic_op(operator.sub, cls)
        cls.__rsub__ = _make_arithmetic_op(ops.rsub, cls)
        cls.__rpow__ = _make_arithmetic_op(ops.rpow, cls)
        cls.__pow__ = _make_arithmetic_op(operator.pow, cls)

        cls.__truediv__ = _make_arithmetic_op(operator.truediv, cls)
        cls.__rtruediv__ = _make_arithmetic_op(ops.rtruediv, cls)
        if not compat.PY3:
            cls.__div__ = _make_arithmetic_op(operator.div, cls)
            cls.__rdiv__ = _make_arithmetic_op(ops.rdiv, cls)

        # TODO: rmod? rdivmod?
        cls.__mod__ = _make_arithmetic_op(operator.mod, cls)
        cls.__floordiv__ = _make_arithmetic_op(operator.floordiv, cls)
        cls.__rfloordiv__ = _make_arithmetic_op(ops.rfloordiv, cls)
        cls.__divmod__ = _make_arithmetic_op(divmod, cls)
        cls.__mul__ = _make_arithmetic_op(operator.mul, cls)
        cls.__rmul__ = _make_arithmetic_op(ops.rmul, cls) 

def test_map_with_string_constructor(self):
        raw = [2005, 2007, 2009]
        index = PeriodIndex(raw, freq='A')
        types = str,

        if PY3:
            # unicode
            types += text_type,

        for t in types:
            expected = Index(lmap(t, raw))
            res = index.map(t)

            # should return an Index
            assert isinstance(res, Index)

            # preserve element types
            assert all(isinstance(resi, t) for resi in res)

            # lastly, values should compare equal
            tm.assert_index_equal(res, expected) 

def test_repr_roundtrip(self):

        ci = CategoricalIndex(['a', 'b'], categories=['a', 'b'], ordered=True)
        str(ci)
        tm.assert_index_equal(eval(repr(ci)), ci, exact=True)

        # formatting
        if PY3:
            str(ci)
        else:
            compat.text_type(ci)

        # long format
        # this is not reprable
        ci = CategoricalIndex(np.random.randint(0, 5, size=100))
        if PY3:
            str(ci)
        else:
            compat.text_type(ci) 

def test_repr(self):
        i = RangeIndex(5, name='Foo')
        result = repr(i)
        if PY3:
            expected = "RangeIndex(start=0, stop=5, step=1, name='Foo')"
        else:
            expected = "RangeIndex(start=0, stop=5, step=1, name=u'Foo')"
        assert result == expected

        result = eval(result)
        tm.assert_index_equal(result, i, exact=True)

        i = RangeIndex(5, 0, -1)
        result = repr(i)
        expected = "RangeIndex(start=5, stop=0, step=-1)"
        assert result == expected

        result = eval(result)
        tm.assert_index_equal(result, i, exact=True) 

def test_msgpacks_legacy(self, current_packers_data, all_packers_data,
                             legacy_packer, datapath):

        version = os.path.basename(os.path.dirname(legacy_packer))

        # GH12142 0.17 files packed in P2 can't be read in P3
        if (compat.PY3 and version.startswith('0.17.') and
                legacy_packer.split('.')[-4][-1] == '2'):
            msg = "Files packed in Py2 can't be read in Py3 ({})"
            pytest.skip(msg.format(version))
        try:
            with catch_warnings(record=True):
                self.compare(current_packers_data, all_packers_data,
                             legacy_packer, version)
        except ImportError:
            # blosc not installed
            pass 

def _add_arithmetic_ops(cls):
        cls.__add__ = cls._create_arithmetic_method(operator.add)
        cls.__radd__ = cls._create_arithmetic_method(ops.radd)
        cls.__sub__ = cls._create_arithmetic_method(operator.sub)
        cls.__rsub__ = cls._create_arithmetic_method(ops.rsub)
        cls.__mul__ = cls._create_arithmetic_method(operator.mul)
        cls.__rmul__ = cls._create_arithmetic_method(ops.rmul)
        cls.__pow__ = cls._create_arithmetic_method(operator.pow)
        cls.__rpow__ = cls._create_arithmetic_method(ops.rpow)
        cls.__mod__ = cls._create_arithmetic_method(operator.mod)
        cls.__rmod__ = cls._create_arithmetic_method(ops.rmod)
        cls.__floordiv__ = cls._create_arithmetic_method(operator.floordiv)
        cls.__rfloordiv__ = cls._create_arithmetic_method(ops.rfloordiv)
        cls.__truediv__ = cls._create_arithmetic_method(operator.truediv)
        cls.__rtruediv__ = cls._create_arithmetic_method(ops.rtruediv)
        if not PY3:
            cls.__div__ = cls._create_arithmetic_method(operator.div)
            cls.__rdiv__ = cls._create_arithmetic_method(ops.rdiv)

        cls.__divmod__ = cls._create_arithmetic_method(divmod)
        cls.__rdivmod__ = cls._create_arithmetic_method(ops.rdivmod) 

def test_columns_dtypes_invalid(self, engine):
        df = pd.DataFrame({'string': list('abc'),
                           'int': list(range(1, 4))})

        # numeric
        df.columns = [0, 1]
        self.check_error_on_write(df, engine, ValueError)

        if PY3:
            # bytes on PY3, on PY2 these are str
            df.columns = [b'foo', b'bar']
            self.check_error_on_write(df, engine, ValueError)

        # python object
        df.columns = [datetime.datetime(2011, 1, 1, 0, 0),
                      datetime.datetime(2011, 1, 1, 1, 1)]
        self.check_error_on_write(df, engine, ValueError) 

def _is_unorderable_exception(e):
    """
    Check if the exception raised is an unorderable exception.

    The error message differs for 3 <= PY <= 3.5 and PY >= 3.6, so
    we need to condition based on Python version.

    Parameters
    ----------
    e : Exception or sub-class
        The exception object to check.

    Returns
    -------
    boolean : Whether or not the exception raised is an unorderable exception.
    """

    if PY36:
        return "'>' not supported between instances of" in str(e)

    elif PY3:
        return 'unorderable' in str(e)
    return False 

def test_bad_stream_exception(all_parsers, csv_dir_path):
    # see gh-13652
    #
    # This test validates that both the Python engine and C engine will
    # raise UnicodeDecodeError instead of C engine raising ParserError
    # and swallowing the exception that caused read to fail.
    path = os.path.join(csv_dir_path, "sauron.SHIFT_JIS.csv")
    codec = codecs.lookup("utf-8")
    utf8 = codecs.lookup('utf-8')
    parser = all_parsers

    msg = ("'utf-8' codec can't decode byte" if compat.PY3
           else "'utf8' codec can't decode byte")

    # Stream must be binary UTF8.
    with open(path, "rb") as handle, codecs.StreamRecoder(
            handle, utf8.encode, utf8.decode, codec.streamreader,
            codec.streamwriter) as stream:

        with pytest.raises(UnicodeDecodeError, match=msg):
            parser.read_csv(stream) 

def test_fwf_compression(compression_only, infer):
    data = """1111111111
    2222222222
    3333333333""".strip()

    compression = compression_only
    extension = "gz" if compression == "gzip" else compression

    kwargs = dict(widths=[5, 5], names=["one", "two"])
    expected = read_fwf(StringIO(data), **kwargs)

    if compat.PY3:
        data = bytes(data, encoding="utf-8")

    with tm.ensure_clean(filename="tmp." + extension) as path:
        tm.write_to_compressed(compression, path, data)

        if infer is not None:
            kwargs["compression"] = "infer" if infer else compression

        result = read_fwf(path, **kwargs)
        tm.assert_frame_equal(result, expected) 

def test_variable_width_unicode():
    if not compat.PY3:
        pytest.skip("Bytes-related test - only needs to work on Python 3")

    data = """
שלום שלום
ום   שלל
של   ום
""".strip("\r\n")
    encoding = "utf8"
    kwargs = dict(header=None, encoding=encoding)

    expected = read_fwf(BytesIO(data.encode(encoding)),
                        colspecs=[(0, 4), (5, 9)], **kwargs)
    result = read_fwf(BytesIO(data.encode(encoding)), **kwargs)
    tm.assert_frame_equal(result, expected) 

def check_whitelist(obj, df, m):
    # check the obj for a particular whitelist m

    gb = obj.groupby(df.letters)

    f = getattr(type(gb), m)

    # name
    try:
        n = f.__name__
    except AttributeError:
        return
    assert n == m

    # qualname
    if compat.PY3:
        try:
            n = f.__qualname__
        except AttributeError:
            return
        assert n.endswith(m) 

def test_inplace_ops_identity2(self, op):

        if compat.PY3 and op == 'div':
            return

        df = DataFrame({'a': [1., 2., 3.],
                        'b': [1, 2, 3]})

        operand = 2
        if op in ('and', 'or', 'xor'):
            # cannot use floats for boolean ops
            df['a'] = [True, False, True]

        df_copy = df.copy()
        iop = '__i{}__'.format(op)
        op = '__{}__'.format(op)

        # no id change and value is correct
        getattr(df, iop)(operand)
        expected = getattr(df_copy, op)(operand)
        assert_frame_equal(df, expected)
        expected = id(df)
        assert id(df) == expected 

def _check_file_or_buffer(self, f, engine):
        # see gh-16530
        if is_file_like(f):
            next_attr = "__next__" if PY3 else "next"

            # The C engine doesn't need the file-like to have the "next" or
            # "__next__" attribute. However, the Python engine explicitly calls
            # "next(...)" when iterating through such an object, meaning it
            # needs to have that attribute ("next" for Python 2.x, "__next__"
            # for Python 3.x)
            if engine != "c" and not hasattr(f, next_attr):
                msg = ("The 'python' engine cannot iterate "
                       "through this file buffer.")
                raise ValueError(msg)

        return engine 

def test_repr(self):
        i = RangeIndex(5, name='Foo')
        result = repr(i)
        if PY3:
            expected = "RangeIndex(start=0, stop=5, step=1, name='Foo')"
        else:
            expected = "RangeIndex(start=0, stop=5, step=1, name=u'Foo')"
        assert result == expected

        result = eval(result)
        tm.assert_index_equal(result, i, exact=True)

        i = RangeIndex(5, 0, -1)
        result = repr(i)
        expected = "RangeIndex(start=5, stop=0, step=-1)"
        assert result == expected

        result = eval(result)
        tm.assert_index_equal(result, i, exact=True) 

def test_to_latex_filename(self, frame):
        with tm.ensure_clean('test.tex') as path:
            frame.to_latex(path)

            with open(path, 'r') as f:
                assert frame.to_latex() == f.read()

        # test with utf-8 and encoding option (GH 7061)
        df = DataFrame([[u'au\xdfgangen']])
        with tm.ensure_clean('test.tex') as path:
            df.to_latex(path, encoding='utf-8')
            with codecs.open(path, 'r', encoding='utf-8') as f:
                assert df.to_latex() == f.read()

        # test with utf-8 without encoding option
        if compat.PY3:  # python3: pandas default encoding is utf-8
            with tm.ensure_clean('test.tex') as path:
                df.to_latex(path)
                with codecs.open(path, 'r', encoding='utf-8') as f:
                    assert df.to_latex() == f.read()
        else:
            # python2 default encoding is ascii, so an error should be raised
            with tm.ensure_clean('test.tex') as path:
                with pytest.raises(UnicodeEncodeError):
                    df.to_latex(path) 

def test_strl_latin1(self):
        # GH 23573, correct GSO data to reflect correct size
        output = DataFrame([[u'pandas'] * 2, [u'þâÑÐŧ'] * 2],
                           columns=['var_str', 'var_strl'])

        with tm.ensure_clean() as path:
            output.to_stata(path, version=117, convert_strl=['var_strl'])
            with open(path, 'rb') as reread:
                content = reread.read()
                expected = u'þâÑÐŧ'
                assert expected.encode('latin-1') in content
                assert expected.encode('utf-8') in content
                gsos = content.split(b'strls')[1][1:-2]
                for gso in gsos.split(b'GSO')[1:]:
                    val = gso.split(b'\x00')[-2]
                    size = gso[gso.find(b'\x82') + 1]
                    if not PY3:
                        size = ord(size)
                    assert len(val) == size - 1 

def get_terminal_size():
    """
    Detect terminal size and return tuple = (width, height).

    Only to be used when running in a terminal. Note that the IPython notebook,
    IPython zmq frontends, or IDLE do not run in a terminal,
    """
    import platform

    if PY3:
        return shutil.get_terminal_size()

    current_os = platform.system()
    tuple_xy = None
    if current_os == 'Windows':
        tuple_xy = _get_terminal_size_windows()
        if tuple_xy is None:
            tuple_xy = _get_terminal_size_tput()
            # needed for window's python in cygwin's xterm!
    if (current_os == 'Linux' or current_os == 'Darwin' or
            current_os.startswith('CYGWIN')):
        tuple_xy = _get_terminal_size_linux()
    if tuple_xy is None:
        tuple_xy = (80, 25)      # default value
    return tuple_xy 

def test_arith_flex_panel(self):
        with catch_warnings(record=True):
            ops = ['add', 'sub', 'mul', 'div',
                   'truediv', 'pow', 'floordiv', 'mod']
            if not compat.PY3:
                aliases = {}
            else:
                aliases = {'div': 'truediv'}
            self.panel = self.panel.to_panel()

            for n in [np.random.randint(-50, -1), np.random.randint(1, 50), 0]:
                for op in ops:
                    alias = aliases.get(op, op)
                    f = getattr(operator, alias)
                    exp = f(self.panel, n)
                    result = getattr(self.panel, op)(n)
                    assert_panel_equal(result, exp, check_panel_type=True)

                    # rops
                    r_f = lambda x, y: f(y, x)
                    exp = r_f(self.panel, n)
                    result = getattr(self.panel, 'r' + op)(n)
                    assert_panel_equal(result, exp) 

def render(self):
        self._write_table()

        if self.should_show_dimensions:
            by = chr(215) if compat.PY3 else unichr(215)  # ×
            self.write(u('<p>{rows} rows {by} {cols} columns</p>')
                       .format(rows=len(self.frame),
                               by=by,
                               cols=len(self.frame.columns)))

        return self.elements 

def str_translate(arr, table, deletechars=None):
    """
    Map all characters in the string through the given mapping table.
    Equivalent to standard :meth:`str.translate`. Note that the optional
    argument deletechars is only valid if you are using python 2. For python 3,
    character deletion should be specified via the table argument.

    Parameters
    ----------
    table : dict (python 3), str or None (python 2)
        In python 3, table is a mapping of Unicode ordinals to Unicode
        ordinals, strings, or None. Unmapped characters are left untouched.
        Characters mapped to None are deleted. :meth:`str.maketrans` is a
        helper function for making translation tables.
        In python 2, table is either a string of length 256 or None. If the
        table argument is None, no translation is applied and the operation
        simply removes the characters in deletechars. :func:`string.maketrans`
        is a helper function for making translation tables.
    deletechars : str, optional (python 2)
        A string of characters to delete. This argument is only valid
        in python 2.

    Returns
    -------
    translated : Series/Index of objects
    """
    if deletechars is None:
        f = lambda x: x.translate(table)
    else:
        if compat.PY3:
            raise ValueError("deletechars is not a valid argument for "
                             "str.translate in python 3. You should simply "
                             "specify character deletions in the table "
                             "argument")
        f = lambda x: x.translate(table, deletechars)
    return _na_map(f, arr) 

def __str__(self):
        """
        Return a string representation for a particular Object

        Invoked by str(df) in both py2/py3.
        Yields Bytestring in Py2, Unicode String in py3.
        """

        if compat.PY3:
            return self.__unicode__()
        return self.__bytes__() 

def _concat_date_cols(date_cols):
    if len(date_cols) == 1:
        if compat.PY3:
            return np.array([compat.text_type(x) for x in date_cols[0]],
                            dtype=object)
        else:
            return np.array([
                str(x) if not isinstance(x, compat.string_types) else x
                for x in date_cols[0]
            ], dtype=object)

    rs = np.array([' '.join(compat.text_type(y) for y in x)
                   for x in zip(*date_cols)], dtype=object)
    return rs 

def __init__(self, lhs, rhs, truediv, *args, **kwargs):
        super(Div, self).__init__('/', lhs, rhs, *args, **kwargs)

        if not isnumeric(lhs.return_type) or not isnumeric(rhs.return_type):
            raise TypeError("unsupported operand type(s) for {0}:"
                            " '{1}' and '{2}'".format(self.op,
                                                      lhs.return_type,
                                                      rhs.return_type))

        if truediv or PY3:
            # do not upcast float32s to float64 un-necessarily
            acceptable_dtypes = [np.float32, np.float_]
            _cast_inplace(com.flatten(self), acceptable_dtypes, np.float_)