Python numpy.compat.unicode() Examples

The following are code examples for showing how to use numpy.compat.unicode(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: LaserTOF   Author: kyleuckert   File: test_unicode.py    MIT License 6 votes vote down vote up
def content_check(self, ua, ua_scalar, nbytes):

        # Check the length of the unicode base type
        self.assertTrue(int(ua.dtype.str[2:]) == self.ulen)
        # Check the length of the data buffer
        self.assertTrue(buffer_length(ua) == nbytes)
        # Small check that data in array element is ok
        self.assertTrue(ua_scalar == self.ucs_value*self.ulen)
        # Encode to UTF-8 and double check
        self.assertTrue(ua_scalar.encode('utf-8') ==
                        (self.ucs_value*self.ulen).encode('utf-8'))
        # Check buffer lengths for scalars
        if ucs4:
            self.assertTrue(buffer_length(ua_scalar) == 4*self.ulen)
        else:
            if self.ucs_value == ucs4_value:
                # In UCS2, the \U0010FFFF will be represented using a
                # surrogate *pair*
                self.assertTrue(buffer_length(ua_scalar) == 2*2*self.ulen)
            else:
                # In UCS2, the \uFFFF will be represented using a
                # regular 2-byte word
                self.assertTrue(buffer_length(ua_scalar) == 2*self.ulen) 
Example 2
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: _internal.py    MIT License 6 votes vote down vote up
def _newnames(datatype, order):
    """
    Given a datatype and an order object, return a new names tuple, with the
    order indicated
    """
    oldnames = datatype.names
    nameslist = list(oldnames)
    if isinstance(order, (str, unicode)):
        order = [order]
    seen = set()
    if isinstance(order, (list, tuple)):
        for name in order:
            try:
                nameslist.remove(name)
            except ValueError:
                if name in seen:
                    raise ValueError("duplicate field name: %s" % (name,))
                else:
                    raise ValueError("unknown field name: %s" % (name,))
            seen.add(name)
        return tuple(list(order) + nameslist)
    raise ValueError("unsupported order value: %s" % (order,)) 
Example 3
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_unicode.py    MIT License 6 votes vote down vote up
def buffer_length(arr):
        if isinstance(arr, unicode):
            arr = str(arr)
            if not arr:
                charmax = 0
            else:
                charmax = max([ord(c) for c in arr])
            if charmax < 256:
                size = 1
            elif charmax < 65536:
                size = 2
            else:
                size = 4
            return size * len(arr)
        v = memoryview(arr)
        if v.shape is None:
            return len(v) * v.itemsize
        else:
            return np.prod(v.shape) * v.itemsize 
Example 4
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_unicode.py    MIT License 6 votes vote down vote up
def content_check(self, ua, ua_scalar, nbytes):

        # Check the length of the unicode base type
        assert_(int(ua.dtype.str[2:]) == self.ulen)
        # Check the length of the data buffer
        assert_(buffer_length(ua) == nbytes)
        # Small check that data in array element is ok
        assert_(ua_scalar == self.ucs_value*self.ulen)
        # Encode to UTF-8 and double check
        assert_(ua_scalar.encode('utf-8') ==
                        (self.ucs_value*self.ulen).encode('utf-8'))
        # Check buffer lengths for scalars
        if ucs4:
            assert_(buffer_length(ua_scalar) == 4*self.ulen)
        else:
            if self.ucs_value == ucs4_value:
                # In UCS2, the \U0010FFFF will be represented using a
                # surrogate *pair*
                assert_(buffer_length(ua_scalar) == 2*2*self.ulen)
            else:
                # In UCS2, the \uFFFF will be represented using a
                # regular 2-byte word
                assert_(buffer_length(ua_scalar) == 2*self.ulen) 
Example 5
Project: recruit   Author: Frank-qlu   File: _internal.py    Apache License 2.0 6 votes vote down vote up
def _newnames(datatype, order):
    """
    Given a datatype and an order object, return a new names tuple, with the
    order indicated
    """
    oldnames = datatype.names
    nameslist = list(oldnames)
    if isinstance(order, (str, unicode)):
        order = [order]
    seen = set()
    if isinstance(order, (list, tuple)):
        for name in order:
            try:
                nameslist.remove(name)
            except ValueError:
                if name in seen:
                    raise ValueError("duplicate field name: %s" % (name,))
                else:
                    raise ValueError("unknown field name: %s" % (name,))
            seen.add(name)
        return tuple(list(order) + nameslist)
    raise ValueError("unsupported order value: %s" % (order,)) 
Example 6
Project: recruit   Author: Frank-qlu   File: test_unicode.py    Apache License 2.0 6 votes vote down vote up
def buffer_length(arr):
        if isinstance(arr, unicode):
            arr = str(arr)
            if not arr:
                charmax = 0
            else:
                charmax = max([ord(c) for c in arr])
            if charmax < 256:
                size = 1
            elif charmax < 65536:
                size = 2
            else:
                size = 4
            return size * len(arr)
        v = memoryview(arr)
        if v.shape is None:
            return len(v) * v.itemsize
        else:
            return np.prod(v.shape) * v.itemsize 
Example 7
Project: recruit   Author: Frank-qlu   File: test_unicode.py    Apache License 2.0 6 votes vote down vote up
def content_check(self, ua, ua_scalar, nbytes):

        # Check the length of the unicode base type
        assert_(int(ua.dtype.str[2:]) == self.ulen)
        # Check the length of the data buffer
        assert_(buffer_length(ua) == nbytes)
        # Small check that data in array element is ok
        assert_(ua_scalar == self.ucs_value*self.ulen)
        # Encode to UTF-8 and double check
        assert_(ua_scalar.encode('utf-8') ==
                        (self.ucs_value*self.ulen).encode('utf-8'))
        # Check buffer lengths for scalars
        if ucs4:
            assert_(buffer_length(ua_scalar) == 4*self.ulen)
        else:
            if self.ucs_value == ucs4_value:
                # In UCS2, the \U0010FFFF will be represented using a
                # surrogate *pair*
                assert_(buffer_length(ua_scalar) == 2*2*self.ulen)
            else:
                # In UCS2, the \uFFFF will be represented using a
                # regular 2-byte word
                assert_(buffer_length(ua_scalar) == 2*self.ulen) 
Example 8
Project: recruit   Author: Frank-qlu   File: test_unicode.py    Apache License 2.0 6 votes vote down vote up
def content_check(self, ua, ua_scalar, nbytes):

        # Check the length of the unicode base type
        assert_(int(ua.dtype.str[2:]) == self.ulen)
        # Check the length of the data buffer
        assert_(buffer_length(ua) == nbytes)
        # Small check that data in array element is ok
        assert_(ua_scalar == self.ucs_value*self.ulen)
        # Encode to UTF-8 and double check
        assert_(ua_scalar.encode('utf-8') ==
                        (self.ucs_value*self.ulen).encode('utf-8'))
        # Check buffer lengths for scalars
        if ucs4:
            assert_(buffer_length(ua_scalar) == 4*self.ulen)
        else:
            if self.ucs_value == ucs4_value:
                # In UCS2, the \U0010FFFF will be represented using a
                # surrogate *pair*
                assert_(buffer_length(ua_scalar) == 2*2*self.ulen)
            else:
                # In UCS2, the \uFFFF will be represented using a
                # regular 2-byte word
                assert_(buffer_length(ua_scalar) == 2*self.ulen) 
Example 9
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_io.py    MIT License 6 votes vote down vote up
def test_unicode_roundtrip(self):
        utf8 = b'\xcf\x96'.decode('UTF-8')
        a = np.array([utf8], dtype=np.unicode)
        # our gz wrapper support encoding
        suffixes = ['', '.gz']
        # stdlib 2 versions do not support encoding
        if MAJVER > 2:
            if HAS_BZ2:
                suffixes.append('.bz2')
            if HAS_LZMA:
                suffixes.extend(['.xz', '.lzma'])
        with tempdir() as tmpdir:
            for suffix in suffixes:
                np.savetxt(os.path.join(tmpdir, 'test.csv' + suffix), a,
                           fmt=['%s'], encoding='UTF-16-LE')
                b = np.loadtxt(os.path.join(tmpdir, 'test.csv' + suffix),
                               encoding='UTF-16-LE', dtype=np.unicode)
                assert_array_equal(a, b) 
Example 10
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_io.py    MIT License 6 votes vote down vote up
def test_utf8_file(self):
        utf8 = b"\xcf\x96"
        latin1 = b"\xf6\xfc\xf6"
        with temppath() as path:
            with open(path, "wb") as f:
                f.write((b"test1,testNonethe" + utf8 + b",test3\n") * 2)
            test = np.genfromtxt(path, dtype=None, comments=None,
                                 delimiter=',', encoding="UTF-8")
            ctl = np.array([
                     ["test1", "testNonethe" + utf8.decode("UTF-8"), "test3"],
                     ["test1", "testNonethe" + utf8.decode("UTF-8"), "test3"]],
                     dtype=np.unicode)
            assert_array_equal(test, ctl)

            # test a mixed dtype
            with open(path, "wb") as f:
                f.write(b"0,testNonethe" + utf8)
            test = np.genfromtxt(path, dtype=None, comments=None,
                                 delimiter=',', encoding="UTF-8")
            assert_equal(test['f0'], 0)
            assert_equal(test['f1'], "testNonethe" + utf8.decode("UTF-8")) 
Example 11
Project: FUTU_Stop_Loss   Author: BigtoC   File: _internal.py    MIT License 6 votes vote down vote up
def _newnames(datatype, order):
    """
    Given a datatype and an order object, return a new names tuple, with the
    order indicated
    """
    oldnames = datatype.names
    nameslist = list(oldnames)
    if isinstance(order, (str, unicode)):
        order = [order]
    seen = set()
    if isinstance(order, (list, tuple)):
        for name in order:
            try:
                nameslist.remove(name)
            except ValueError:
                if name in seen:
                    raise ValueError("duplicate field name: %s" % (name,))
                else:
                    raise ValueError("unknown field name: %s" % (name,))
            seen.add(name)
        return tuple(list(order) + nameslist)
    raise ValueError("unsupported order value: %s" % (order,)) 
Example 12
Project: LaserTOF   Author: kyleuckert   File: test_io.py    MIT License 5 votes vote down vote up
def test_comments_unicode(self):
        c = TextIO()
        c.write('# comment\n1,2,3,5\n')
        c.seek(0)
        x = np.loadtxt(c, dtype=int, delimiter=',',
                       comments=unicode('#'))
        a = np.array([1, 2, 3, 5], int)
        assert_array_equal(x, a) 
Example 13
Project: LaserTOF   Author: kyleuckert   File: test_io.py    MIT License 5 votes vote down vote up
def test_gft_using_generator(self):
        # gft doesn't work with unicode.
        def count():
            for i in range(10):
                yield asbytes("%d" % i)

        res = np.genfromtxt(count())
        assert_array_equal(res, np.arange(10)) 
Example 14
Project: LaserTOF   Author: kyleuckert   File: test_regression.py    MIT License 5 votes vote down vote up
def test_include_dirs(self):
        # As a sanity check, just test that get_include
        # includes something reasonable.  Somewhat
        # related to ticket #1405.
        include_dirs = [np.get_include()]
        for path in include_dirs:
            assert_(isinstance(path, (str, unicode)))
            assert_(path != '') 
Example 15
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_searchsorted_unicode(self):
        # Test searchsorted on unicode strings.

        # 1.6.1 contained a string length miscalculation in
        # arraytypes.c.src:UNICODE_compare() which manifested as
        # incorrect/inconsistent results from searchsorted.
        a = np.array(['P:\\20x_dapi_cy3\\20x_dapi_cy3_20100185_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100186_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100187_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100189_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100190_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100191_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100192_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100193_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100194_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100195_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100196_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100197_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100198_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100199_1'],
                     dtype=np.unicode)
        ind = np.arange(len(a))
        assert_equal([a.searchsorted(v, 'left') for v in a], ind)
        assert_equal([a.searchsorted(v, 'right') for v in a], ind + 1)
        assert_equal([a.searchsorted(a[i], 'left') for i in ind], ind)
        assert_equal([a.searchsorted(a[i], 'right') for i in ind], ind + 1) 
Example 16
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_ip_types(self):
        unchecked_types = [bytes, unicode, np.void, object]

        x = np.random.random(1000)*100
        mask = x < 40

        for val in [-100, 0, 15]:
            for types in np.sctypes.values():
                for T in types:
                    if T not in unchecked_types:
                        yield self.tst_basic, x.copy().astype(T), T, mask, val 
Example 17
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_multiple_field_name_unicode(self):
            def test_assign_unicode():
                dt = np.dtype([("\u20B9", "f8"),
                               ("B", "f8"),
                               ("\u20B9", "f8")])

            # Error raised when multiple fields have the same name(unicode included)
            assert_raises(ValueError, test_assign_unicode) 
Example 18
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_unicode_field_titles(self):
            # Unicode field titles are added to field dict on Py2
            title = unicode('b')
            dt = np.dtype([((title, 'a'), int)])
            dt[title]
            dt['a']
            x = np.array([(1,), (2,), (3,)], dtype=dt)
            x[title]
            x['a']
            y = x[0]
            y[title]
            y['a'] 
Example 19
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_unicode_field_names(self):
            # Unicode field names are not allowed on Py2
            title = unicode('b')
            assert_raises(TypeError, np.dtype, [(title, int)])
            assert_raises(TypeError, np.dtype, [(('a', title), int)]) 
Example 20
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_empty_ustring_array_is_falsey(self):
        self.assertFalse(np.array([''], dtype=np.unicode)) 
Example 21
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_whitespace_ustring_array_is_falsey(self):
        a = np.array(['eggs'], dtype=np.unicode)
        a[0] = '  \0\0'
        self.assertFalse(a) 
Example 22
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_null_inside_ustring_array_is_truthy(self):
        a = np.array(['eggs'], dtype=np.unicode)
        a[0] = ' \0 \0'
        self.assertTrue(a) 
Example 23
Project: LaserTOF   Author: kyleuckert   File: test_unicode.py    MIT License 5 votes vote down vote up
def buffer_length(arr):
        if isinstance(arr, unicode):
            arr = str(arr)
            return (sys.getsizeof(arr+"a") - sys.getsizeof(arr)) * len(arr)
        v = memoryview(arr)
        if v.shape is None:
            return len(v) * v.itemsize
        else:
            return np.prod(v.shape) * v.itemsize 
Example 24
Project: LaserTOF   Author: kyleuckert   File: test_unicode.py    MIT License 5 votes vote down vote up
def buffer_length(arr):
        if isinstance(arr, unicode):
            return _array.array('u').itemsize * len(arr)
        v = memoryview(arr)
        if v.shape is None:
            return len(v) * v.itemsize
        else:
            return np.prod(v.shape) * v.itemsize 
Example 25
Project: LaserTOF   Author: kyleuckert   File: test_unicode.py    MIT License 5 votes vote down vote up
def buffer_length(arr):
        if isinstance(arr, np.ndarray):
            return len(arr.data)
        return len(buffer(arr))

# In both cases below we need to make sure that the byte swapped value (as
# UCS4) is still a valid unicode:
# Value that can be represented in UCS2 interpreters 
Example 26
Project: LaserTOF   Author: kyleuckert   File: test_unicode.py    MIT License 5 votes vote down vote up
def content_check(self, ua, ua_scalar, nbytes):

        # Check the length of the unicode base type
        self.assertTrue(int(ua.dtype.str[2:]) == self.ulen)
        # Check the length of the data buffer
        self.assertTrue(buffer_length(ua) == nbytes)
        # Small check that data in array element is ok
        self.assertTrue(ua_scalar == sixu(''))
        # Encode to ascii and double check
        self.assertTrue(ua_scalar.encode('ascii') == asbytes(''))
        # Check buffer lengths for scalars
        if ucs4:
            self.assertTrue(buffer_length(ua_scalar) == 0)
        else:
            self.assertTrue(buffer_length(ua_scalar) == 0) 
Example 27
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_upgrade(self):
        "Tests the upgrade method."

        converter = StringConverter()
        assert_equal(converter._status, 0)

        # test int
        assert_equal(converter.upgrade('0'), 0)
        assert_equal(converter._status, 1)

        # On systems where long defaults to 32-bit, the statuses will be
        # offset by one, so we check for this here.
        import numpy.core.numeric as nx
        status_offset = int(nx.dtype(nx.int_).itemsize < nx.dtype(nx.int64).itemsize)

        # test int > 2**32
        assert_equal(converter.upgrade('17179869184'), 17179869184)
        assert_equal(converter._status, 1 + status_offset)

        # test float
        assert_allclose(converter.upgrade('0.'), 0.0)
        assert_equal(converter._status, 2 + status_offset)

        # test complex
        assert_equal(converter.upgrade('0j'), complex('0j'))
        assert_equal(converter._status, 3 + status_offset)

        # test str
        # note that the longdouble type has been skipped, so the
        # _status increases by 2. Everything should succeed with
        # unicode conversion (5).
        for s in ['a', u'a', b'a']:
            res = converter.upgrade(s)
            assert_(type(res) is unicode)
            assert_equal(res, u'a')
            assert_equal(converter._status, 5 + status_offset) 
Example 28
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_regression.py    MIT License 5 votes vote down vote up
def test_include_dirs(self):
        # As a sanity check, just test that get_include
        # includes something reasonable.  Somewhat
        # related to ticket #1405.
        include_dirs = [np.get_include()]
        for path in include_dirs:
            assert_(isinstance(path, (str, unicode)))
            assert_(path != '') 
Example 29
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_create(self):
        zs = self._zeros(10, bytes)
        assert_equal(zs.itemsize, 0)
        zs = self._zeros(10, np.void)
        assert_equal(zs.itemsize, 0)
        zs = self._zeros(10, unicode)
        assert_equal(zs.itemsize, 0) 
Example 30
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_resize(self):
        # previously an error
        for dt in [bytes, np.void, unicode]:
            zs = self._zeros(10, dt)
            zs.resize(25)
            zs.resize((10, 10)) 
Example 31
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_view(self):
        for dt in [bytes, np.void, unicode]:
            zs = self._zeros(10, dt)

            # viewing as itself should be allowed
            assert_equal(zs.view(dt).dtype, np.dtype(dt))

            # viewing as any non-empty type gives an empty result
            assert_equal(zs.view((dt, 1)).shape, (0,)) 
Example 32
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_pickle(self):
        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
            for dt in [bytes, np.void, unicode]:
                zs = self._zeros(10, dt)
                p = pickle.dumps(zs, protocol=proto)
                zs2 = pickle.loads(p)

                assert_equal(zs.dtype, zs2.dtype) 
Example 33
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_searchsorted_unicode(self):
        # Test searchsorted on unicode strings.

        # 1.6.1 contained a string length miscalculation in
        # arraytypes.c.src:UNICODE_compare() which manifested as
        # incorrect/inconsistent results from searchsorted.
        a = np.array(['P:\\20x_dapi_cy3\\20x_dapi_cy3_20100185_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100186_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100187_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100189_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100190_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100191_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100192_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100193_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100194_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100195_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100196_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100197_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100198_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100199_1'],
                     dtype=np.unicode)
        ind = np.arange(len(a))
        assert_equal([a.searchsorted(v, 'left') for v in a], ind)
        assert_equal([a.searchsorted(v, 'right') for v in a], ind + 1)
        assert_equal([a.searchsorted(a[i], 'left') for i in ind], ind)
        assert_equal([a.searchsorted(a[i], 'right') for i in ind], ind + 1) 
Example 34
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_ip_types(self):
        unchecked_types = [bytes, unicode, np.void, object]

        x = np.random.random(1000)*100
        mask = x < 40

        for val in [-100, 0, 15]:
            for types in np.sctypes.values():
                for T in types:
                    if T not in unchecked_types:
                        self.tst_basic(x.copy().astype(T), T, mask, val) 
Example 35
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_multiple_field_name_unicode(self):
        def test_dtype_unicode():
            np.dtype([("\u20B9", "f8"), ("B", "f8"), ("\u20B9", "f8")])

        # Error raised when multiple fields have the same name(unicode included)
        assert_raises(ValueError, test_dtype_unicode) 
Example 36
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_fromarrays_unicode(self):
        # A single name string provided to fromarrays() is allowed to be unicode
        # on both Python 2 and 3:
        x = np.core.records.fromarrays([[0], [1]], names=u'a,b', formats=u'i4,i4')
        assert_equal(x['a'][0], 0)
        assert_equal(x['b'][0], 1) 
Example 37
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_unicode_order(self):
        # Test that we can sort with order as a unicode field name in both Python 2 and
        # 3:
        name = u'b'
        x = np.array([1, 3, 2], dtype=[(name, int)])
        x.sort(order=name)
        assert_equal(x[u'b'], np.array([1, 2, 3])) 
Example 38
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_empty_ustring_array_is_falsey(self):
        assert_(not np.array([''], dtype=np.unicode)) 
Example 39
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_all_null_ustring_array_is_falsey(self):
        a = np.array(['eggs'], dtype=np.unicode)
        a[0] = '\0\0\0\0'
        assert_(not a) 
Example 40
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_null_inside_ustring_array_is_truthy(self):
        a = np.array(['eggs'], dtype=np.unicode)
        a[0] = ' \0 \0'
        assert_(a) 
Example 41
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_unicode.py    MIT License 5 votes vote down vote up
def buffer_length(arr):
        if isinstance(arr, np.ndarray):
            return len(arr.data)
        return len(buffer(arr))

# In both cases below we need to make sure that the byte swapped value (as
# UCS4) is still a valid unicode:
# Value that can be represented in UCS2 interpreters 
Example 42
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_unicode.py    MIT License 5 votes vote down vote up
def content_check(self, ua, ua_scalar, nbytes):

        # Check the length of the unicode base type
        assert_(int(ua.dtype.str[2:]) == self.ulen)
        # Check the length of the data buffer
        assert_(buffer_length(ua) == nbytes)
        # Small check that data in array element is ok
        assert_(ua_scalar == u'')
        # Encode to ascii and double check
        assert_(ua_scalar.encode('ascii') == b'')
        # Check buffer lengths for scalars
        if ucs4:
            assert_(buffer_length(ua_scalar) == 0)
        else:
            assert_(buffer_length(ua_scalar) == 0) 
Example 43
Project: recruit   Author: Frank-qlu   File: test__iotools.py    Apache License 2.0 5 votes vote down vote up
def test_upgrade(self):
        "Tests the upgrade method."

        converter = StringConverter()
        assert_equal(converter._status, 0)

        # test int
        assert_equal(converter.upgrade('0'), 0)
        assert_equal(converter._status, 1)

        # On systems where long defaults to 32-bit, the statuses will be
        # offset by one, so we check for this here.
        import numpy.core.numeric as nx
        status_offset = int(nx.dtype(nx.int_).itemsize < nx.dtype(nx.int64).itemsize)

        # test int > 2**32
        assert_equal(converter.upgrade('17179869184'), 17179869184)
        assert_equal(converter._status, 1 + status_offset)

        # test float
        assert_allclose(converter.upgrade('0.'), 0.0)
        assert_equal(converter._status, 2 + status_offset)

        # test complex
        assert_equal(converter.upgrade('0j'), complex('0j'))
        assert_equal(converter._status, 3 + status_offset)

        # test str
        # note that the longdouble type has been skipped, so the
        # _status increases by 2. Everything should succeed with
        # unicode conversion (5).
        for s in ['a', u'a', b'a']:
            res = converter.upgrade(s)
            assert_(type(res) is unicode)
            assert_equal(res, u'a')
            assert_equal(converter._status, 5 + status_offset) 
Example 44
Project: recruit   Author: Frank-qlu   File: test_regression.py    Apache License 2.0 5 votes vote down vote up
def test_include_dirs(self):
        # As a sanity check, just test that get_include
        # includes something reasonable.  Somewhat
        # related to ticket #1405.
        include_dirs = [np.get_include()]
        for path in include_dirs:
            assert_(isinstance(path, (str, unicode)))
            assert_(path != '') 
Example 45
Project: recruit   Author: Frank-qlu   File: records.py    Apache License 2.0 5 votes vote down vote up
def _setfieldnames(self, names, titles):
        """convert input field names into a list and assign to the _names
        attribute """

        if (names):
            if (type(names) in [list, tuple]):
                pass
            elif isinstance(names, (str, unicode)):
                names = names.split(',')
            else:
                raise NameError("illegal input names %s" % repr(names))

            self._names = [n.strip() for n in names[:self._nfields]]
        else:
            self._names = []

        # if the names are not specified, they will be assigned as
        #  "f0, f1, f2,..."
        # if not enough names are specified, they will be assigned as "f[n],
        # f[n+1],..." etc. where n is the number of specified names..."
        self._names += ['f%d' % i for i in range(len(self._names),
                                                 self._nfields)]
        # check for redundant names
        _dup = find_duplicate(self._names)
        if _dup:
            raise ValueError("Duplicate field names: %s" % _dup)

        if (titles):
            self._titles = [n.strip() for n in titles[:self._nfields]]
        else:
            self._titles = []
            titles = []

        if (self._nfields > len(titles)):
            self._titles += [None] * (self._nfields - len(titles)) 
Example 46
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 5 votes vote down vote up
def _test_sort_partition(self, name, kinds, **kwargs):
        # Previously, these would all hang
        for dt in [bytes, np.void, unicode]:
            zs = self._zeros(10, dt)
            sort_method = getattr(zs, name)
            sort_func = getattr(np, name)
            for kind in kinds:
                sort_method(kind=kind, **kwargs)
                sort_func(zs, kind=kind, **kwargs) 
Example 47
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 5 votes vote down vote up
def test_resize(self):
        # previously an error
        for dt in [bytes, np.void, unicode]:
            zs = self._zeros(10, dt)
            zs.resize(25)
            zs.resize((10, 10)) 
Example 48
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 5 votes vote down vote up
def test_view(self):
        for dt in [bytes, np.void, unicode]:
            zs = self._zeros(10, dt)

            # viewing as itself should be allowed
            assert_equal(zs.view(dt).dtype, np.dtype(dt))

            # viewing as any non-empty type gives an empty result
            assert_equal(zs.view((dt, 1)).shape, (0,)) 
Example 49
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 5 votes vote down vote up
def test_pickle(self):
        for proto in range(2, pickle.HIGHEST_PROTOCOL + 1):
            for dt in [bytes, np.void, unicode]:
                zs = self._zeros(10, dt)
                p = pickle.dumps(zs, protocol=proto)
                zs2 = pickle.loads(p)

                assert_equal(zs.dtype, zs2.dtype) 
Example 50
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 5 votes vote down vote up
def test_searchsorted_unicode(self):
        # Test searchsorted on unicode strings.

        # 1.6.1 contained a string length miscalculation in
        # arraytypes.c.src:UNICODE_compare() which manifested as
        # incorrect/inconsistent results from searchsorted.
        a = np.array(['P:\\20x_dapi_cy3\\20x_dapi_cy3_20100185_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100186_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100187_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100189_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100190_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100191_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100192_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100193_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100194_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100195_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100196_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100197_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100198_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100199_1'],
                     dtype=np.unicode)
        ind = np.arange(len(a))
        assert_equal([a.searchsorted(v, 'left') for v in a], ind)
        assert_equal([a.searchsorted(v, 'right') for v in a], ind + 1)
        assert_equal([a.searchsorted(a[i], 'left') for i in ind], ind)
        assert_equal([a.searchsorted(a[i], 'right') for i in ind], ind + 1) 
Example 51
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 5 votes vote down vote up
def test_ip_types(self):
        unchecked_types = [bytes, unicode, np.void, object]

        x = np.random.random(24)*100
        x.shape = 2, 3, 4
        for types in np.sctypes.values():
            for T in types:
                if T not in unchecked_types:
                    self.tst_basic(x.copy().astype(T)) 
Example 52
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 5 votes vote down vote up
def test_multiple_field_name_unicode(self):
        def test_dtype_unicode():
            np.dtype([("\u20B9", "f8"), ("B", "f8"), ("\u20B9", "f8")])

        # Error raised when multiple fields have the same name(unicode included)
        assert_raises(ValueError, test_dtype_unicode) 
Example 53
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 5 votes vote down vote up
def test_fromarrays_unicode(self):
        # A single name string provided to fromarrays() is allowed to be unicode
        # on both Python 2 and 3:
        x = np.core.records.fromarrays([[0], [1]], names=u'a,b', formats=u'i4,i4')
        assert_equal(x['a'][0], 0)
        assert_equal(x['b'][0], 1) 
Example 54
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 5 votes vote down vote up
def test_unicode_order(self):
        # Test that we can sort with order as a unicode field name in both Python 2 and
        # 3:
        name = u'b'
        x = np.array([1, 3, 2], dtype=[(name, int)])
        x.sort(order=name)
        assert_equal(x[u'b'], np.array([1, 2, 3])) 
Example 55
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 5 votes vote down vote up
def test_whitespace_ustring_array_is_falsey(self):
        a = np.array(['eggs'], dtype=np.unicode)
        a[0] = '  \0\0'
        assert_(not a) 
Example 56
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 5 votes vote down vote up
def test_all_null_ustring_array_is_falsey(self):
        a = np.array(['eggs'], dtype=np.unicode)
        a[0] = '\0\0\0\0'
        assert_(not a) 
Example 57
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 5 votes vote down vote up
def test_null_inside_ustring_array_is_truthy(self):
        a = np.array(['eggs'], dtype=np.unicode)
        a[0] = ' \0 \0'
        assert_(a) 
Example 58
Project: recruit   Author: Frank-qlu   File: test_unicode.py    Apache License 2.0 5 votes vote down vote up
def buffer_length(arr):
        if isinstance(arr, np.ndarray):
            return len(arr.data)
        return len(buffer(arr))

# In both cases below we need to make sure that the byte swapped value (as
# UCS4) is still a valid unicode:
# Value that can be represented in UCS2 interpreters 
Example 59
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_io.py    MIT License 5 votes vote down vote up
def test_unicode(self):
        utf8 = b'\xcf\x96'.decode('UTF-8')
        a = np.array([utf8], dtype=np.unicode)
        with tempdir() as tmpdir:
            # set encoding as on windows it may not be unicode even on py3
            np.savetxt(os.path.join(tmpdir, 'test.csv'), a, fmt=['%s'],
                       encoding='UTF-8') 
Example 60
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_io.py    MIT License 5 votes vote down vote up
def test_unicode_bytestream(self):
        utf8 = b'\xcf\x96'.decode('UTF-8')
        a = np.array([utf8], dtype=np.unicode)
        s = BytesIO()
        np.savetxt(s, a, fmt=['%s'], encoding='UTF-8')
        s.seek(0)
        assert_equal(s.read().decode('UTF-8'), utf8 + '\n') 
Example 61
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_io.py    MIT License 5 votes vote down vote up
def test_unicode_stringstream(self):
        utf8 = b'\xcf\x96'.decode('UTF-8')
        a = np.array([utf8], dtype=np.unicode)
        s = StringIO()
        np.savetxt(s, a, fmt=['%s'], encoding='UTF-8')
        s.seek(0)
        assert_equal(s.read(), utf8 + '\n') 
Example 62
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_io.py    MIT License 5 votes vote down vote up
def test_stringload(self):
        # umlaute
        nonascii = b'\xc3\xb6\xc3\xbc\xc3\xb6'.decode("UTF-8")
        with temppath() as path:
            with open(path, "wb") as f:
                f.write(nonascii.encode("UTF-16"))
            x = self.loadfunc(path, encoding="UTF-16", dtype=np.unicode)
            assert_array_equal(x, nonascii) 
Example 63
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_io.py    MIT License 5 votes vote down vote up
def test_converters_decode(self):
        # test converters that decode strings
        c = TextIO()
        c.write(b'\xcf\x96')
        c.seek(0)
        x = self.loadfunc(c, dtype=np.unicode,
                          converters={0: lambda x: x.decode('UTF-8')})
        a = np.array([b'\xcf\x96'.decode('UTF-8')])
        assert_array_equal(x, a) 
Example 64
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_io.py    MIT License 5 votes vote down vote up
def test_converters_nodecode(self):
        # test native string converters enabled by setting an encoding
        utf8 = b'\xcf\x96'.decode('UTF-8')
        with temppath() as path:
            with io.open(path, 'wt', encoding='UTF-8') as f:
                f.write(utf8)
            x = self.loadfunc(path, dtype=np.unicode,
                              converters={0: lambda x: x + 't'},
                              encoding='UTF-8')
            a = np.array([utf8 + 't'])
            assert_array_equal(x, a) 
Example 65
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_io.py    MIT License 5 votes vote down vote up
def test_binary_load(self):
        butf8 = b"5,6,7,\xc3\x95scarscar\n\r15,2,3,hello\n\r"\
                b"20,2,3,\xc3\x95scar\n\r"
        sutf8 = butf8.decode("UTF-8").replace("\r", "").splitlines()
        with temppath() as path:
            with open(path, "wb") as f:
                f.write(butf8)
            with open(path, "rb") as f:
                x = np.loadtxt(f, encoding="UTF-8", dtype=np.unicode)
            assert_array_equal(x, sutf8)
            # test broken latin1 conversion people now rely on
            with open(path, "rb") as f:
                x = np.loadtxt(f, encoding="UTF-8", dtype="S")
            x = [b'5,6,7,\xc3\x95scarscar', b'15,2,3,hello', b'20,2,3,\xc3\x95scar']
            assert_array_equal(x, np.array(x, dtype="S")) 
Example 66
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_io.py    MIT License 5 votes vote down vote up
def test_utf8_userconverters_with_explicit_dtype(self):
        utf8 = b'\xcf\x96'
        with temppath() as path:
            with open(path, 'wb') as f:
                f.write(b'skip,skip,2001-01-01' + utf8 + b',1.0,skip')
            test = np.genfromtxt(path, delimiter=",", names=None, dtype=float,
                                 usecols=(2, 3), converters={2: np.unicode},
                                 encoding='UTF-8')
        control = np.array([('2001-01-01' + utf8.decode('UTF-8'), 1.)],
                           dtype=[('', '|U11'), ('', float)])
        assert_equal(test, control) 
Example 67
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_io.py    MIT License 5 votes vote down vote up
def test_gft_using_generator(self):
        # gft doesn't work with unicode.
        def count():
            for i in range(10):
                yield asbytes("%d" % i)

        res = np.genfromtxt(count())
        assert_array_equal(res, np.arange(10)) 
Example 68
Project: FUTU_Stop_Loss   Author: BigtoC   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_upgrade(self):
        "Tests the upgrade method."

        converter = StringConverter()
        assert_equal(converter._status, 0)

        # test int
        assert_equal(converter.upgrade('0'), 0)
        assert_equal(converter._status, 1)

        # On systems where long defaults to 32-bit, the statuses will be
        # offset by one, so we check for this here.
        import numpy.core.numeric as nx
        status_offset = int(nx.dtype(nx.int_).itemsize < nx.dtype(nx.int64).itemsize)

        # test int > 2**32
        assert_equal(converter.upgrade('17179869184'), 17179869184)
        assert_equal(converter._status, 1 + status_offset)

        # test float
        assert_allclose(converter.upgrade('0.'), 0.0)
        assert_equal(converter._status, 2 + status_offset)

        # test complex
        assert_equal(converter.upgrade('0j'), complex('0j'))
        assert_equal(converter._status, 3 + status_offset)

        # test str
        # note that the longdouble type has been skipped, so the
        # _status increases by 2. Everything should succeed with
        # unicode conversion (5).
        for s in ['a', u'a', b'a']:
            res = converter.upgrade(s)
            assert_(type(res) is unicode)
            assert_equal(res, u'a')
            assert_equal(converter._status, 5 + status_offset) 
Example 69
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_regression.py    MIT License 5 votes vote down vote up
def test_include_dirs(self):
        # As a sanity check, just test that get_include
        # includes something reasonable.  Somewhat
        # related to ticket #1405.
        include_dirs = [np.get_include()]
        for path in include_dirs:
            assert_(isinstance(path, (str, unicode)))
            assert_(path != '') 
Example 70
Project: FUTU_Stop_Loss   Author: BigtoC   File: records.py    MIT License 5 votes vote down vote up
def _setfieldnames(self, names, titles):
        """convert input field names into a list and assign to the _names
        attribute """

        if (names):
            if (type(names) in [list, tuple]):
                pass
            elif isinstance(names, (str, unicode)):
                names = names.split(',')
            else:
                raise NameError("illegal input names %s" % repr(names))

            self._names = [n.strip() for n in names[:self._nfields]]
        else:
            self._names = []

        # if the names are not specified, they will be assigned as
        #  "f0, f1, f2,..."
        # if not enough names are specified, they will be assigned as "f[n],
        # f[n+1],..." etc. where n is the number of specified names..."
        self._names += ['f%d' % i for i in range(len(self._names),
                                                 self._nfields)]
        # check for redundant names
        _dup = find_duplicate(self._names)
        if _dup:
            raise ValueError("Duplicate field names: %s" % _dup)

        if (titles):
            self._titles = [n.strip() for n in titles[:self._nfields]]
        else:
            self._titles = []
            titles = []

        if (self._nfields > len(titles)):
            self._titles += [None] * (self._nfields - len(titles)) 
Example 71
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_multiarray.py    MIT License 5 votes vote down vote up
def _test_sort_partition(self, name, kinds, **kwargs):
        # Previously, these would all hang
        for dt in [bytes, np.void, unicode]:
            zs = self._zeros(10, dt)
            sort_method = getattr(zs, name)
            sort_func = getattr(np, name)
            for kind in kinds:
                sort_method(kind=kind, **kwargs)
                sort_func(zs, kind=kind, **kwargs) 
Example 72
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_resize(self):
        # previously an error
        for dt in [bytes, np.void, unicode]:
            zs = self._zeros(10, dt)
            zs.resize(25)
            zs.resize((10, 10)) 
Example 73
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_view(self):
        for dt in [bytes, np.void, unicode]:
            zs = self._zeros(10, dt)

            # viewing as itself should be allowed
            assert_equal(zs.view(dt).dtype, np.dtype(dt))

            # viewing as any non-empty type gives an empty result
            assert_equal(zs.view((dt, 1)).shape, (0,)) 
Example 74
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_pickle(self):
        import pickle
        for dt in [bytes, np.void, unicode]:
            zs = self._zeros(10, dt)
            p = pickle.dumps(zs)
            zs2 = pickle.loads(p)

            assert_equal(zs.dtype, zs2.dtype) 
Example 75
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_searchsorted_unicode(self):
        # Test searchsorted on unicode strings.

        # 1.6.1 contained a string length miscalculation in
        # arraytypes.c.src:UNICODE_compare() which manifested as
        # incorrect/inconsistent results from searchsorted.
        a = np.array(['P:\\20x_dapi_cy3\\20x_dapi_cy3_20100185_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100186_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100187_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100189_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100190_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100191_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100192_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100193_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100194_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100195_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100196_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100197_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100198_1',
                      'P:\\20x_dapi_cy3\\20x_dapi_cy3_20100199_1'],
                     dtype=np.unicode)
        ind = np.arange(len(a))
        assert_equal([a.searchsorted(v, 'left') for v in a], ind)
        assert_equal([a.searchsorted(v, 'right') for v in a], ind + 1)
        assert_equal([a.searchsorted(a[i], 'left') for i in ind], ind)
        assert_equal([a.searchsorted(a[i], 'right') for i in ind], ind + 1) 
Example 76
Project: FUTU_Stop_Loss   Author: BigtoC   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_ip_types(self):
        unchecked_types = [bytes, unicode, np.void, object]

        x = np.random.random(24)*100
        x.shape = 2, 3, 4
        for types in np.sctypes.values():
            for T in types:
                if T not in unchecked_types:
                    self.tst_basic(x.copy().astype(T)) 
Example 77
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: _type_aliases.py    MIT License 4 votes vote down vote up
def _set_up_aliases():
    type_pairs = [('complex_', 'cdouble'),
                  ('int0', 'intp'),
                  ('uint0', 'uintp'),
                  ('single', 'float'),
                  ('csingle', 'cfloat'),
                  ('singlecomplex', 'cfloat'),
                  ('float_', 'double'),
                  ('intc', 'int'),
                  ('uintc', 'uint'),
                  ('int_', 'long'),
                  ('uint', 'ulong'),
                  ('cfloat', 'cdouble'),
                  ('longfloat', 'longdouble'),
                  ('clongfloat', 'clongdouble'),
                  ('longcomplex', 'clongdouble'),
                  ('bool_', 'bool'),
                  ('bytes_', 'string'),
                  ('string_', 'string'),
                  ('unicode_', 'unicode'),
                  ('object_', 'object')]
    if sys.version_info[0] >= 3:
        type_pairs.extend([('str_', 'unicode')])
    else:
        type_pairs.extend([('str_', 'string')])
    for alias, t in type_pairs:
        allTypes[alias] = allTypes[t]
        sctypeDict[alias] = sctypeDict[t]
    # Remove aliases overriding python types and modules
    to_remove = ['ulong', 'object', 'int', 'float',
                 'complex', 'bool', 'string', 'datetime', 'timedelta']
    if sys.version_info[0] >= 3:
        to_remove.extend(['bytes', 'str'])
    else:
        to_remove.extend(['unicode', 'long'])

    for t in to_remove:
        try:
            del allTypes[t]
            del sctypeDict[t]
        except KeyError:
            pass 
Example 78
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_multiarray.py    MIT License 4 votes vote down vote up
def test_field_names(self):
        # Test unicode and 8-bit / byte strings can be used
        a = np.zeros((1,), dtype=[('f1', 'i4'),
                                  ('f2', 'i4'),
                                  ('f3', [('sf1', 'i4')])])
        is_py3 = sys.version_info[0] >= 3
        if is_py3:
            funcs = (str,)
            # byte string indexing fails gracefully
            assert_raises(IndexError, a.__setitem__, b'f1', 1)
            assert_raises(IndexError, a.__getitem__, b'f1')
            assert_raises(IndexError, a['f1'].__setitem__, b'sf1', 1)
            assert_raises(IndexError, a['f1'].__getitem__, b'sf1')
        else:
            funcs = (str, unicode)
        for func in funcs:
            b = a.copy()
            fn1 = func('f1')
            b[fn1] = 1
            assert_equal(b[fn1], 1)
            fnn = func('not at all')
            assert_raises(ValueError, b.__setitem__, fnn, 1)
            assert_raises(ValueError, b.__getitem__, fnn)
            b[0][fn1] = 2
            assert_equal(b[fn1], 2)
            # Subfield
            assert_raises(ValueError, b[0].__setitem__, fnn, 1)
            assert_raises(ValueError, b[0].__getitem__, fnn)
            # Subfield
            fn3 = func('f3')
            sfn1 = func('sf1')
            b[fn3][sfn1] = 1
            assert_equal(b[fn3][sfn1], 1)
            assert_raises(ValueError, b[fn3].__setitem__, fnn, 1)
            assert_raises(ValueError, b[fn3].__getitem__, fnn)
            # multiple subfields
            fn2 = func('f2')
            b[fn2] = 3

            assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3))
            assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2))
            assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,)))

        # non-ascii unicode field indexing is well behaved
        if not is_py3:
            pytest.skip('non ascii unicode field indexing skipped; '
                        'raises segfault on python 2.x')
        else:
            assert_raises(ValueError, a.__setitem__, u'\u03e0', 1)
            assert_raises(ValueError, a.__getitem__, u'\u03e0') 
Example 79
Project: recruit   Author: Frank-qlu   File: _type_aliases.py    Apache License 2.0 4 votes vote down vote up
def _set_up_aliases():
    type_pairs = [('complex_', 'cdouble'),
                  ('int0', 'intp'),
                  ('uint0', 'uintp'),
                  ('single', 'float'),
                  ('csingle', 'cfloat'),
                  ('singlecomplex', 'cfloat'),
                  ('float_', 'double'),
                  ('intc', 'int'),
                  ('uintc', 'uint'),
                  ('int_', 'long'),
                  ('uint', 'ulong'),
                  ('cfloat', 'cdouble'),
                  ('longfloat', 'longdouble'),
                  ('clongfloat', 'clongdouble'),
                  ('longcomplex', 'clongdouble'),
                  ('bool_', 'bool'),
                  ('bytes_', 'string'),
                  ('string_', 'string'),
                  ('unicode_', 'unicode'),
                  ('object_', 'object')]
    if sys.version_info[0] >= 3:
        type_pairs.extend([('str_', 'unicode')])
    else:
        type_pairs.extend([('str_', 'string')])
    for alias, t in type_pairs:
        allTypes[alias] = allTypes[t]
        sctypeDict[alias] = sctypeDict[t]
    # Remove aliases overriding python types and modules
    to_remove = ['ulong', 'object', 'int', 'float',
                 'complex', 'bool', 'string', 'datetime', 'timedelta']
    if sys.version_info[0] >= 3:
        to_remove.extend(['bytes', 'str'])
    else:
        to_remove.extend(['unicode', 'long'])

    for t in to_remove:
        try:
            del allTypes[t]
            del sctypeDict[t]
        except KeyError:
            pass 
Example 80
Project: recruit   Author: Frank-qlu   File: test_multiarray.py    Apache License 2.0 4 votes vote down vote up
def test_field_names(self):
        # Test unicode and 8-bit / byte strings can be used
        a = np.zeros((1,), dtype=[('f1', 'i4'),
                                  ('f2', 'i4'),
                                  ('f3', [('sf1', 'i4')])])
        is_py3 = sys.version_info[0] >= 3
        if is_py3:
            funcs = (str,)
            # byte string indexing fails gracefully
            assert_raises(IndexError, a.__setitem__, b'f1', 1)
            assert_raises(IndexError, a.__getitem__, b'f1')
            assert_raises(IndexError, a['f1'].__setitem__, b'sf1', 1)
            assert_raises(IndexError, a['f1'].__getitem__, b'sf1')
        else:
            funcs = (str, unicode)
        for func in funcs:
            b = a.copy()
            fn1 = func('f1')
            b[fn1] = 1
            assert_equal(b[fn1], 1)
            fnn = func('not at all')
            assert_raises(ValueError, b.__setitem__, fnn, 1)
            assert_raises(ValueError, b.__getitem__, fnn)
            b[0][fn1] = 2
            assert_equal(b[fn1], 2)
            # Subfield
            assert_raises(ValueError, b[0].__setitem__, fnn, 1)
            assert_raises(ValueError, b[0].__getitem__, fnn)
            # Subfield
            fn3 = func('f3')
            sfn1 = func('sf1')
            b[fn3][sfn1] = 1
            assert_equal(b[fn3][sfn1], 1)
            assert_raises(ValueError, b[fn3].__setitem__, fnn, 1)
            assert_raises(ValueError, b[fn3].__getitem__, fnn)
            # multiple subfields
            fn2 = func('f2')
            b[fn2] = 3

            assert_equal(b[['f1', 'f2']][0].tolist(), (2, 3))
            assert_equal(b[['f2', 'f1']][0].tolist(), (3, 2))
            assert_equal(b[['f1', 'f3']][0].tolist(), (2, (1,)))

        # non-ascii unicode field indexing is well behaved
        if not is_py3:
            pytest.skip('non ascii unicode field indexing skipped; '
                        'raises segfault on python 2.x')
        else:
            assert_raises(ValueError, a.__setitem__, u'\u03e0', 1)
            assert_raises(ValueError, a.__getitem__, u'\u03e0')