Python numpy.compat.asbytes() Examples

The following are code examples for showing how to use numpy.compat.asbytes(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: LaserTOF   Author: kyleuckert   File: _iotools.py    MIT License 6 votes vote down vote up
def __init__(self, delimiter=None, comments=asbytes('#'), autostrip=True):
        self.comments = comments
        # Delimiter is a character
        if isinstance(delimiter, unicode):
            delimiter = delimiter.encode('ascii')
        if (delimiter is None) or _is_bytes_like(delimiter):
            delimiter = delimiter or None
            _handyman = self._delimited_splitter
        # Delimiter is a list of field widths
        elif hasattr(delimiter, '__iter__'):
            _handyman = self._variablewidth_splitter
            idx = np.cumsum([0] + list(delimiter))
            delimiter = [slice(i, j) for (i, j) in zip(idx[:-1], idx[1:])]
        # Delimiter is a single integer
        elif int(delimiter):
            (_handyman, delimiter) = (
                    self._fixedwidth_splitter, int(delimiter))
        else:
            (_handyman, delimiter) = (self._delimited_splitter, None)
        self.delimiter = delimiter
        if autostrip:
            self._handyman = self.autostrip(_handyman)
        else:
            self._handyman = _handyman
    # 
Example 2
Project: LaserTOF   Author: kyleuckert   File: test_format.py    MIT License 6 votes vote down vote up
def test_bad_header():
    # header of length less than 2 should fail
    s = BytesIO()
    assert_raises(ValueError, format.read_array_header_1_0, s)
    s = BytesIO(asbytes('1'))
    assert_raises(ValueError, format.read_array_header_1_0, s)

    # header shorter than indicated size should fail
    s = BytesIO(asbytes('\x01\x00'))
    assert_raises(ValueError, format.read_array_header_1_0, s)

    # headers without the exact keys required should fail
    d = {"shape": (1, 2),
         "descr": "x"}
    s = BytesIO()
    format.write_array_header_1_0(s, d)
    assert_raises(ValueError, format.read_array_header_1_0, s)

    d = {"shape": (1, 2),
         "fortran_order": False,
         "descr": "x",
         "extrakey": -1}
    s = BytesIO()
    format.write_array_header_1_0(s, d)
    assert_raises(ValueError, format.read_array_header_1_0, s) 
Example 3
Project: LaserTOF   Author: kyleuckert   File: test_mrecords.py    MIT License 6 votes vote down vote up
def test_exotic_formats(self):
        # Test that 'exotic' formats are processed properly
        easy = mrecarray(1, dtype=[('i', int), ('s', '|S8'), ('f', float)])
        easy[0] = masked
        assert_equal(easy.filled(1).item(), (1, asbytes('1'), 1.))

        solo = mrecarray(1, dtype=[('f0', '<f8', (2, 2))])
        solo[0] = masked
        assert_equal(solo.filled(1).item(),
                     np.array((1,), dtype=solo.dtype).item())

        mult = mrecarray(2, dtype="i4, (2,3)float, float")
        mult[0] = masked
        mult[1] = (1, 1, 1)
        mult.filled(0)
        assert_equal_records(mult.filled(0),
                             np.array([(0, 0, 0), (1, 1, 1)],
                                      dtype=mult.dtype)) 
Example 4
Project: LaserTOF   Author: kyleuckert   File: test_core.py    MIT License 6 votes vote down vote up
def test_fillvalue(self):
        # Yet more fun with the fill_value
        data = masked_array([1, 2, 3], fill_value=-999)
        series = data[[0, 2, 1]]
        assert_equal(series._fill_value, data._fill_value)

        mtype = [('f', float), ('s', '|S3')]
        x = array([(1, 'a'), (2, 'b'), (pi, 'pi')], dtype=mtype)
        x.fill_value = 999
        assert_equal(x.fill_value.item(), [999., asbytes('999')])
        assert_equal(x['f'].fill_value, 999)
        assert_equal(x['s'].fill_value, asbytes('999'))

        x.fill_value = (9, '???')
        assert_equal(x.fill_value.item(), (9, asbytes('???')))
        assert_equal(x['f'].fill_value, 9)
        assert_equal(x['s'].fill_value, asbytes('???'))

        x = array([1, 2, 3.1])
        x.fill_value = 999
        assert_equal(np.asarray(x.fill_value).dtype, float)
        assert_equal(x.fill_value, 999.)
        assert_equal(x._fill_value, np.array(999.)) 
Example 5
Project: LaserTOF   Author: kyleuckert   File: test_datetime.py    MIT License 6 votes vote down vote up
def test_pickle(self):
        # Check that pickle roundtripping works
        dt = np.dtype('M8[7D]')
        assert_equal(pickle.loads(pickle.dumps(dt)), dt)
        dt = np.dtype('M8[W]')
        assert_equal(pickle.loads(pickle.dumps(dt)), dt)

        # Check that loading pickles from 1.6 works
        pkl = "cnumpy\ndtype\np0\n(S'M8'\np1\nI0\nI1\ntp2\nRp3\n" + \
              "(I4\nS'<'\np4\nNNNI-1\nI-1\nI0\n((dp5\n(S'D'\np6\n" + \
              "I7\nI1\nI1\ntp7\ntp8\ntp9\nb."
        assert_equal(pickle.loads(asbytes(pkl)), np.dtype('<M8[7D]'))
        pkl = "cnumpy\ndtype\np0\n(S'M8'\np1\nI0\nI1\ntp2\nRp3\n" + \
              "(I4\nS'<'\np4\nNNNI-1\nI-1\nI0\n((dp5\n(S'W'\np6\n" + \
              "I1\nI1\nI1\ntp7\ntp8\ntp9\nb."
        assert_equal(pickle.loads(asbytes(pkl)), np.dtype('<M8[W]'))
        pkl = "cnumpy\ndtype\np0\n(S'M8'\np1\nI0\nI1\ntp2\nRp3\n" + \
              "(I4\nS'>'\np4\nNNNI-1\nI-1\nI0\n((dp5\n(S'us'\np6\n" + \
              "I1\nI1\nI1\ntp7\ntp8\ntp9\nb."
        assert_equal(pickle.loads(asbytes(pkl)), np.dtype('>M8[us]')) 
Example 6
Project: LaserTOF   Author: kyleuckert   File: test_defchararray.py    MIT License 6 votes vote down vote up
def test_replace(self):
        R = self.A.replace(asbytes_nested(['3', 'a']),
                           asbytes_nested(['##########', '@']))
        tgt = asbytes_nested([[' abc ', ''],
                              ['12##########45', '[email protected]'],
                              ['12########## \t ##########45 \x00', 'UPPER']])
        assert_(issubclass(R.dtype.type, np.string_))
        assert_array_equal(R, tgt)

        if sys.version_info[0] < 3:
            # NOTE: b'abc'.replace(b'a', 'b') is not allowed on Py3
            R = self.A.replace(asbytes('a'), sixu('\u03a3'))
            tgt = [[sixu(' \u03a3bc '), ''],
                   ['12345', sixu('MixedC\u03a3se')],
                   ['123 \t 345 \x00', 'UPPER']]
            assert_(issubclass(R.dtype.type, np.unicode_))
            assert_array_equal(R, tgt) 
Example 7
Project: LaserTOF   Author: kyleuckert   File: test_defchararray.py    MIT License 6 votes vote down vote up
def test_rjust(self):
        assert_(issubclass(self.A.rjust(10).dtype.type, np.string_))

        C = self.A.rjust([10, 20])
        assert_array_equal(np.char.str_len(C), [[10, 20], [10, 20], [12, 20]])

        C = self.A.rjust(20, asbytes('#'))
        assert_(np.all(C.startswith(asbytes('#'))))
        assert_array_equal(C.endswith(asbytes('#')),
                           [[False, True], [False, False], [False, False]])

        C = np.char.rjust(asbytes('FOO'), [[10, 20], [15, 8]])
        tgt = asbytes_nested([['       FOO', '                 FOO'],
                              ['            FOO', '     FOO']])
        assert_(issubclass(C.dtype.type, np.string_))
        assert_array_equal(C, tgt) 
Example 8
Project: LaserTOF   Author: kyleuckert   File: test_defchararray.py    MIT License 6 votes vote down vote up
def test_slice(self):
        """Regression test for https://github.com/numpy/numpy/issues/5982"""

        arr = np.array([['abc ', 'def '], ['geh ', 'ijk ']],
                       dtype='S4').view(np.chararray)
        sl1 = arr[:]
        assert_array_equal(sl1, arr)
        assert_(sl1.base is arr)
        assert_(sl1.base.base is arr.base)

        sl2 = arr[:, :]
        assert_array_equal(sl2, arr)
        assert_(sl2.base is arr)
        assert_(sl2.base.base is arr.base)

        assert_(arr[0, 0] == asbytes('abc')) 
Example 9
Project: LaserTOF   Author: kyleuckert   File: test_nditer.py    MIT License 6 votes vote down vote up
def test_iter_buffering_string():
    # Safe casting disallows shrinking strings
    a = np.array(['abc', 'a', 'abcd'], dtype=np.bytes_)
    assert_equal(a.dtype, np.dtype('S4'))
    assert_raises(TypeError, nditer, a, ['buffered'], ['readonly'],
                  op_dtypes='S2')
    i = nditer(a, ['buffered'], ['readonly'], op_dtypes='S6')
    assert_equal(i[0], asbytes('abc'))
    assert_equal(i[0].dtype, np.dtype('S6'))

    a = np.array(['abc', 'a', 'abcd'], dtype=np.unicode)
    assert_equal(a.dtype, np.dtype('U4'))
    assert_raises(TypeError, nditer, a, ['buffered'], ['readonly'],
                    op_dtypes='U2')
    i = nditer(a, ['buffered'], ['readonly'], op_dtypes='U6')
    assert_equal(i[0], sixu('abc'))
    assert_equal(i[0].dtype, np.dtype('U6')) 
Example 10
Project: LaserTOF   Author: kyleuckert   File: _iotools.py    MIT License 5 votes vote down vote up
def _is_bytes_like(obj):
    """
    Check whether obj behaves like a bytes object.
    """
    try:
        obj + asbytes('')
    except (TypeError, ValueError):
        return False
    return True 
Example 11
Project: LaserTOF   Author: kyleuckert   File: _iotools.py    MIT License 5 votes vote down vote up
def _delimited_splitter(self, line):
        if self.comments is not None:
            line = line.split(self.comments)[0]
        line = line.strip(asbytes(" \r\n"))
        if not line:
            return []
        return line.split(self.delimiter)
    # 
Example 12
Project: LaserTOF   Author: kyleuckert   File: _iotools.py    MIT License 5 votes vote down vote up
def _fixedwidth_splitter(self, line):
        if self.comments is not None:
            line = line.split(self.comments)[0]
        line = line.strip(asbytes("\r\n"))
        if not line:
            return []
        fixed = self.delimiter
        slices = [slice(i, i + fixed) for i in range(0, len(line), fixed)]
        return [line[s] for s in slices]
    # 
Example 13
Project: LaserTOF   Author: kyleuckert   File: _iotools.py    MIT License 5 votes vote down vote up
def str2bool(value):
    """
    Tries to transform a string supposed to represent a boolean to a boolean.

    Parameters
    ----------
    value : str
        The string that is transformed to a boolean.

    Returns
    -------
    boolval : bool
        The boolean representation of `value`.

    Raises
    ------
    ValueError
        If the string is not 'True' or 'False' (case independent)

    Examples
    --------
    >>> np.lib._iotools.str2bool('TRUE')
    True
    >>> np.lib._iotools.str2bool('false')
    False

    """
    value = value.upper()
    if value == asbytes('TRUE'):
        return True
    elif value == asbytes('FALSE'):
        return False
    else:
        raise ValueError("Invalid boolean") 
Example 14
Project: LaserTOF   Author: kyleuckert   File: test_io.py    MIT License 5 votes vote down vote up
def __init__(self, s=""):
        BytesIO.__init__(self, asbytes(s)) 
Example 15
Project: LaserTOF   Author: kyleuckert   File: test_io.py    MIT License 5 votes vote down vote up
def write(self, s):
        BytesIO.write(self, asbytes(s)) 
Example 16
Project: LaserTOF   Author: kyleuckert   File: test_io.py    MIT License 5 votes vote down vote up
def writelines(self, lines):
        BytesIO.writelines(self, [asbytes(s) for s in lines]) 
Example 17
Project: LaserTOF   Author: kyleuckert   File: test_io.py    MIT License 5 votes vote down vote up
def test_array(self):
        a = np.array([[1, 2], [3, 4]], float)
        fmt = "%.18e"
        c = BytesIO()
        np.savetxt(c, a, fmt=fmt)
        c.seek(0)
        assert_equal(c.readlines(),
                     [asbytes((fmt + ' ' + fmt + '\n') % (1, 2)),
                      asbytes((fmt + ' ' + fmt + '\n') % (3, 4))])

        a = np.array([[1, 2], [3, 4]], int)
        c = BytesIO()
        np.savetxt(c, a, fmt='%d')
        c.seek(0)
        assert_equal(c.readlines(), [b'1 2\n', b'3 4\n']) 
Example 18
Project: LaserTOF   Author: kyleuckert   File: test_io.py    MIT License 5 votes vote down vote up
def test_gft_using_generator(self):
        # gft doesn't work with unicode.
        def count():
            for i in range(10):
                yield asbytes("%d" % i)

        res = np.genfromtxt(count())
        assert_array_equal(res, np.arange(10)) 
Example 19
Project: LaserTOF   Author: kyleuckert   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_no_delimiter(self):
        "Test LineSplitter w/o delimiter"
        strg = asbytes(" 1 2 3 4  5 # test")
        test = LineSplitter()(strg)
        assert_equal(test, asbytes_nested(['1', '2', '3', '4', '5']))
        test = LineSplitter('')(strg)
        assert_equal(test, asbytes_nested(['1', '2', '3', '4', '5'])) 
Example 20
Project: LaserTOF   Author: kyleuckert   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_space_delimiter(self):
        "Test space delimiter"
        strg = asbytes(" 1 2 3 4  5 # test")
        test = LineSplitter(asbytes(' '))(strg)
        assert_equal(test, asbytes_nested(['1', '2', '3', '4', '', '5']))
        test = LineSplitter(asbytes('  '))(strg)
        assert_equal(test, asbytes_nested(['1 2 3 4', '5'])) 
Example 21
Project: LaserTOF   Author: kyleuckert   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_tab_delimiter(self):
        "Test tab delimiter"
        strg = asbytes(" 1\t 2\t 3\t 4\t 5  6")
        test = LineSplitter(asbytes('\t'))(strg)
        assert_equal(test, asbytes_nested(['1', '2', '3', '4', '5  6']))
        strg = asbytes(" 1  2\t 3  4\t 5  6")
        test = LineSplitter(asbytes('\t'))(strg)
        assert_equal(test, asbytes_nested(['1  2', '3  4', '5  6'])) 
Example 22
Project: LaserTOF   Author: kyleuckert   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_constant_fixed_width(self):
        "Test LineSplitter w/ fixed-width fields"
        strg = asbytes("  1  2  3  4     5   # test")
        test = LineSplitter(3)(strg)
        assert_equal(test, asbytes_nested(['1', '2', '3', '4', '', '5', '']))
        #
        strg = asbytes("  1     3  4  5  6# test")
        test = LineSplitter(20)(strg)
        assert_equal(test, asbytes_nested(['1     3  4  5  6']))
        #
        strg = asbytes("  1     3  4  5  6# test")
        test = LineSplitter(30)(strg)
        assert_equal(test, asbytes_nested(['1     3  4  5  6'])) 
Example 23
Project: LaserTOF   Author: kyleuckert   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_variable_fixed_width(self):
        strg = asbytes("  1     3  4  5  6# test")
        test = LineSplitter((3, 6, 6, 3))(strg)
        assert_equal(test, asbytes_nested(['1', '3', '4  5', '6']))
        #
        strg = asbytes("  1     3  4  5  6# test")
        test = LineSplitter((6, 6, 9))(strg)
        assert_equal(test, asbytes_nested(['1', '3  4', '5  6']))

# ----------------------------------------------------------------------------- 
Example 24
Project: LaserTOF   Author: kyleuckert   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_upgrade(self):
        "Tests the upgrade method."

        converter = StringConverter()
        assert_equal(converter._status, 0)

        # test int
        assert_equal(converter.upgrade(asbytes('0')), 0)
        assert_equal(converter._status, 1)

        # On systems where integer defaults to 32-bit, the statuses will be
        # offset by one, so we check for this here.
        import numpy.core.numeric as nx
        status_offset = int(nx.dtype(nx.integer).itemsize < nx.dtype(nx.int64).itemsize)

        # test int > 2**32
        assert_equal(converter.upgrade(asbytes('17179869184')), 17179869184)
        assert_equal(converter._status, 1 + status_offset)

        # test float
        assert_allclose(converter.upgrade(asbytes('0.')), 0.0)
        assert_equal(converter._status, 2 + status_offset)

        # test complex
        assert_equal(converter.upgrade(asbytes('0j')), complex('0j'))
        assert_equal(converter._status, 3 + status_offset)

        # test str
        assert_equal(converter.upgrade(asbytes('a')), asbytes('a'))
        assert_equal(converter._status, len(converter._mapper) - 1) 
Example 25
Project: LaserTOF   Author: kyleuckert   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_missing(self):
        "Tests the use of missing values."
        converter = StringConverter(missing_values=(asbytes('missing'),
                                                    asbytes('missed')))
        converter.upgrade(asbytes('0'))
        assert_equal(converter(asbytes('0')), 0)
        assert_equal(converter(asbytes('')), converter.default)
        assert_equal(converter(asbytes('missing')), converter.default)
        assert_equal(converter(asbytes('missed')), converter.default)
        try:
            converter('miss')
        except ValueError:
            pass 
Example 26
Project: LaserTOF   Author: kyleuckert   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_upgrademapper(self):
        "Tests updatemapper"
        dateparser = _bytes_to_date
        StringConverter.upgrade_mapper(dateparser, date(2000, 1, 1))
        convert = StringConverter(dateparser, date(2000, 1, 1))
        test = convert(asbytes('2001-01-01'))
        assert_equal(test, date(2001, 1, 1))
        test = convert(asbytes('2009-01-01'))
        assert_equal(test, date(2009, 1, 1))
        test = convert(asbytes(''))
        assert_equal(test, date(2000, 1, 1)) 
Example 27
Project: LaserTOF   Author: kyleuckert   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_keep_default_zero(self):
        "Check that we don't lose a default of 0"
        converter = StringConverter(int, default=0,
                                    missing_values=asbytes("N/A"))
        assert_equal(converter.default, 0) 
Example 28
Project: LaserTOF   Author: kyleuckert   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_keep_missing_values(self):
        "Check that we're not losing missing values"
        converter = StringConverter(int, default=0,
                                    missing_values=asbytes("N/A"))
        assert_equal(
            converter.missing_values, set(asbytes_nested(['', 'N/A']))) 
Example 29
Project: LaserTOF   Author: kyleuckert   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_int64_dtype(self):
        "Check that int64 integer types can be specified"
        converter = StringConverter(np.int64, default=0)
        val = asbytes("-9223372036854775807")
        assert_(converter(val) == -9223372036854775807)
        val = asbytes("9223372036854775807")
        assert_(converter(val) == 9223372036854775807) 
Example 30
Project: LaserTOF   Author: kyleuckert   File: test__iotools.py    MIT License 5 votes vote down vote up
def test_uint64_dtype(self):
        "Check that uint64 integer types can be specified"
        converter = StringConverter(np.uint64, default=0)
        val = asbytes("9223372043271415339")
        assert_(converter(val) == 9223372043271415339) 
Example 31
Project: LaserTOF   Author: kyleuckert   File: test_mrecords.py    MIT License 5 votes vote down vote up
def test_get(self):
        # Tests fields retrieval
        base = self.base.copy()
        mbase = base.view(mrecarray)
        # As fields..........
        for field in ('a', 'b', 'c'):
            assert_equal(getattr(mbase, field), mbase[field])
            assert_equal(base[field], mbase[field])
        # as elements .......
        mbase_first = mbase[0]
        assert_(isinstance(mbase_first, mrecarray))
        assert_equal(mbase_first.dtype, mbase.dtype)
        assert_equal(mbase_first.tolist(), (1, 1.1, asbytes('one')))
        # Used to be mask, now it's recordmask
        assert_equal(mbase_first.recordmask, nomask)
        assert_equal(mbase_first._mask.item(), (False, False, False))
        assert_equal(mbase_first['a'], mbase['a'][0])
        mbase_last = mbase[-1]
        assert_(isinstance(mbase_last, mrecarray))
        assert_equal(mbase_last.dtype, mbase.dtype)
        assert_equal(mbase_last.tolist(), (None, None, None))
        # Used to be mask, now it's recordmask
        assert_equal(mbase_last.recordmask, True)
        assert_equal(mbase_last._mask.item(), (True, True, True))
        assert_equal(mbase_last['a'], mbase['a'][-1])
        assert_((mbase_last['a'] is masked))
        # as slice ..........
        mbase_sl = mbase[:2]
        assert_(isinstance(mbase_sl, mrecarray))
        assert_equal(mbase_sl.dtype, mbase.dtype)
        # Used to be mask, now it's recordmask
        assert_equal(mbase_sl.recordmask, [0, 1])
        assert_equal_records(mbase_sl.mask,
                             np.array([(False, False, False),
                                       (True, True, True)],
                                      dtype=mbase._mask.dtype))
        assert_equal_records(mbase_sl, base[:2].view(mrecarray))
        for field in ('a', 'b', 'c'):
            assert_equal(getattr(mbase_sl, field), base[:2][field]) 
Example 32
Project: LaserTOF   Author: kyleuckert   File: test_mrecords.py    MIT License 5 votes vote down vote up
def test_tolist(self):
        # Test tolist.
        _a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
        _b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
        _c = ma.array(['one', 'two', 'three'], mask=[1, 0, 0], dtype='|S8')
        ddtype = [('a', int), ('b', float), ('c', '|S8')]
        mrec = fromarrays([_a, _b, _c], dtype=ddtype,
                          fill_value=(99999, 99999., 'N/A'))

        assert_equal(mrec.tolist(),
                     [(1, 1.1, None), (2, 2.2, asbytes('two')),
                      (None, None, asbytes('three'))]) 
Example 33
Project: LaserTOF   Author: kyleuckert   File: test_mrecords.py    MIT License 5 votes vote down vote up
def setup(self):
        # Generic setup
        _a = ma.array([1, 2, 3], mask=[0, 0, 1], dtype=int)
        _b = ma.array([1.1, 2.2, 3.3], mask=[0, 0, 1], dtype=float)
        _c = ma.array(list(map(asbytes, ['one', 'two', 'three'])),
                      mask=[0, 0, 1], dtype='|S8')
        ddtype = [('a', int), ('b', float), ('c', '|S8')]
        mrec = fromarrays([_a, _b, _c], dtype=ddtype,
                          fill_value=(asbytes('99999'), asbytes('99999.'),
                                      asbytes('N/A')))
        nrec = recfromarrays((_a._data, _b._data, _c._data), dtype=ddtype)
        self.data = (mrec, nrec, ddtype) 
Example 34
Project: LaserTOF   Author: kyleuckert   File: test_core.py    MIT License 5 votes vote down vote up
def test_tolist(self):
        # Tests to list
        # ... on 1D
        x = array(np.arange(12))
        x[[1, -2]] = masked
        xlist = x.tolist()
        self.assertTrue(xlist[1] is None)
        self.assertTrue(xlist[-2] is None)
        # ... on 2D
        x.shape = (3, 4)
        xlist = x.tolist()
        ctrl = [[0, None, 2, 3], [4, 5, 6, 7], [8, 9, None, 11]]
        assert_equal(xlist[0], [0, None, 2, 3])
        assert_equal(xlist[1], [4, 5, 6, 7])
        assert_equal(xlist[2], [8, 9, None, 11])
        assert_equal(xlist, ctrl)
        # ... on structured array w/ masked records
        x = array(list(zip([1, 2, 3],
                           [1.1, 2.2, 3.3],
                           ['one', 'two', 'thr'])),
                  dtype=[('a', int), ('b', float), ('c', '|S8')])
        x[-1] = masked
        assert_equal(x.tolist(),
                     [(1, 1.1, asbytes('one')),
                      (2, 2.2, asbytes('two')),
                      (None, None, None)])
        # ... on structured array w/ masked fields
        a = array([(1, 2,), (3, 4)], mask=[(0, 1), (0, 0)],
                  dtype=[('a', int), ('b', int)])
        test = a.tolist()
        assert_equal(test, [[1, None], [3, 4]])
        # ... on mvoid
        a = a[0]
        test = a.tolist()
        assert_equal(test, [1, None]) 
Example 35
Project: LaserTOF   Author: kyleuckert   File: test_random.py    MIT License 5 votes vote down vote up
def test_bytes(self):
        np.random.seed(self.seed)
        actual = np.random.bytes(10)
        desired = asbytes('\x82Ui\x9e\xff\x97+Wf\xa5')
        assert_equal(actual, desired) 
Example 36
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_version0_float32(self):
        s = '\x80\x02cnumpy.core._internal\n_reconstruct\nq\x01cnumpy\nndarray\nq\x02K\x00\x85U\x01b\x87Rq\x03(K\x04\x85cnumpy\ndtype\nq\x04U\x02f4K\x00K\x01\x87Rq\x05(U\x01<NNJ\xff\xff\xff\xffJ\xff\xff\xff\xfftb\x89U\x10\x00\x00\x80?\x00\x00\[email protected]\x00\[email protected]@\x00\x00\[email protected]'
        a = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32)
        p = self._loads(asbytes(s))
        assert_equal(a, p) 
Example 37
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_version0_object(self):
        s = '\x80\x02cnumpy.core._internal\n_reconstruct\nq\x01cnumpy\nndarray\nq\x02K\x00\x85U\x01b\x87Rq\x03(K\x02\x85cnumpy\ndtype\nq\x04U\x02O8K\x00K\x01\x87Rq\x05(U\x01|NNJ\xff\xff\xff\xffJ\xff\xff\xff\xfftb\x89]q\x06(}q\x07U\x01aK\x01s}q\x08U\x01bK\x02setb.'
        a = np.array([{'a': 1}, {'b': 2}])
        p = self._loads(asbytes(s))
        assert_equal(a, p)

    # version 1 pickles, using protocol=2 to pickle 
Example 38
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_version1_int8(self):
        s = '\x80\x02cnumpy.core._internal\n_reconstruct\nq\x01cnumpy\nndarray\nq\x02K\x00\x85U\x01b\x87Rq\x03(K\x01K\x04\x85cnumpy\ndtype\nq\x04U\x02i1K\x00K\x01\x87Rq\x05(K\x01U\x01|NNJ\xff\xff\xff\xffJ\xff\xff\xff\xfftb\x89U\x04\x01\x02\x03\x04tb.'
        a = np.array([1, 2, 3, 4], dtype=np.int8)
        p = self._loads(asbytes(s))
        assert_equal(a, p) 
Example 39
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_version1_float32(self):
        s = '\x80\x02cnumpy.core._internal\n_reconstruct\nq\x01cnumpy\nndarray\nq\x02K\x00\x85U\x01b\x87Rq\x03(K\x01K\x04\x85cnumpy\ndtype\nq\x04U\x02f4K\x00K\x01\x87Rq\x05(K\x01U\x01<NNJ\xff\xff\xff\xffJ\xff\xff\xff\xfftb\x89U\x10\x00\x00\x80?\x00\x00\[email protected]\x00\[email protected]@\x00\x00\[email protected]'
        a = np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32)
        p = self._loads(asbytes(s))
        assert_equal(a, p) 
Example 40
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_version1_object(self):
        s = '\x80\x02cnumpy.core._internal\n_reconstruct\nq\x01cnumpy\nndarray\nq\x02K\x00\x85U\x01b\x87Rq\x03(K\x01K\x02\x85cnumpy\ndtype\nq\x04U\x02O8K\x00K\x01\x87Rq\x05(K\x01U\x01|NNJ\xff\xff\xff\xffJ\xff\xff\xff\xfftb\x89]q\x06(}q\x07U\x01aK\x01s}q\x08U\x01bK\x02setb.'
        a = np.array([{'a': 1}, {'b': 2}])
        p = self._loads(asbytes(s))
        assert_equal(a, p) 
Example 41
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def _check_from(self, s, value, **kw):
        y = np.fromstring(asbytes(s), **kw)
        assert_array_equal(y, value)

        f = open(self.filename, 'wb')
        f.write(asbytes(s))
        f.close()
        y = np.fromfile(self.filename, **kw)
        assert_array_equal(y, value) 
Example 42
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_dtype_bool(self):
        # can't use _check_from because fromstring can't handle True/False
        v = np.array([True, False, True, False], dtype=np.bool_)
        s = '1,0,-2.3,0'
        f = open(self.filename, 'wb')
        f.write(asbytes(s))
        f.close()
        y = np.fromfile(self.filename, sep=',', dtype=np.bool_)
        assert_(y.dtype == '?')
        assert_array_equal(y, v) 
Example 43
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_empty(self):
        yield self.tst_basic, asbytes(''), np.array([]), {} 
Example 44
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_bytes_fields(self):
            # Bytes are not allowed in field names and not recognized in titles
            # on Py3
            assert_raises(TypeError, np.dtype, [(asbytes('a'), int)])
            assert_raises(TypeError, np.dtype, [(('b', asbytes('a')), int)])

            dt = np.dtype([((asbytes('a'), 'b'), int)])
            assert_raises(ValueError, dt.__getitem__, asbytes('a'))

            x = np.array([(1,), (2,), (3,)], dtype=dt)
            assert_raises(IndexError, x.__getitem__, asbytes('a'))

            y = x[0]
            assert_raises(IndexError, y.__getitem__, asbytes('a')) 
Example 45
Project: LaserTOF   Author: kyleuckert   File: test_multiarray.py    MIT License 5 votes vote down vote up
def test_export_record(self):
        dt = [('a', 'b'),
              ('b', 'h'),
              ('c', 'i'),
              ('d', 'l'),
              ('dx', 'q'),
              ('e', 'B'),
              ('f', 'H'),
              ('g', 'I'),
              ('h', 'L'),
              ('hx', 'Q'),
              ('i', np.single),
              ('j', np.double),
              ('k', np.longdouble),
              ('ix', np.csingle),
              ('jx', np.cdouble),
              ('kx', np.clongdouble),
              ('l', 'S4'),
              ('m', 'U4'),
              ('n', 'V3'),
              ('o', '?'),
              ('p', np.half),
              ]
        x = np.array(
                [(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
                    asbytes('aaaa'), 'bbbb', asbytes('   '), True, 1.0)],
                dtype=dt)
        y = memoryview(x)
        assert_equal(y.shape, (1,))
        assert_equal(y.ndim, 1)
        assert_equal(y.suboffsets, EMPTY)

        sz = sum([np.dtype(b).itemsize for a, b in dt])
        if np.dtype('l').itemsize == 4:
            assert_equal(y.format, 'T{b:a:=h:b:i:c:l:d:q:dx:B:e:@H:f:=I:g:L:h:Q:hx:f:i:d:j:^g:k:=Zf:ix:Zd:jx:^Zg:kx:4s:l:=4w:m:3x:n:?:o:@e:p:}')
        else:
            assert_equal(y.format, 'T{b:a:=h:b:i:c:q:d:q:dx:B:e:@H:f:=I:g:Q:h:Q:hx:f:i:d:j:^g:k:=Zf:ix:Zd:jx:^Zg:kx:4s:l:=4w:m:3x:n:?:o:@e:p:}')
        # Cannot test if NPY_RELAXED_STRIDES_CHECKING changes the strides
        if not (np.ones(1).strides[0] == np.iinfo(np.intp).max):
            assert_equal(y.strides, (sz,))
        assert_equal(y.itemsize, sz) 
Example 46
Project: LaserTOF   Author: kyleuckert   File: test_ufunc.py    MIT License 5 votes vote down vote up
def test_pickle_withstring(self):
        import pickle
        astring = asbytes("cnumpy.core\n_ufunc_reconstruct\np0\n"
                "(S'numpy.core.umath'\np1\nS'cos'\np2\ntp3\nRp4\n.")
        assert_(pickle.loads(astring) is np.cos) 
Example 47
Project: LaserTOF   Author: kyleuckert   File: test_records.py    MIT License 5 votes vote down vote up
def test_method_array(self):
        r = np.rec.array(asbytes('abcdefg') * 100, formats='i2,a3,i4', shape=3, byteorder='big')
        assert_equal(r[1].item(), (25444, asbytes('efg'), 1633837924)) 
Example 48
Project: LaserTOF   Author: kyleuckert   File: test_records.py    MIT License 5 votes vote down vote up
def test_method_array2(self):
        r = np.rec.array([(1, 11, 'a'), (2, 22, 'b'), (3, 33, 'c'), (4, 44, 'd'), (5, 55, 'ex'),
                     (6, 66, 'f'), (7, 77, 'g')], formats='u1,f4,a1')
        assert_equal(r[1].item(), (2, 22.0, asbytes('b'))) 
Example 49
Project: LaserTOF   Author: kyleuckert   File: test_records.py    MIT License 5 votes vote down vote up
def test_recarray_slices(self):
        r = np.rec.array([(1, 11, 'a'), (2, 22, 'b'), (3, 33, 'c'), (4, 44, 'd'), (5, 55, 'ex'),
                     (6, 66, 'f'), (7, 77, 'g')], formats='u1,f4,a1')
        assert_equal(r[1::2][1].item(), (4, 44.0, asbytes('d'))) 
Example 50
Project: LaserTOF   Author: kyleuckert   File: test_defchararray.py    MIT License 5 votes vote down vote up
def test_from_string(self):
        A = np.char.array(asbytes('abc'))
        assert_equal(len(A), 1)
        assert_equal(len(A[0]), 3)
        assert_(issubclass(A.dtype.type, np.string_)) 
Example 51
Project: LaserTOF   Author: kyleuckert   File: test_defchararray.py    MIT License 5 votes vote down vote up
def test_it(self):
        assert_equal(self.A.shape, (4,))
        assert_equal(self.A.upper()[:2].tobytes(), asbytes('AB')) 
Example 52
Project: LaserTOF   Author: kyleuckert   File: test_defchararray.py    MIT License 5 votes vote down vote up
def test_center(self):
        assert_(issubclass(self.A.center(10).dtype.type, np.string_))
        C = self.A.center([10, 20])
        assert_array_equal(np.char.str_len(C), [[10, 20], [10, 20], [12, 20]])

        C = self.A.center(20, asbytes('#'))
        assert_(np.all(C.startswith(asbytes('#'))))
        assert_(np.all(C.endswith(asbytes('#'))))

        C = np.char.center(asbytes('FOO'), [[10, 20], [15, 8]])
        tgt = asbytes_nested([['   FOO    ', '        FOO         '],
                              ['      FOO      ', '  FOO   ']])
        assert_(issubclass(C.dtype.type, np.string_))
        assert_array_equal(C, tgt) 
Example 53
Project: LaserTOF   Author: kyleuckert   File: test_defchararray.py    MIT License 5 votes vote down vote up
def test_decode(self):
        if sys.version_info[0] >= 3:
            A = np.char.array([asbytes('\\u03a3')])
            assert_(A.decode('unicode-escape')[0] == '\u03a3')
        else:
            A = np.char.array(['736563726574206d657373616765'])
            assert_(A.decode('hex_codec')[0] == 'secret message') 
Example 54
Project: LaserTOF   Author: kyleuckert   File: test_defchararray.py    MIT License 5 votes vote down vote up
def test_expandtabs(self):
        T = self.A.expandtabs()
        assert_(T[2, 0] == asbytes('123      345 \0')) 
Example 55
Project: LaserTOF   Author: kyleuckert   File: test_defchararray.py    MIT License 5 votes vote down vote up
def test_rsplit(self):
        A = self.A.rsplit(asbytes('3'))
        tgt = asbytes_nested([[[' abc '], ['']],
                              [['12', '45'], ['MixedCase']],
                              [['12', ' \t ', '45 \x00 '], ['UPPER']]])
        assert_(issubclass(A.dtype.type, np.object_))
        assert_equal(A.tolist(), tgt) 
Example 56
Project: LaserTOF   Author: kyleuckert   File: test_defchararray.py    MIT License 5 votes vote down vote up
def test_split(self):
        A = self.A.split(asbytes('3'))
        tgt = asbytes_nested([
                              [[' abc '], ['']],
                              [['12', '45'], ['MixedCase']],
                              [['12', ' \t ', '45 \x00 '], ['UPPER']]])
        assert_(issubclass(A.dtype.type, np.object_))
        assert_equal(A.tolist(), tgt) 
Example 57
Project: LaserTOF   Author: kyleuckert   File: test_regression.py    MIT License 5 votes vote down vote up
def test_pickle_py2_bytes_encoding(self):
        # Check that arrays and scalars pickled on Py2 are
        # unpickleable on Py3 using encoding='bytes'

        test_data = [
            # (original, py2_pickle)
            (np.unicode_('\u6f2c'),
             asbytes("cnumpy.core.multiarray\nscalar\np0\n(cnumpy\ndtype\np1\n"
                     "(S'U1'\np2\nI0\nI1\ntp3\nRp4\n(I3\nS'<'\np5\nNNNI4\nI4\n"
                     "I0\ntp6\nbS',o\\x00\\x00'\np7\ntp8\nRp9\n.")),

            (np.array([9e123], dtype=np.float64),
             asbytes("cnumpy.core.multiarray\n_reconstruct\np0\n(cnumpy\nndarray\n"
                     "p1\n(I0\ntp2\nS'b'\np3\ntp4\nRp5\n(I1\n(I1\ntp6\ncnumpy\ndtype\n"
                     "p7\n(S'f8'\np8\nI0\nI1\ntp9\nRp10\n(I3\nS'<'\np11\nNNNI-1\nI-1\n"
                     "I0\ntp12\nbI00\nS'O\\x81\\xb7Z\\xaa:\\xabY'\np13\ntp14\nb.")),

            (np.array([(9e123,)], dtype=[('name', float)]),
             asbytes("cnumpy.core.multiarray\n_reconstruct\np0\n(cnumpy\nndarray\np1\n"
                     "(I0\ntp2\nS'b'\np3\ntp4\nRp5\n(I1\n(I1\ntp6\ncnumpy\ndtype\np7\n"
                     "(S'V8'\np8\nI0\nI1\ntp9\nRp10\n(I3\nS'|'\np11\nN(S'name'\np12\ntp13\n"
                     "(dp14\ng12\n(g7\n(S'f8'\np15\nI0\nI1\ntp16\nRp17\n(I3\nS'<'\np18\nNNNI-1\n"
                     "I-1\nI0\ntp19\nbI0\ntp20\nsI8\nI1\nI0\ntp21\n"
                     "bI00\nS'O\\x81\\xb7Z\\xaa:\\xabY'\np22\ntp23\nb.")),
        ]

        if sys.version_info[:2] >= (3, 4):
            # encoding='bytes' was added in Py3.4
            for original, data in test_data:
                result = pickle.loads(data, encoding='bytes')
                assert_equal(result, original)

                if isinstance(result, np.ndarray) and result.dtype.names:
                    for name in result.dtype.names:
                        assert_(isinstance(name, str)) 
Example 58
Project: LaserTOF   Author: kyleuckert   File: test_regression.py    MIT License 5 votes vote down vote up
def test_numeric_carray_compare(self, level=rlevel):
        # Ticket #341
        assert_equal(np.array(['X'], 'c'), asbytes('X')) 
Example 59
Project: LaserTOF   Author: kyleuckert   File: test_regression.py    MIT License 5 votes vote down vote up
def test_junk_in_string_fields_of_recarray(self, level=rlevel):
        # Ticket #483
        r = np.array([[asbytes('abc')]], dtype=[('var1', '|S20')])
        assert_(asbytes(r['var1'][0][0]) == asbytes('abc')) 
Example 60
Project: LaserTOF   Author: kyleuckert   File: test_regression.py    MIT License 5 votes vote down vote up
def test_unaligned_unicode_access(self, level=rlevel):
        # Ticket #825
        for i in range(1, 9):
            msg = 'unicode offset: %d chars' % i
            t = np.dtype([('a', 'S%d' % i), ('b', 'U2')])
            x = np.array([(asbytes('a'), sixu('b'))], dtype=t)
            if sys.version_info[0] >= 3:
                assert_equal(str(x), "[(b'a', 'b')]", err_msg=msg)
            else:
                assert_equal(str(x), "[('a', u'b')]", err_msg=msg) 
Example 61
Project: LaserTOF   Author: kyleuckert   File: test_regression.py    MIT License 5 votes vote down vote up
def test_fromstring_crash(self):
        # Ticket #1345: the following should not cause a crash
        np.fromstring(asbytes('aa, aa, 1.0'), sep=',') 
Example 62
Project: LaserTOF   Author: kyleuckert   File: test_regression.py    MIT License 5 votes vote down vote up
def test_setting_rank0_string(self):
        "Ticket #1736"
        s1 = asbytes("hello1")
        s2 = asbytes("hello2")
        a = np.zeros((), dtype="S10")
        a[()] = s1
        assert_equal(a, np.array(s1))
        a[()] = np.array(s2)
        assert_equal(a, np.array(s2))

        a = np.zeros((), dtype='f4')
        a[()] = 3
        assert_equal(a, np.array(3))
        a[()] = np.array(4)
        assert_equal(a, np.array(4)) 
Example 63
Project: LaserTOF   Author: kyleuckert   File: test_regression.py    MIT License 5 votes vote down vote up
def test_string_astype(self):
        "Ticket #1748"
        s1 = asbytes('black')
        s2 = asbytes('white')
        s3 = asbytes('other')
        a = np.array([[s1], [s2], [s3]])
        assert_equal(a.dtype, np.dtype('S5'))
        b = a.astype(np.dtype('S0'))
        assert_equal(b.dtype, np.dtype('S5')) 
Example 64
Project: LaserTOF   Author: kyleuckert   File: test_regression.py    MIT License 5 votes vote down vote up
def test_ticket_1756(self):
        # Ticket #1756
        s = asbytes('0123456789abcdef')
        a = np.array([s]*5)
        for i in range(1, 17):
            a1 = np.array(a, "|S%d" % i)
            a2 = np.array([s[:i]]*5)
            assert_equal(a1, a2) 
Example 65
Project: LaserTOF   Author: kyleuckert   File: test_regression.py    MIT License 5 votes vote down vote up
def test_pickle_py2_array_latin1_hack(self):
        # Check that unpickling hacks in Py3 that support
        # encoding='latin1' work correctly.

        # Python2 output for pickle.dumps(numpy.array([129], dtype='b'))
        data = asbytes("cnumpy.core.multiarray\n_reconstruct\np0\n(cnumpy\nndarray\np1\n(I0\n"
                       "tp2\nS'b'\np3\ntp4\nRp5\n(I1\n(I1\ntp6\ncnumpy\ndtype\np7\n(S'i1'\np8\n"
                       "I0\nI1\ntp9\nRp10\n(I3\nS'|'\np11\nNNNI-1\nI-1\nI0\ntp12\nbI00\nS'\\x81'\n"
                       "p13\ntp14\nb.")
        if sys.version_info[0] >= 3:
            # This should work:
            result = pickle.loads(data, encoding='latin1')
            assert_array_equal(result, np.array([129], dtype='b'))
            # Should not segfault:
            assert_raises(Exception, pickle.loads, data, encoding='koi8-r') 
Example 66
Project: LaserTOF   Author: kyleuckert   File: defchararray.py    MIT License 5 votes vote down vote up
def center(a, width, fillchar=' '):
    """
    Return a copy of `a` with its elements centered in a string of
    length `width`.

    Calls `str.center` element-wise.

    Parameters
    ----------
    a : array_like of str or unicode

    width : int
        The length of the resulting strings
    fillchar : str or unicode, optional
        The padding character to use (default is space).

    Returns
    -------
    out : ndarray
        Output array of str or unicode, depending on input
        types

    See also
    --------
    str.center

    """
    a_arr = numpy.asarray(a)
    width_arr = numpy.asarray(width)
    size = long(numpy.max(width_arr.flat))
    if numpy.issubdtype(a_arr.dtype, numpy.string_):
        fillchar = asbytes(fillchar)
    return _vec_string(
        a_arr, (a_arr.dtype.type, size), 'center', (width_arr, fillchar)) 
Example 67
Project: LaserTOF   Author: kyleuckert   File: defchararray.py    MIT License 5 votes vote down vote up
def ljust(a, width, fillchar=' '):
    """
    Return an array with the elements of `a` left-justified in a
    string of length `width`.

    Calls `str.ljust` element-wise.

    Parameters
    ----------
    a : array_like of str or unicode

    width : int
        The length of the resulting strings
    fillchar : str or unicode, optional
        The character to use for padding

    Returns
    -------
    out : ndarray
        Output array of str or unicode, depending on input type

    See also
    --------
    str.ljust

    """
    a_arr = numpy.asarray(a)
    width_arr = numpy.asarray(width)
    size = long(numpy.max(width_arr.flat))
    if numpy.issubdtype(a_arr.dtype, numpy.string_):
        fillchar = asbytes(fillchar)
    return _vec_string(
        a_arr, (a_arr.dtype.type, size), 'ljust', (width_arr, fillchar)) 
Example 68
Project: LaserTOF   Author: kyleuckert   File: defchararray.py    MIT License 5 votes vote down vote up
def rjust(a, width, fillchar=' '):
    """
    Return an array with the elements of `a` right-justified in a
    string of length `width`.

    Calls `str.rjust` element-wise.

    Parameters
    ----------
    a : array_like of str or unicode

    width : int
        The length of the resulting strings
    fillchar : str or unicode, optional
        The character to use for padding

    Returns
    -------
    out : ndarray
        Output array of str or unicode, depending on input type

    See also
    --------
    str.rjust

    """
    a_arr = numpy.asarray(a)
    width_arr = numpy.asarray(width)
    size = long(numpy.max(width_arr.flat))
    if numpy.issubdtype(a_arr.dtype, numpy.string_):
        fillchar = asbytes(fillchar)
    return _vec_string(
        a_arr, (a_arr.dtype.type, size), 'rjust', (width_arr, fillchar)) 
Example 69
Project: LaserTOF   Author: kyleuckert   File: test_build.py    MIT License 5 votes vote down vote up
def grep_dependencies(self, file, deps):
        stdout = self.get_dependencies(file)

        rdeps = dict([(asbytes(dep), re.compile(asbytes(dep))) for dep in deps])
        founds = []
        for l in stdout.splitlines():
            for k, v in rdeps.items():
                if v.search(l):
                    founds.append(k)

        return founds 
Example 70
Project: LaserTOF   Author: kyleuckert   File: netcdf.py    MIT License 5 votes vote down vote up
def _pack_string(self, s):
        count = len(s)
        self._pack_int(count)
        self.fp.write(asbytes(s))
        self.fp.write(b'0' * (-count % 4))  # pad 
Example 71
Project: LaserTOF   Author: kyleuckert   File: mio5.py    MIT License 5 votes vote down vote up
def put_variables(self, mdict, write_header=None):
        ''' Write variables in `mdict` to stream

        Parameters
        ----------
        mdict : mapping
           mapping with method ``items`` returns name, contents pairs where
           ``name`` which will appear in the matlab workspace in file load, and
           ``contents`` is something writeable to a matlab file, such as a numpy
           array.
        write_header : {None, True, False}, optional
           If True, then write the matlab file header before writing the
           variables.  If None (the default) then write the file header
           if we are at position 0 in the stream.  By setting False
           here, and setting the stream position to the end of the file,
           you can append variables to a matlab file
        '''
        # write header if requested, or None and start of file
        if write_header is None:
            write_header = self.file_stream.tell() == 0
        if write_header:
            self.write_file_header()
        self._matrix_writer = VarWriter5(self)
        for name, var in mdict.items():
            if name[0] == '_':
                continue
            is_global = name in self.global_vars
            if self.do_compression:
                stream = BytesIO()
                self._matrix_writer.file_stream = stream
                self._matrix_writer.write_top(var, asbytes(name), is_global)
                out_str = zlib.compress(stream.getvalue())
                tag = np.empty((), NDT_TAG_FULL)
                tag['mdtype'] = miCOMPRESSED
                tag['byte_count'] = len(out_str)
                self.file_stream.write(tag.tostring())
                self.file_stream.write(out_str)
            else:  # not compressing
                self._matrix_writer.write_top(var, asbytes(name), is_global) 
Example 72
Project: LaserTOF   Author: kyleuckert   File: mio4.py    MIT License 5 votes vote down vote up
def write_header(self, name, shape, P=miDOUBLE, T=mxFULL_CLASS, imagf=0):
        ''' Write header for given data options

        Parameters
        ----------
        name : str
            name of variable
        shape : sequence
           Shape of array as it will be read in matlab
        P : int, optional
            code for mat4 data type, one of ``miDOUBLE, miSINGLE, miINT32,
            miINT16, miUINT16, miUINT8``
        T : int, optional
            code for mat4 matrix class, one of ``mxFULL_CLASS, mxCHAR_CLASS,
            mxSPARSE_CLASS``
        imagf : int, optional
            flag indicating complex
        '''
        header = np.empty((), mdtypes_template['header'])
        M = not SYS_LITTLE_ENDIAN
        O = 0
        header['mopt'] = (M * 1000 +
                          O * 100 +
                          P * 10 +
                          T)
        header['mrows'] = shape[0]
        header['ncols'] = shape[1]
        header['imagf'] = imagf
        header['namlen'] = len(name) + 1
        self.write_bytes(header)
        self.write_string(asbytes(name + '\0')) 
Example 73
Project: FX-RER-Value-Extraction   Author: tsKenneth   File: test_io.py    MIT License 5 votes vote down vote up
def __init__(self, s=""):
        BytesIO.__init__(self, asbytes(s)) 
Example 74
Project: LaserTOF   Author: kyleuckert   File: format.py    MIT License 4 votes vote down vote up
def _write_array_header(fp, d, version=None):
    """ Write the header for an array and returns the version used

    Parameters
    ----------
    fp : filelike object
    d : dict
        This has the appropriate entries for writing its string representation
        to the header of the file.
    version: tuple or None
        None means use oldest that works
        explicit version will raise a ValueError if the format does not
        allow saving this data.  Default: None
    Returns
    -------
    version : tuple of int
        the file version which needs to be used to store the data
    """
    import struct
    header = ["{"]
    for key, value in sorted(d.items()):
        # Need to use repr here, since we eval these when reading
        header.append("'%s': %s, " % (key, repr(value)))
    header.append("}")
    header = "".join(header)
    # Pad the header with spaces and a final newline such that the magic
    # string, the header-length short and the header are aligned on a
    # 16-byte boundary.  Hopefully, some system, possibly memory-mapping,
    # can take advantage of our premature optimization.
    current_header_len = MAGIC_LEN + 2 + len(header) + 1  # 1 for the newline
    topad = 16 - (current_header_len % 16)
    header = header + ' '*topad + '\n'
    header = asbytes(_filter_header(header))

    hlen = len(header)
    if hlen < 256*256 and version in (None, (1, 0)):
        version = (1, 0)
        header_prefix = magic(1, 0) + struct.pack('<H', hlen)
    elif hlen < 2**32 and version in (None, (2, 0)):
        version = (2, 0)
        header_prefix = magic(2, 0) + struct.pack('<I', hlen)
    else:
        msg = "Header length %s too big for version=%s"
        msg %= (hlen, version)
        raise ValueError(msg)

    fp.write(header_prefix)
    fp.write(header)
    return version 
Example 75
Project: LaserTOF   Author: kyleuckert   File: util.py    MIT License 4 votes vote down vote up
def _get_compiler_status():
    global _compiler_status
    if _compiler_status is not None:
        return _compiler_status

    _compiler_status = (False, False, False)

    # XXX: this is really ugly. But I don't know how to invoke Distutils
    #      in a safer way...
    code = """
import os
import sys
sys.path = %(syspath)s

def configuration(parent_name='',top_path=None):
    global config
    from numpy.distutils.misc_util import Configuration
    config = Configuration('', parent_name, top_path)
    return config

from numpy.distutils.core import setup
setup(configuration=configuration)

config_cmd = config.get_config_cmd()
have_c = config_cmd.try_compile('void foo() {}')
print('COMPILERS:%%d,%%d,%%d' %% (have_c,
                                  config.have_f77c(),
                                  config.have_f90c()))
sys.exit(99)
"""
    code = code % dict(syspath=repr(sys.path))

    with temppath(suffix='.py') as script:
        with open(script, 'w') as f:
            f.write(code)

        cmd = [sys.executable, script, 'config']
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                             stderr=subprocess.STDOUT)
        out, err = p.communicate()

    m = re.search(asbytes(r'COMPILERS:(\d+),(\d+),(\d+)'), out)
    if m:
        _compiler_status = (bool(int(m.group(1))), bool(int(m.group(2))),
                            bool(int(m.group(3))))
    # Finished
    return _compiler_status 
Example 76
Project: LaserTOF   Author: kyleuckert   File: test_mrecords.py    MIT License 4 votes vote down vote up
def test_set_fields(self):
        # Tests setting fields.
        base = self.base.copy()
        mbase = base.view(mrecarray)
        mbase = mbase.copy()
        mbase.fill_value = (999999, 1e20, 'N/A')
        # Change the data, the mask should be conserved
        mbase.a._data[:] = 5
        assert_equal(mbase['a']._data, [5, 5, 5, 5, 5])
        assert_equal(mbase['a']._mask, [0, 1, 0, 0, 1])
        # Change the elements, and the mask will follow
        mbase.a = 1
        assert_equal(mbase['a']._data, [1]*5)
        assert_equal(ma.getmaskarray(mbase['a']), [0]*5)
        # Use to be _mask, now it's recordmask
        assert_equal(mbase.recordmask, [False]*5)
        assert_equal(mbase._mask.tolist(),
                     np.array([(0, 0, 0),
                               (0, 1, 1),
                               (0, 0, 0),
                               (0, 0, 0),
                               (0, 1, 1)],
                              dtype=bool))
        # Set a field to mask ........................
        mbase.c = masked
        # Use to be mask, and now it's still mask !
        assert_equal(mbase.c.mask, [1]*5)
        assert_equal(mbase.c.recordmask, [1]*5)
        assert_equal(ma.getmaskarray(mbase['c']), [1]*5)
        assert_equal(ma.getdata(mbase['c']), [asbytes('N/A')]*5)
        assert_equal(mbase._mask.tolist(),
                     np.array([(0, 0, 1),
                               (0, 1, 1),
                               (0, 0, 1),
                               (0, 0, 1),
                               (0, 1, 1)],
                              dtype=bool))
        # Set fields by slices .......................
        mbase = base.view(mrecarray).copy()
        mbase.a[3:] = 5
        assert_equal(mbase.a, [1, 2, 3, 5, 5])
        assert_equal(mbase.a._mask, [0, 1, 0, 0, 0])
        mbase.b[3:] = masked
        assert_equal(mbase.b, base['b'])
        assert_equal(mbase.b._mask, [0, 1, 0, 1, 1])
        # Set fields globally..........................
        ndtype = [('alpha', '|S1'), ('num', int)]
        data = ma.array([('a', 1), ('b', 2), ('c', 3)], dtype=ndtype)
        rdata = data.view(MaskedRecords)
        val = ma.array([10, 20, 30], mask=[1, 0, 0])

        rdata['num'] = val
        assert_equal(rdata.num, val)
        assert_equal(rdata.num.mask, [1, 0, 0]) 
Example 77
Project: LaserTOF   Author: kyleuckert   File: test_core.py    MIT License 4 votes vote down vote up
def test_check_on_fields(self):
        # Tests _check_fill_value with records
        _check_fill_value = np.ma.core._check_fill_value
        ndtype = [('a', int), ('b', float), ('c', "|S3")]
        # A check on a list should return a single record
        fval = _check_fill_value([-999, -12345678.9, "???"], ndtype)
        self.assertTrue(isinstance(fval, ndarray))
        assert_equal(fval.item(), [-999, -12345678.9, asbytes("???")])
        # A check on None should output the defaults
        fval = _check_fill_value(None, ndtype)
        self.assertTrue(isinstance(fval, ndarray))
        assert_equal(fval.item(), [default_fill_value(0),
                                   default_fill_value(0.),
                                   asbytes(default_fill_value("0"))])
        #.....Using a structured type as fill_value should work
        fill_val = np.array((-999, -12345678.9, "???"), dtype=ndtype)
        fval = _check_fill_value(fill_val, ndtype)
        self.assertTrue(isinstance(fval, ndarray))
        assert_equal(fval.item(), [-999, -12345678.9, asbytes("???")])

        #.....Using a flexible type w/ a different type shouldn't matter
        # BEHAVIOR in 1.5 and earlier: match structured types by position
        #fill_val = np.array((-999, -12345678.9, "???"),
        #                    dtype=[("A", int), ("B", float), ("C", "|S3")])
        # BEHAVIOR in 1.6 and later: match structured types by name
        fill_val = np.array(("???", -999, -12345678.9),
                            dtype=[("c", "|S3"), ("a", int), ("b", float), ])
        # suppress deprecation warning in 1.12 (remove in 1.13)
        with assert_warns(FutureWarning):
            fval = _check_fill_value(fill_val, ndtype)
        self.assertTrue(isinstance(fval, ndarray))
        assert_equal(fval.item(), [-999, -12345678.9, asbytes("???")])

        #.....Using an object-array shouldn't matter either
        fill_val = np.ndarray(shape=(1,), dtype=object)
        fill_val[0] = (-999, -12345678.9, asbytes("???"))
        fval = _check_fill_value(fill_val, object)
        self.assertTrue(isinstance(fval, ndarray))
        assert_equal(fval.item(), [-999, -12345678.9, asbytes("???")])
        # NOTE: This test was never run properly as "fill_value" rather than
        # "fill_val" was assigned.  Written properly, it fails.
        #fill_val = np.array((-999, -12345678.9, "???"))
        #fval = _check_fill_value(fill_val, ndtype)
        #self.assertTrue(isinstance(fval, ndarray))
        #assert_equal(fval.item(), [-999, -12345678.9, asbytes("???")])
        #.....One-field-only flexible type should work as well
        ndtype = [("a", int)]
        fval = _check_fill_value(-999999999, ndtype)
        self.assertTrue(isinstance(fval, ndarray))
        assert_equal(fval.item(), (-999999999,)) 
Example 78
Project: LaserTOF   Author: kyleuckert   File: _internal.py    MIT License 4 votes vote down vote up
def _commastring(astr):
    startindex = 0
    result = []
    while startindex < len(astr):
        mo = format_re.match(astr, pos=startindex)
        try:
            (order1, repeats, order2, dtype) = mo.groups()
        except (TypeError, AttributeError):
            raise ValueError('format number %d of "%s" is not recognized' %
                                            (len(result)+1, astr))
        startindex = mo.end()
        # Separator or ending padding
        if startindex < len(astr):
            if space_re.match(astr, pos=startindex):
                startindex = len(astr)
            else:
                mo = sep_re.match(astr, pos=startindex)
                if not mo:
                    raise ValueError(
                        'format number %d of "%s" is not recognized' %
                        (len(result)+1, astr))
                startindex = mo.end()

        if order2 == asbytes(''):
            order = order1
        elif order1 == asbytes(''):
            order = order2
        else:
            order1 = _convorder.get(order1, order1)
            order2 = _convorder.get(order2, order2)
            if (order1 != order2):
                raise ValueError(
                    'inconsistent byte-order specification %s and %s' %
                    (order1, order2))
            order = order1

        if order in [asbytes('|'), asbytes('='), _nbo]:
            order = asbytes('')
        dtype = order + dtype
        if (repeats == asbytes('')):
            newitem = dtype
        else:
            newitem = (dtype, eval(repeats))
        result.append(newitem)

    return result 
Example 79
Project: LaserTOF   Author: kyleuckert   File: test_regression.py    MIT License 4 votes vote down vote up
def test_pickle_py2_scalar_latin1_hack(self):
        # Check that scalar unpickling hack in Py3 that supports
        # encoding='latin1' work correctly.

        # Python2 output for pickle.dumps(...)
        datas = [
            # (original, python2_pickle, koi8r_validity)
            (np.unicode_('\u6bd2'),
             asbytes("cnumpy.core.multiarray\nscalar\np0\n(cnumpy\ndtype\np1\n"
                     "(S'U1'\np2\nI0\nI1\ntp3\nRp4\n(I3\nS'<'\np5\nNNNI4\nI4\nI0\n"
                     "tp6\nbS'\\xd2k\\x00\\x00'\np7\ntp8\nRp9\n."),
             'invalid'),

            (np.float64(9e123),
             asbytes("cnumpy.core.multiarray\nscalar\np0\n(cnumpy\ndtype\np1\n(S'f8'\n"
                     "p2\nI0\nI1\ntp3\nRp4\n(I3\nS'<'\np5\nNNNI-1\nI-1\nI0\ntp6\n"
                     "bS'O\\x81\\xb7Z\\xaa:\\xabY'\np7\ntp8\nRp9\n."),
             'invalid'),

            (np.bytes_(asbytes('\x9c')),  # different 8-bit code point in KOI8-R vs latin1
             asbytes("cnumpy.core.multiarray\nscalar\np0\n(cnumpy\ndtype\np1\n(S'S1'\np2\n"
                     "I0\nI1\ntp3\nRp4\n(I3\nS'|'\np5\nNNNI1\nI1\nI0\ntp6\nbS'\\x9c'\np7\n"
                     "tp8\nRp9\n."),
             'different'),
        ]
        if sys.version_info[0] >= 3:
            for original, data, koi8r_validity in datas:
                result = pickle.loads(data, encoding='latin1')
                assert_equal(result, original)

                # Decoding under non-latin1 encoding (e.g.) KOI8-R can
                # produce bad results, but should not segfault.
                if koi8r_validity == 'different':
                    # Unicode code points happen to lie within latin1,
                    # but are different in koi8-r, resulting to silent
                    # bogus results
                    result = pickle.loads(data, encoding='koi8-r')
                    assert_(result != original)
                elif koi8r_validity == 'invalid':
                    # Unicode code points outside latin1, so results
                    # to an encoding exception
                    assert_raises(ValueError, pickle.loads, data, encoding='koi8-r')
                else:
                    raise ValueError(koi8r_validity) 
Example 80
Project: LaserTOF   Author: kyleuckert   File: netcdf.py    MIT License 4 votes vote down vote up
def _write_values(self, values):
        if hasattr(values, 'dtype'):
            nc_type = REVERSE[values.dtype.char, values.dtype.itemsize]
        else:
            types = [(t, NC_INT) for t in integer_types]
            types += [
                    (float, NC_FLOAT),
                    (str, NC_CHAR)
                    ]
            # bytes index into scalars in py3k.  Check for "string" types
            if isinstance(values, text_type) or isinstance(values, binary_type):
                sample = values
            else:
                try:
                    sample = values[0]  # subscriptable?
                except TypeError:
                    sample = values     # scalar

            for class_, nc_type in types:
                if isinstance(sample, class_):
                    break

        typecode, size = TYPEMAP[nc_type]
        dtype_ = '>%s' % typecode
        # asarray() dies with bytes and '>c' in py3k.  Change to 'S'
        dtype_ = 'S' if dtype_ == '>c' else dtype_

        values = asarray(values, dtype=dtype_)

        self.fp.write(asbytes(nc_type))

        if values.dtype.char == 'S':
            nelems = values.itemsize
        else:
            nelems = values.size
        self._pack_int(nelems)

        if not values.shape and (values.dtype.byteorder == '<' or
                (values.dtype.byteorder == '=' and LITTLE_ENDIAN)):
            values = values.byteswap()
        self.fp.write(values.tostring())
        count = values.size * values.itemsize
        self.fp.write(b'0' * (-count % 4))  # pad