Python codecs.utf_16_be_decode() Examples

The following are 30 code examples of codecs.utf_16_be_decode(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module codecs , or try the search function

Example #1

Source File: test_codecs.py From Fluid-Designer with GNU General Public License v3.0

6 votes

def test_decode_unicode(self):
        # Most decoders don't accept unicode input
        decoders = [
            codecs.utf_7_decode,
            codecs.utf_8_decode,
            codecs.utf_16_le_decode,
            codecs.utf_16_be_decode,
            codecs.utf_16_ex_decode,
            codecs.utf_32_decode,
            codecs.utf_32_le_decode,
            codecs.utf_32_be_decode,
            codecs.utf_32_ex_decode,
            codecs.latin_1_decode,
            codecs.ascii_decode,
            codecs.charmap_decode,
        ]
        if hasattr(codecs, "mbcs_decode"):
            decoders.append(codecs.mbcs_decode)
        for decoder in decoders:
            self.assertRaises(TypeError, decoder, "xxx")

Example #2

Source File: test_codecs.py From ironpython3 with Apache License 2.0

6 votes

def test_decode_unicode(self):
        # Most decoders don't accept unicode input
        decoders = [
            codecs.utf_7_decode,
            codecs.utf_8_decode,
            codecs.utf_16_le_decode,
            codecs.utf_16_be_decode,
            codecs.utf_16_ex_decode,
            codecs.utf_32_decode,
            codecs.utf_32_le_decode,
            codecs.utf_32_be_decode,
            codecs.utf_32_ex_decode,
            codecs.latin_1_decode,
            codecs.ascii_decode,
            codecs.charmap_decode,
        ]
        if hasattr(codecs, "mbcs_decode"):
            decoders.append(codecs.mbcs_decode)
        for decoder in decoders:
            self.assertRaises(TypeError, decoder, "xxx")

Example #3

Source File: test_codecs.py From ironpython2 with Apache License 2.0

6 votes

def test_errors(self):
        tests = [
            (b'\xff', u'\ufffd'),
            (b'\x00A\xff', u'A\ufffd'),
            (b'\x00A\x00B\x00C\x00DZ', u'ABCD\ufffd'),
            (b'\xd8\x00', u'\ufffd'),
            (b'\xd8\x00\xdc', u'\ufffd'),
            (b'\xd8\x00\x00A', u'\ufffdA'),
            (b'\xdc\x00\x00A', u'\ufffdA'),
        ]
        for raw, expected in tests:
            try:
                with self.assertRaises(UnicodeDecodeError):
                    codecs.utf_16_be_decode(raw, 'strict', True)
                self.assertEqual(raw.decode('utf-16be', 'replace'), expected)
            except:
                print 'raw=%r' % raw
                raise

Example #4

Source File: utf_16.py From ironpython3 with Apache License 2.0

5 votes

def setstate(self, state):
        # state[1] will be ignored by BufferedIncrementalDecoder.setstate()
        codecs.BufferedIncrementalDecoder.setstate(self, state)
        state = state[1]
        if state == 0:
            self.decoder = (codecs.utf_16_be_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_le_decode)
        elif state == 1:
            self.decoder = (codecs.utf_16_le_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_be_decode)
        else:
            self.decoder = None

Example #5

Source File: utf_16.py From Imogen with MIT License

5 votes

def setstate(self, state):
        # state[1] will be ignored by BufferedIncrementalDecoder.setstate()
        codecs.BufferedIncrementalDecoder.setstate(self, state)
        state = state[1]
        if state == 0:
            self.decoder = (codecs.utf_16_be_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_le_decode)
        elif state == 1:
            self.decoder = (codecs.utf_16_le_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_be_decode)
        else:
            self.decoder = None

Example #6

Source File: utf_16.py From Imogen with MIT License

5 votes

def decode(self, input, errors='strict'):
        (object, consumed, byteorder) = \
            codecs.utf_16_ex_decode(input, errors, 0, False)
        if byteorder == -1:
            self.decode = codecs.utf_16_le_decode
        elif byteorder == 1:
            self.decode = codecs.utf_16_be_decode
        elif consumed>=2:
            raise UnicodeError("UTF-16 stream does not start with BOM")
        return (object, consumed)

### encodings module API

Example #7

Source File: utf_16_be.py From Imogen with MIT License

5 votes

def decode(input, errors='strict'):
    return codecs.utf_16_be_decode(input, errors, True)

Example #8

Source File: utf_16_be.py From scylla with Apache License 2.0

5 votes

def decode(input, errors='strict'):
    return codecs.utf_16_be_decode(input, errors, True)

Example #9

Source File: utf_16.py From scylla with Apache License 2.0

5 votes

def _buffer_decode(self, input, errors, final):
        if self.decoder is None:
            (output, consumed, byteorder) = \
                codecs.utf_16_ex_decode(input, errors, 0, final)
            if byteorder == -1:
                self.decoder = codecs.utf_16_le_decode
            elif byteorder == 1:
                self.decoder = codecs.utf_16_be_decode
            elif consumed >= 2:
                raise UnicodeError("UTF-16 stream does not start with BOM")
            return (output, consumed)
        return self.decoder(input, self.errors, final)

Example #10

Source File: test_codecs.py From ironpython3 with Apache License 2.0

5 votes

def test_utf_16_be_decode_incremental(self):
        b = b"\xff\xfe\x00\x41\xd9\x00\xdd\x00\xdc\x00\xd8\x00\xdc\x00"
        expected = [
            ('', 0),
            ('', 0),
            ('\ufffe', 2),
            ('\ufffe', 2),
            ('\ufffeA', 4),
            ('\ufffeA', 4),
            ('\ufffeA', 4),
            ('\ufffeA', 4),
            ('\ufffeA\U00050100', 8),
            ('\ufffeA\U00050100', 8),
            ('\ufffeA\U00050100\ufffd', 10),
            ('\ufffeA\U00050100\ufffd', 10),
            ('\ufffeA\U00050100\ufffd', 10),
            ('\ufffeA\U00050100\ufffd', 10),
            ('\ufffeA\U00050100\ufffd\U00010000', 14)
        ]
        if not is_cli:
            # CPython's strings are UTF-32 so an invalid surrogate pair results in one replacement char.
            # Therefore CPython cannot report error on a dangling low surrogate until it verifies
            # that the next char is not an invalid surrogate as well.
            expected[10] = expected[11] = ('\ufffeA\U00050100', 8)

        for i in range(len(b) + 1):
            res = codecs.utf_16_be_decode(b[:i], 'replace')
            self.assertEqual(res, expected[i])

        self.assertRaises(UnicodeDecodeError, codecs.utf_16_be_decode, b"\x41\x00\xd8\x00\xd8\x00", 'strict', False)

Example #11

Source File: utf_16.py From Imogen with MIT License

5 votes

def _buffer_decode(self, input, errors, final):
        if self.decoder is None:
            (output, consumed, byteorder) = \
                codecs.utf_16_ex_decode(input, errors, 0, final)
            if byteorder == -1:
                self.decoder = codecs.utf_16_le_decode
            elif byteorder == 1:
                self.decoder = codecs.utf_16_be_decode
            elif consumed >= 2:
                raise UnicodeError("UTF-16 stream does not start with BOM")
            return (output, consumed)
        return self.decoder(input, self.errors, final)

Example #12

Source File: utf_16.py From ironpython3 with Apache License 2.0

5 votes

def decode(self, input, errors='strict'):
        (object, consumed, byteorder) = \
            codecs.utf_16_ex_decode(input, errors, 0, False)
        if byteorder == -1:
            self.decode = codecs.utf_16_le_decode
        elif byteorder == 1:
            self.decode = codecs.utf_16_be_decode
        elif consumed>=2:
            raise UnicodeError("UTF-16 stream does not start with BOM")
        return (object, consumed)

### encodings module API

Example #13

Source File: utf_16_be.py From ironpython3 with Apache License 2.0

5 votes

def decode(input, errors='strict'):
    return codecs.utf_16_be_decode(input, errors, True)

Example #14

Source File: test_codecs.py From ironpython3 with Apache License 2.0

5 votes

def test_utf_16_be_decode(self):
        string, num_processed = codecs.utf_16_be_decode(b'\0a\0b\0c')
        self.assertEqual(string, "abc")
        self.assertEqual(num_processed, 3 * 2)

        string, num_processed = codecs.utf_16_be_decode(codecs.BOM_UTF16_BE + b'\0a\0b\0c')
        self.assertEqual(string, "\uFEFFabc")
        self.assertEqual(num_processed, 4 * 2)

        self.assertEqual(codecs.utf_16_be_decode(array.array('I', (1633771873,))), ("慡慡", 4))

        self.assertRaises(TypeError, codecs.utf_16_be_decode, "abc")
        self.assertRaises(TypeError, codecs.utf_16_be_decode, None)
        self.assertRaises(TypeError, codecs.utf_16_be_decode, None, None)
        self.assertEquals(codecs.utf_16_be_decode(b"", None), ("", 0))

Example #15

Source File: utf_16.py From python with Apache License 2.0

5 votes

def getstate(self):
        # additional state info from the base class must be None here,
        # as it isn't passed along to the caller
        state = codecs.BufferedIncrementalDecoder.getstate(self)[0]
        # additional state info we pass to the caller:
        # 0: stream is in natural order for this platform
        # 1: stream is in unnatural order
        # 2: endianness hasn't been determined yet
        if self.decoder is None:
            return (state, 2)
        addstate = int((sys.byteorder == "big") !=
                       (self.decoder is codecs.utf_16_be_decode))
        return (state, addstate)

Example #16

Source File: utf_16.py From python with Apache License 2.0

5 votes

def decode(self, input, errors='strict'):
        (object, consumed, byteorder) = \
            codecs.utf_16_ex_decode(input, errors, 0, False)
        if byteorder == -1:
            self.decode = codecs.utf_16_le_decode
        elif byteorder == 1:
            self.decode = codecs.utf_16_be_decode
        elif consumed>=2:
            raise UnicodeError("UTF-16 stream does not start with BOM")
        return (object, consumed)

### encodings module API

Example #17

Source File: utf_16.py From python with Apache License 2.0

5 votes

def setstate(self, state):
        # state[1] will be ignored by BufferedIncrementalDecoder.setstate()
        codecs.BufferedIncrementalDecoder.setstate(self, state)
        state = state[1]
        if state == 0:
            self.decoder = (codecs.utf_16_be_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_le_decode)
        elif state == 1:
            self.decoder = (codecs.utf_16_le_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_be_decode)
        else:
            self.decoder = None

Example #18

Source File: utf_16.py From Imogen with MIT License

5 votes

def getstate(self):
        # additional state info from the base class must be None here,
        # as it isn't passed along to the caller
        state = codecs.BufferedIncrementalDecoder.getstate(self)[0]
        # additional state info we pass to the caller:
        # 0: stream is in natural order for this platform
        # 1: stream is in unnatural order
        # 2: endianness hasn't been determined yet
        if self.decoder is None:
            return (state, 2)
        addstate = int((sys.byteorder == "big") !=
                       (self.decoder is codecs.utf_16_be_decode))
        return (state, addstate)

Example #19

Source File: utf_16.py From scylla with Apache License 2.0

5 votes

def setstate(self, state):
        # state[1] will be ignored by BufferedIncrementalDecoder.setstate()
        codecs.BufferedIncrementalDecoder.setstate(self, state)
        state = state[1]
        if state == 0:
            self.decoder = (codecs.utf_16_be_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_le_decode)
        elif state == 1:
            self.decoder = (codecs.utf_16_le_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_be_decode)
        else:
            self.decoder = None

Example #20

Source File: codec.py From naz with MIT License

5 votes

def decode(input: bytes, errors: str = "strict") -> typing.Tuple[str, int]:
        """
        return a string decoded from the given bytes and its length.

        Parameters:
            input: the bytes to decode
            errors:	same meaning as the errors argument to pythons' `encode <https://docs.python.org/3/library/codecs.html#codecs.encode>`_ method
        """
        return codecs.utf_16_be_decode(input, errors)

Example #21

Source File: utf_16_be.py From Fluid-Designer with GNU General Public License v3.0

5 votes

def decode(input, errors='strict'):
    return codecs.utf_16_be_decode(input, errors, True)

Example #22

Source File: utf_16.py From Fluid-Designer with GNU General Public License v3.0

5 votes

def decode(self, input, errors='strict'):
        (object, consumed, byteorder) = \
            codecs.utf_16_ex_decode(input, errors, 0, False)
        if byteorder == -1:
            self.decode = codecs.utf_16_le_decode
        elif byteorder == 1:
            self.decode = codecs.utf_16_be_decode
        elif consumed>=2:
            raise UnicodeError("UTF-16 stream does not start with BOM")
        return (object, consumed)

### encodings module API

Example #23

Source File: utf_16.py From Fluid-Designer with GNU General Public License v3.0

5 votes

def setstate(self, state):
        # state[1] will be ignored by BufferedIncrementalDecoder.setstate()
        codecs.BufferedIncrementalDecoder.setstate(self, state)
        state = state[1]
        if state == 0:
            self.decoder = (codecs.utf_16_be_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_le_decode)
        elif state == 1:
            self.decoder = (codecs.utf_16_le_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_be_decode)
        else:
            self.decoder = None

Example #24

Source File: utf_16.py From Fluid-Designer with GNU General Public License v3.0

5 votes

def getstate(self):
        # additonal state info from the base class must be None here,
        # as it isn't passed along to the caller
        state = codecs.BufferedIncrementalDecoder.getstate(self)[0]
        # additional state info we pass to the caller:
        # 0: stream is in natural order for this platform
        # 1: stream is in unnatural order
        # 2: endianness hasn't been determined yet
        if self.decoder is None:
            return (state, 2)
        addstate = int((sys.byteorder == "big") !=
                       (self.decoder is codecs.utf_16_be_decode))
        return (state, addstate)

Example #25

Source File: utf_16.py From Fluid-Designer with GNU General Public License v3.0

5 votes

def _buffer_decode(self, input, errors, final):
        if self.decoder is None:
            (output, consumed, byteorder) = \
                codecs.utf_16_ex_decode(input, errors, 0, final)
            if byteorder == -1:
                self.decoder = codecs.utf_16_le_decode
            elif byteorder == 1:
                self.decoder = codecs.utf_16_be_decode
            elif consumed >= 2:
                raise UnicodeError("UTF-16 stream does not start with BOM")
            return (output, consumed)
        return self.decoder(input, self.errors, final)

Example #26

Source File: utf_16_be.py From GraphicDesignPatternByPython with MIT License

5 votes

def decode(input, errors='strict'):
    return codecs.utf_16_be_decode(input, errors, True)

Example #27

Source File: utf_16.py From GraphicDesignPatternByPython with MIT License

5 votes

def decode(self, input, errors='strict'):
        (object, consumed, byteorder) = \
            codecs.utf_16_ex_decode(input, errors, 0, False)
        if byteorder == -1:
            self.decode = codecs.utf_16_le_decode
        elif byteorder == 1:
            self.decode = codecs.utf_16_be_decode
        elif consumed>=2:
            raise UnicodeError("UTF-16 stream does not start with BOM")
        return (object, consumed)

### encodings module API

Example #28

Source File: utf_16.py From GraphicDesignPatternByPython with MIT License

5 votes

def setstate(self, state):
        # state[1] will be ignored by BufferedIncrementalDecoder.setstate()
        codecs.BufferedIncrementalDecoder.setstate(self, state)
        state = state[1]
        if state == 0:
            self.decoder = (codecs.utf_16_be_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_le_decode)
        elif state == 1:
            self.decoder = (codecs.utf_16_le_decode
                            if sys.byteorder == "big"
                            else codecs.utf_16_be_decode)
        else:
            self.decoder = None

Example #29

Source File: utf_16.py From GraphicDesignPatternByPython with MIT License

5 votes

def getstate(self):
        # additional state info from the base class must be None here,
        # as it isn't passed along to the caller
        state = codecs.BufferedIncrementalDecoder.getstate(self)[0]
        # additional state info we pass to the caller:
        # 0: stream is in natural order for this platform
        # 1: stream is in unnatural order
        # 2: endianness hasn't been determined yet
        if self.decoder is None:
            return (state, 2)
        addstate = int((sys.byteorder == "big") !=
                       (self.decoder is codecs.utf_16_be_decode))
        return (state, addstate)

Example #30

Source File: utf_16.py From GraphicDesignPatternByPython with MIT License

5 votes

def _buffer_decode(self, input, errors, final):
        if self.decoder is None:
            (output, consumed, byteorder) = \
                codecs.utf_16_ex_decode(input, errors, 0, final)
            if byteorder == -1:
                self.decoder = codecs.utf_16_le_decode
            elif byteorder == 1:
                self.decoder = codecs.utf_16_be_decode
            elif consumed >= 2:
                raise UnicodeError("UTF-16 stream does not start with BOM")
            return (output, consumed)
        return self.decoder(input, self.errors, final)