Python pickle.loads() Examples

def test_struct(self):
        X.init_called = 0

        x = X()
        x.a = 42
        self.assertEqual(X.init_called, 1)

        y = self.loads(self.dumps(x))

        # loads must NOT call __init__
        self.assertEqual(X.init_called, 1)

        # ctypes instances are identical when the instance __dict__
        # and the memory buffer are identical
        self.assertEqual(y.__dict__, x.__dict__)
        self.assertEqual(memoryview(y).tobytes(),
                             memoryview(x).tobytes()) 

def ListTableColumns(self, table):
        """Return a list of columns in the given table.
        [] if the table doesn't exist.
        """
        assert isinstance(table, str)
        if contains_metastrings(table):
            raise ValueError, "bad table name: contains reserved metastrings"

        columnlist_key = _columns_key(table)
        if not getattr(self.db, "has_key")(columnlist_key):
            return []
        pickledcolumnlist = getattr(self.db, "get_bytes",
                self.db.get)(columnlist_key)
        if pickledcolumnlist:
            return pickle.loads(pickledcolumnlist)
        else:
            return [] 

def test_pickling_subclass(self):
        # Make sure we can pickle/unpickle an instance of a subclass.
        offset = timedelta(minutes=-300)
        orig = PicklableFixedOffset(offset, 'cookie')
        self.assertIsInstance(orig, tzinfo)
        self.assertTrue(type(orig) is PicklableFixedOffset)
        self.assertEqual(orig.utcoffset(None), offset)
        self.assertEqual(orig.tzname(None), 'cookie')
        for pickler, unpickler, proto in pickle_choices:
            green = pickler.dumps(orig, proto)
            derived = unpickler.loads(green)
            self.assertIsInstance(derived, tzinfo)
            self.assertTrue(type(derived) is PicklableFixedOffset)
            self.assertEqual(derived.utcoffset(None), offset)
            self.assertEqual(derived.tzname(None), 'cookie')

#############################################################################
# Base class for testing a particular aspect of timedelta, time, date and
# datetime comparisons. 

def test_pickling(self):
        # Try one without a tzinfo.
        args = 6, 7, 23, 20, 59, 1, 64**2
        orig = self.theclass(*args)
        for pickler, unpickler, proto in pickle_choices:
            green = pickler.dumps(orig, proto)
            derived = unpickler.loads(green)
            self.assertEqual(orig, derived)

        # Try one with a tzinfo.
        tinfo = PicklableFixedOffset(-300, 'cookie')
        orig = self.theclass(*args, **{'tzinfo': tinfo})
        derived = self.theclass(1, 1, 1, tzinfo=FixedOffset(0, "", 0))
        for pickler, unpickler, proto in pickle_choices:
            green = pickler.dumps(orig, proto)
            derived = unpickler.loads(green)
            self.assertEqual(orig, derived)
            self.assertIsInstance(derived.tzinfo, PicklableFixedOffset)
            self.assertEqual(derived.utcoffset(), timedelta(minutes=-300))
            self.assertEqual(derived.tzname(), 'cookie') 

def test_copying(self):
        # Check that ordered dicts are copyable, deepcopyable, picklable,
        # and have a repr/eval round-trip
        pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
        od = OrderedDict(pairs)
        update_test = OrderedDict()
        update_test.update(od)
        for i, dup in enumerate([
                    od.copy(),
                    copy.copy(od),
                    copy.deepcopy(od),
                    pickle.loads(pickle.dumps(od, 0)),
                    pickle.loads(pickle.dumps(od, 1)),
                    pickle.loads(pickle.dumps(od, 2)),
                    pickle.loads(pickle.dumps(od, -1)),
                    eval(repr(od)),
                    update_test,
                    OrderedDict(od),
                    ]):
            self.assertTrue(dup is not od)
            self.assertEqual(dup, od)
            self.assertEqual(list(dup.items()), list(od.items()))
            self.assertEqual(len(dup), len(od))
            self.assertEqual(type(dup), type(od)) 

def load_object(self, value):
        """The reversal of :meth:`dump_object`.  This might be callde with
        None.
        """
        if value is None:
            return None
        if value.startswith(b'!'):
            try:
                return pickle.loads(value[1:])
            except pickle.PickleError:
                return None
        try:
            return int(value)
        except ValueError:
            # before 0.8 we did not have serialization.  Still support that.
            return value 

def load_object(self, value):
        """The reversal of :meth:`dump_object`.  This might be callde with
        None.
        """
        if value is None:
            return None
        if value.startswith(b'!'):
            try:
                return pickle.loads(value[1:])
            except pickle.PickleError:
                return None
        try:
            return int(value)
        except ValueError:
            # before 0.8 we did not have serialization.  Still support that.
            return value 

def _controller(self):
        """Return the server controller."""
        def server_controller(cmd_id, cmd_body, _):
            """Server controler."""
            if not self.init_logginig:
                # the reason put the codes here is because we cannot get
                # kvstore.rank earlier
                head = '%(asctime)-15s Server[' + str(
                    self.kvstore.rank) + '] %(message)s'
                logging.basicConfig(level=logging.DEBUG, format=head)
                self.init_logginig = True

            if cmd_id == 0:
                try:
                    optimizer = pickle.loads(cmd_body)
                except:
                    raise
                self.kvstore.set_optimizer(optimizer)
            else:
                print("server %d, unknown command (%d, %s)" % (
                    self.kvstore.rank, cmd_id, cmd_body))
        return server_controller 

def test_pickle_unpickle(self):
        # Issue #7197: a TextTestRunner should be (un)pickleable. This is
        # required by test_multiprocessing under Windows (in verbose mode).
        from StringIO import StringIO as PickleableIO
        # cStringIO objects are not pickleable, but StringIO objects are.
        stream = PickleableIO("foo")
        runner = unittest.TextTestRunner(stream)
        for protocol in range(pickle.HIGHEST_PROTOCOL + 1):
            s = pickle.dumps(runner, protocol=protocol)
            obj = pickle.loads(s)
            # StringIO objects never compare equal, a cheap test instead.
            self.assertEqual(obj.stream.getvalue(), stream.getvalue()) 

def testPickle(self):
        # Issue 10326

        # Can't use TestCase classes defined in Test class as
        # pickle does not work with inner classes
        test = unittest.TestCase('run')
        for protocol in range(pickle.HIGHEST_PROTOCOL + 1):

            # blew up prior to fix
            pickled_test = pickle.dumps(test, protocol=protocol)

            unpickled_test = pickle.loads(pickled_test)
            self.assertEqual(test, unpickled_test) 

def value_decode(self, val):
        # This could raise an exception!
        return loads( _unquote(val) ), val 

def value_decode(self, val):
        strval = _unquote(val)
        try:
            return loads(strval), val
        except:
            return strval, val 

def loads(self, item):
        return pickle.loads(item) 

def ListTables(self):
        """Return a list of tables in this database."""
        pickledtablelist = self.db.get_get(_table_names_key)
        if pickledtablelist:
            return pickle.loads(pickledtablelist)
        else:
            return [] 

def test01(self):
        tabname = "test01"
        colname = 'cool numbers'
        try:
            self.tdb.Drop(tabname)
        except dbtables.TableDBError:
            pass
        self.tdb.CreateTable(tabname, [colname])
        import sys
        if sys.version_info[0] < 3 :
            self.tdb.Insert(tabname, {colname: pickle.dumps(3.14159, 1)})
        else :
            self.tdb.Insert(tabname, {colname: pickle.dumps(3.14159,
                1).decode("iso8859-1")})  # 8 bits

        if verbose:
            self.tdb._db_print()

        values = self.tdb.Select(
            tabname, [colname], conditions={colname: None})

        import sys
        if sys.version_info[0] < 3 :
            colval = pickle.loads(values[0][colname])
        else :
            colval = pickle.loads(bytes(values[0][colname], "iso8859-1"))
        self.assertTrue(colval > 3.141)
        self.assertTrue(colval < 3.142) 

def _base_test_pickle_DBError(self, pickle):
        self.env = db.DBEnv()
        self.env.open(self.homeDir, db.DB_CREATE | db.DB_INIT_MPOOL)
        self.db = db.DB(self.env)
        self.db.open(self.db_name, db.DB_HASH, db.DB_CREATE)
        self.db.put('spam', 'eggs')
        self.assertEqual(self.db['spam'], 'eggs')
        try:
            self.db.put('spam', 'ham', flags=db.DB_NOOVERWRITE)
        except db.DBError, egg:
            pickledEgg = pickle.dumps(egg)
            #print repr(pickledEgg)
            rottenEgg = pickle.loads(pickledEgg)
            if rottenEgg.args != egg.args or type(rottenEgg) != type(egg):
                raise Exception, (rottenEgg, '!=', egg) 

def test_pickling(self):
        p = pickle.dumps(self.set)
        copy = pickle.loads(p)
        self.assertEqual(self.set, copy,
                         "%s != %s" % (self.set, copy))

#------------------------------------------------------------------------------ 

def test_pickling(self):
        testcases = [(13,), (0, 11), (-22, 10), (20, 3, -1),
                     (13, 21, 3), (-2, 2, 2)]
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            for t in testcases:
                r = xrange(*t)
                self.assertEqual(list(pickle.loads(pickle.dumps(r, proto))),
                                 list(r))

        M = min(sys.maxint, sys.maxsize)
        large_testcases = testcases + [
            (0, M, 1),
            (M, 0, -1),
            (0, M, M - 1),
            (M // 2, M, 1),
            (0, -M, -1),
            (0, -M, 1 - M),
            (-M, M, 2),
            (-M, M, 1024),
            (-M, M, 10585),
            (M, -M, -2),
            (M, -M, -1024),
            (M, -M, -10585),
            ]
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            for t in large_testcases:
                r = xrange(*t)
                r_out = pickle.loads(pickle.dumps(r, proto))
                self.assert_xranges_equivalent(r_out, r) 

def check_params(self, f, nchannels, sampwidth, framerate, nframes,
                     comptype, compname):
        self.assertEqual(f.getnchannels(), nchannels)
        self.assertEqual(f.getsampwidth(), sampwidth)
        self.assertEqual(f.getframerate(), framerate)
        self.assertEqual(f.getnframes(), nframes)
        self.assertEqual(f.getcomptype(), comptype)
        self.assertEqual(f.getcompname(), compname)

        params = f.getparams()
        self.assertEqual(params,
                (nchannels, sampwidth, framerate, nframes, comptype, compname))

        dump = pickle.dumps(params)
        self.assertEqual(pickle.loads(dump), params) 

def loads(self, buf):
        # Ignore fast
        return pickle.loads(buf) 

def loads(self, buf):
        # Ignore fast
        return cPickle.loads(buf) 

def loads(self, input):
        return self.module.loads(input)

    # These tests are disabled because they require some special setup
    # on the worker that's hard to keep in sync. 

def test_unicode(self):
            endcases = [u'', u'<\\u>', u'<\\\u1234>', u'<\n>', u'<\\>']
            for proto in pickletester.protocols:
                for u in endcases:
                    p = self.dumps(u, proto)
                    u2 = self.loads(p)
                    self.assertEqual(u2, u) 

def test_pickling_base(self):
        # There's no point to pickling tzinfo objects on their own (they
        # carry no data), but they need to be picklable anyway else
        # concrete subclasses can't be pickled.
        orig = tzinfo.__new__(tzinfo)
        self.assertIs(type(orig), tzinfo)
        for pickler, unpickler, proto in pickle_choices:
            green = pickler.dumps(orig, proto)
            derived = unpickler.loads(green)
            self.assertIs(type(derived), tzinfo) 

def test_pickling(self):
        args = 12, 34, 56
        orig = timedelta(*args)
        for pickler, unpickler, proto in pickle_choices:
            green = pickler.dumps(orig, proto)
            derived = unpickler.loads(green)
            self.assertEqual(orig, derived) 

def test_pickling(self):
        args = 6, 7, 23
        orig = self.theclass(*args)
        for pickler, unpickler, proto in pickle_choices:
            green = pickler.dumps(orig, proto)
            derived = unpickler.loads(green)
            self.assertEqual(orig, derived) 

def test_pickling(self):
        args = 6, 7, 23, 20, 59, 1, 64**2
        orig = self.theclass(*args)
        for pickler, unpickler, proto in pickle_choices:
            green = pickler.dumps(orig, proto)
            derived = unpickler.loads(green)
            self.assertEqual(orig, derived) 

def test_more_pickling(self):
        a = self.theclass(2003, 2, 7, 16, 48, 37, 444116)
        s = pickle.dumps(a)
        b = pickle.loads(s)
        self.assertEqual(b.year, 2003)
        self.assertEqual(b.month, 2)
        self.assertEqual(b.day, 7) 

def test_pickling_subclass_datetime(self):
        args = 6, 7, 23, 20, 59, 1, 64**2
        orig = SubclassDatetime(*args)
        for pickler, unpickler, proto in pickle_choices:
            green = pickler.dumps(orig, proto)
            derived = unpickler.loads(green)
            self.assertEqual(orig, derived) 

def test_pickling(self):
        args = 20, 59, 16, 64**2
        orig = self.theclass(*args)
        for pickler, unpickler, proto in pickle_choices:
            green = pickler.dumps(orig, proto)
            derived = unpickler.loads(green)
            self.assertEqual(orig, derived) 

def test_pickling_subclass_time(self):
        args = 20, 59, 16, 64**2
        orig = SubclassTime(*args)
        for pickler, unpickler, proto in pickle_choices:
            green = pickler.dumps(orig, proto)
            derived = unpickler.loads(green)
            self.assertEqual(orig, derived) 

def test_pickling(self):
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            for b in b"", b"a", b"abc", b"\xffab\x80", b"\0\0\377\0\0":
                b = self.type2test(b)
                ps = pickle.dumps(b, proto)
                q = pickle.loads(ps)
                self.assertEqual(b, q) 

def test_pickle(self):
        a = ByteArraySubclass(b"abcd")
        a.x = 10
        a.y = ByteArraySubclass(b"efgh")
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            b = pickle.loads(pickle.dumps(a, proto))
            self.assertNotEqual(id(a), id(b))
            self.assertEqual(a, b)
            self.assertEqual(a.x, b.x)
            self.assertEqual(a.y, b.y)
            self.assertEqual(type(a), type(b))
            self.assertEqual(type(a.y), type(b.y)) 

def test_pickle(self):
        p = TestNT(x=10, y=20, z=30)
        for module in pickle, cPickle:
            loads = getattr(module, 'loads')
            dumps = getattr(module, 'dumps')
            for protocol in -1, 0, 1, 2:
                q = loads(dumps(p, protocol))
                self.assertEqual(p, q)
                self.assertEqual(p._fields, q._fields) 

def test_pickling_bug_18015(self):
        # http://bugs.python.org/issue18015
        pt = pickle.loads(py273_named_tuple_pickle)
        self.assertEqual(pt.x, 10) 

def test_error(self):
        import pickle
        e1 = ConfigParser.Error('value')
        pickled = pickle.dumps(e1)
        e2 = pickle.loads(pickled)
        self.assertEqual(e1.message, e2.message)
        self.assertEqual(repr(e1), repr(e2)) 

def test_nosectionerror(self):
        import pickle
        e1 = ConfigParser.NoSectionError('section')
        pickled = pickle.dumps(e1)
        e2 = pickle.loads(pickled)
        self.assertEqual(e1.message, e2.message)
        self.assertEqual(e1.args, e2.args)
        self.assertEqual(e1.section, e2.section)
        self.assertEqual(repr(e1), repr(e2)) 

def test_nooptionerror(self):
        import pickle
        e1 = ConfigParser.NoOptionError('option', 'section')
        pickled = pickle.dumps(e1)
        e2 = pickle.loads(pickled)
        self.assertEqual(e1.message, e2.message)
        self.assertEqual(e1.args, e2.args)
        self.assertEqual(e1.section, e2.section)
        self.assertEqual(e1.option, e2.option)
        self.assertEqual(repr(e1), repr(e2)) 

def test_duplicatesectionerror(self):
        import pickle
        e1 = ConfigParser.DuplicateSectionError('section')
        pickled = pickle.dumps(e1)
        e2 = pickle.loads(pickled)
        self.assertEqual(e1.message, e2.message)
        self.assertEqual(e1.args, e2.args)
        self.assertEqual(e1.section, e2.section)
        self.assertEqual(repr(e1), repr(e2)) 

def test_interpolationmissingoptionerror(self):
        import pickle
        e1 = ConfigParser.InterpolationMissingOptionError('option', 'section',
            'rawval', 'reference')
        pickled = pickle.dumps(e1)
        e2 = pickle.loads(pickled)
        self.assertEqual(e1.message, e2.message)
        self.assertEqual(e1.args, e2.args)
        self.assertEqual(e1.section, e2.section)
        self.assertEqual(e1.option, e2.option)
        self.assertEqual(e1.reference, e2.reference)
        self.assertEqual(repr(e1), repr(e2)) 

def test_interpolationsyntaxerror(self):
        import pickle
        e1 = ConfigParser.InterpolationSyntaxError('option', 'section', 'msg')
        pickled = pickle.dumps(e1)
        e2 = pickle.loads(pickled)
        self.assertEqual(e1.message, e2.message)
        self.assertEqual(e1.args, e2.args)
        self.assertEqual(e1.section, e2.section)
        self.assertEqual(e1.option, e2.option)
        self.assertEqual(repr(e1), repr(e2)) 

def test_interpolationdeptherror(self):
        import pickle
        e1 = ConfigParser.InterpolationDepthError('option', 'section',
            'rawval')
        pickled = pickle.dumps(e1)
        e2 = pickle.loads(pickled)
        self.assertEqual(e1.message, e2.message)
        self.assertEqual(e1.args, e2.args)
        self.assertEqual(e1.section, e2.section)
        self.assertEqual(e1.option, e2.option)
        self.assertEqual(repr(e1), repr(e2)) 

def test_parsingerror(self):
        import pickle
        e1 = ConfigParser.ParsingError('source')
        e1.append(1, 'line1')
        e1.append(2, 'line2')
        e1.append(3, 'line3')
        pickled = pickle.dumps(e1)
        e2 = pickle.loads(pickled)
        self.assertEqual(e1.message, e2.message)
        self.assertEqual(e1.args, e2.args)
        self.assertEqual(e1.filename, e2.filename)
        self.assertEqual(e1.errors, e2.errors)
        self.assertEqual(repr(e1), repr(e2)) 

def test_missingsectionheadererror(self):
        import pickle
        e1 = ConfigParser.MissingSectionHeaderError('filename', 123, 'line')
        pickled = pickle.dumps(e1)
        e2 = pickle.loads(pickled)
        self.assertEqual(e1.message, e2.message)
        self.assertEqual(e1.args, e2.args)
        self.assertEqual(e1.line, e2.line)
        self.assertEqual(e1.filename, e2.filename)
        self.assertEqual(e1.lineno, e2.lineno)
        self.assertEqual(repr(e1), repr(e2)) 

def test_pickle(self):
        rawdata = 'Customer="WILE_E_COYOTE"; Path=/acme; Version=1'
        expected_output = 'Set-Cookie: %s' % rawdata

        C = Cookie.SimpleCookie()
        C.load(rawdata)
        self.assertEqual(C.output(), expected_output)

        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            C1 = pickle.loads(pickle.dumps(C, protocol=proto))
            self.assertEqual(C1.output(), expected_output) 

def test_pickling(self):
        for i in range(pickle.HIGHEST_PROTOCOL + 1):
            p = pickle.dumps(self.s, i)
            dup = pickle.loads(p)
            self.assertEqual(self.s, dup, "%s != %s" % (self.s, dup))
            if type(self.s) not in (set, frozenset):
                self.s.x = 10
                p = pickle.dumps(self.s)
                dup = pickle.loads(p)
                self.assertEqual(self.s.x, dup.x) 

def test_pickling(self):
        p = pickle.dumps(self.set)
        copy = pickle.loads(p)
        self.assertEqual(self.set, copy,
                         "%s != %s" % (self.set, copy))

#------------------------------------------------------------------------------ 

def test_count(self):
        self.assertEqual(zip('abc',count()), [('a', 0), ('b', 1), ('c', 2)])
        self.assertEqual(zip('abc',count(3)), [('a', 3), ('b', 4), ('c', 5)])
        self.assertEqual(take(2, zip('abc',count(3))), [('a', 3), ('b', 4)])
        self.assertEqual(take(2, zip('abc',count(-1))), [('a', -1), ('b', 0)])
        self.assertEqual(take(2, zip('abc',count(-3))), [('a', -3), ('b', -2)])
        self.assertRaises(TypeError, count, 2, 3, 4)
        self.assertRaises(TypeError, count, 'a')
        self.assertEqual(list(islice(count(maxsize-5), 10)), range(maxsize-5, maxsize+5))
        self.assertEqual(list(islice(count(-maxsize-5), 10)), range(-maxsize-5, -maxsize+5))
        c = count(3)
        self.assertEqual(repr(c), 'count(3)')
        c.next()
        self.assertEqual(repr(c), 'count(4)')
        c = count(-9)
        self.assertEqual(repr(c), 'count(-9)')
        c.next()
        self.assertEqual(repr(count(10.25)), 'count(10.25)')
        self.assertEqual(c.next(), -8)
        for i in (-sys.maxint-5, -sys.maxint+5 ,-10, -1, 0, 10, sys.maxint-5, sys.maxint+5):
            # Test repr (ignoring the L in longs)
            r1 = repr(count(i)).replace('L', '')
            r2 = 'count(%r)'.__mod__(i).replace('L', '')
            self.assertEqual(r1, r2)

        # check copy, deepcopy, pickle
        for value in -3, 3, sys.maxint-5, sys.maxint+5:
            c = count(value)
            self.assertEqual(next(copy.copy(c)), value)
            self.assertEqual(next(copy.deepcopy(c)), value)
            self.assertEqual(next(pickle.loads(pickle.dumps(c))), value) 

def loads(self, buf):
        # Ignore fast
        return pickle.loads(buf) 

def loads(self, buf):
        f = StringIO(buf)
        u = pickle.Unpickler(f)
        return u.load() 

def loads(self, buf):
        class PersUnpickler(pickle.Unpickler):
            def persistent_load(subself, obj):
                return self.persistent_load(obj)
        f = StringIO(buf)
        u = PersUnpickler(f)
        return u.load() 

def loads(self, buf):
        # Ignore fast
        return pickle.loads(buf) 

def test_pickling(self):
        state = pickle.dumps(self.gen)
        origseq = [self.gen.random() for i in xrange(10)]
        newgen = pickle.loads(state)
        restoredseq = [newgen.random() for i in xrange(10)]
        self.assertEqual(origseq, restoredseq) 

def test_pickle(self):
        import pickle
        self.assertIs(pickle.loads(pickle.dumps(True)), True)
        self.assertIs(pickle.loads(pickle.dumps(False)), False)
        self.assertIs(pickle.loads(pickle.dumps(True, True)), True)
        self.assertIs(pickle.loads(pickle.dumps(False, True)), False) 

def test_cpickle(self):
        import cPickle
        self.assertIs(cPickle.loads(cPickle.dumps(True)), True)
        self.assertIs(cPickle.loads(cPickle.dumps(False)), False)
        self.assertIs(cPickle.loads(cPickle.dumps(True, True)), True)
        self.assertIs(cPickle.loads(cPickle.dumps(False, True)), False) 

def test_mixedpickle(self):
        import pickle, cPickle
        self.assertIs(pickle.loads(cPickle.dumps(True)), True)
        self.assertIs(pickle.loads(cPickle.dumps(False)), False)
        self.assertIs(pickle.loads(cPickle.dumps(True, True)), True)
        self.assertIs(pickle.loads(cPickle.dumps(False, True)), False)

        self.assertIs(cPickle.loads(pickle.dumps(True)), True)
        self.assertIs(cPickle.loads(pickle.dumps(False)), False)
        self.assertIs(cPickle.loads(pickle.dumps(True, True)), True)
        self.assertIs(cPickle.loads(pickle.dumps(False, True)), False) 

def test_pickling(self):
        buf = self.buftype("1234567890")
        memio = self.ioclass(buf)
        memio.foo = 42
        memio.seek(2)

        class PickleTestMemIO(self.ioclass):
            def __init__(me, initvalue, foo):
                self.ioclass.__init__(me, initvalue)
                me.foo = foo
            # __getnewargs__ is undefined on purpose. This checks that PEP 307
            # is used to provide pickling support.

        # Pickle expects the class to be on the module level. Here we use a
        # little hack to allow the PickleTestMemIO class to derive from
        # self.ioclass without having to define all combinations explicitly on
        # the module-level.
        import __main__
        PickleTestMemIO.__module__ = '__main__'
        __main__.PickleTestMemIO = PickleTestMemIO
        submemio = PickleTestMemIO(buf, 80)
        submemio.seek(2)

        # We only support pickle protocol 2 and onward since we use extended
        # __reduce__ API of PEP 307 to provide pickling support.
        for proto in range(2, pickle.HIGHEST_PROTOCOL):
            for obj in (memio, submemio):
                obj2 = pickle.loads(pickle.dumps(obj, protocol=proto))
                self.assertEqual(obj.getvalue(), obj2.getvalue())
                self.assertEqual(obj.__class__, obj2.__class__)
                self.assertEqual(obj.foo, obj2.foo)
                self.assertEqual(obj.tell(), obj2.tell())
                obj.close()
                self.assertRaises(ValueError, pickle.dumps, obj, proto)
        del __main__.PickleTestMemIO 

def __new__(cls, *args, **kwargs):
            return pickle.loads(pickle.dumps(pyio.StringIO(*args, **kwargs))) 

def test_pickle(self):
        d = Decimal('-3.141590000')
        p = pickle.dumps(d)
        e = pickle.loads(p)
        self.assertEqual(d, e) 

def test_pickle(self):
        c = Context()
        e = pickle.loads(pickle.dumps(c))
        for k in vars(c):
            v1 = vars(c)[k]
            v2 = vars(e)[k]
            self.assertEqual(v1, v2) 

def test_pickleable(self):
        cache = LRUCache(2)
        cache["foo"] = 42
        cache["bar"] = 23
        cache["foo"]

        for protocol in range(3):
            copy = pickle.loads(pickle.dumps(cache, protocol))
            assert copy.capacity == cache.capacity
            assert copy._mapping == cache._mapping
            assert copy._queue == cache._queue 

def test_session_special_types(self):
        app = flask.Flask(__name__)
        app.secret_key = 'development-key'
        app.testing = True
        now = datetime.utcnow().replace(microsecond=0)
        the_uuid = uuid.uuid4()

        @app.after_request
        def modify_session(response):
            flask.session['m'] = flask.Markup('Hello!')
            flask.session['u'] = the_uuid
            flask.session['dt'] = now
            flask.session['t'] = (1, 2, 3)
            return response

        @app.route('/')
        def dump_session_contents():
            return pickle.dumps(dict(flask.session))

        c = app.test_client()
        c.get('/')
        rv = pickle.loads(c.get('/').data)
        self.assert_equal(rv['m'], flask.Markup('Hello!'))
        self.assert_equal(type(rv['m']), flask.Markup)
        self.assert_equal(rv['dt'], now)
        self.assert_equal(rv['u'], the_uuid)
        self.assert_equal(rv['t'], (1, 2, 3)) 

def test_pickleable(self):
        cache = LRUCache(2)
        cache["foo"] = 42
        cache["bar"] = 23
        cache["foo"]

        for protocol in range(3):
            copy = pickle.loads(pickle.dumps(cache, protocol))
            assert copy.capacity == cache.capacity
            assert copy._mapping == cache._mapping
            assert copy._queue == cache._queue 

def test_session_special_types(self):
        app = flask.Flask(__name__)
        app.secret_key = 'development-key'
        app.testing = True
        now = datetime.utcnow().replace(microsecond=0)
        the_uuid = uuid.uuid4()

        @app.after_request
        def modify_session(response):
            flask.session['m'] = flask.Markup('Hello!')
            flask.session['u'] = the_uuid
            flask.session['dt'] = now
            flask.session['t'] = (1, 2, 3)
            return response

        @app.route('/')
        def dump_session_contents():
            return pickle.dumps(dict(flask.session))

        c = app.test_client()
        c.get('/')
        rv = pickle.loads(c.get('/').data)
        self.assert_equal(rv['m'], flask.Markup('Hello!'))
        self.assert_equal(type(rv['m']), flask.Markup)
        self.assert_equal(rv['dt'], now)
        self.assert_equal(rv['u'], the_uuid)
        self.assert_equal(rv['t'], (1, 2, 3)) 

def get(self, key):
        try:
            expires, value = self._cache[key]
            if expires > time():
                return pickle.loads(value)
        except (KeyError, pickle.PickleError):
            return None 

def get(self, key):
        try:
            expires, value = self._cache[key]
            if expires > time():
                return pickle.loads(value)
        except (KeyError, pickle.PickleError):
            return None 

def handle_read(self):
        try:
            data = self.recv(self.dlen)
            if len(data) == 0:
                return
        except socket.error as e:
            if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
                return

        self.data += data
        self.dlen -= len(data)
        if self.dlen > 0:
            # don't have complete record yet. wait for more data to read
            return

        if self.rlen == 0:
            self.dlen = self.rlen = struct.unpack('>L', self.data)[0]
            self.data = ''
            # got record length. now read record
            return

        # got complete record
        obj = pickle.loads(self.data)
        record = logging.makeLogRecord(obj)

        # Note: EVERY record gets logged. This is because Logger.handle
        # is normally called AFTER logger-level filtering.
        # Filter (e.g., only WARNING or higher)
        # at the sender to save network bandwidth.
        globalLogger.handle(record)

        # reset for next record
        self.data = ''
        self.rlen = 0
        self.dlen = 4 

def collect_results_gpu(result_part, size):
    rank, world_size = get_dist_info()
    # dump result part to tensor with pickle
    part_tensor = torch.tensor(
        bytearray(pickle.dumps(result_part)), dtype=torch.uint8, device='cuda')
    # gather all result part tensor shape
    shape_tensor = torch.tensor(part_tensor.shape, device='cuda')
    shape_list = [shape_tensor.clone() for _ in range(world_size)]
    dist.all_gather(shape_list, shape_tensor)
    # padding result part tensor to max length
    shape_max = torch.tensor(shape_list).max()
    part_send = torch.zeros(shape_max, dtype=torch.uint8, device='cuda')
    part_send[:shape_tensor[0]] = part_tensor
    part_recv_list = [
        part_tensor.new_zeros(shape_max) for _ in range(world_size)
    ]
    # gather all result part
    dist.all_gather(part_recv_list, part_send)

    if rank == 0:
        part_list = []
        for recv, shape in zip(part_recv_list, shape_list):
            part_list.append(
                pickle.loads(recv[:shape[0]].cpu().numpy().tobytes()))
        # sort the results
        ordered_results = []
        for res in zip(*part_list):
            ordered_results.extend(list(res))
        # the dataloader may pad some samples
        ordered_results = ordered_results[:size]
        return ordered_results 

def test_pickleable(self):
        cache = LRUCache(2)
        cache["foo"] = 42
        cache["bar"] = 23
        cache["foo"]

        for protocol in range(3):
            copy = pickle.loads(pickle.dumps(cache, protocol))
            assert copy.capacity == cache.capacity
            assert copy._mapping == cache._mapping
            assert copy._queue == cache._queue 

def test_session_special_types(self):
        app = flask.Flask(__name__)
        app.secret_key = 'development-key'
        app.testing = True
        now = datetime.utcnow().replace(microsecond=0)
        the_uuid = uuid.uuid4()

        @app.after_request
        def modify_session(response):
            flask.session['m'] = flask.Markup('Hello!')
            flask.session['u'] = the_uuid
            flask.session['dt'] = now
            flask.session['b'] = b'\xff'
            flask.session['t'] = (1, 2, 3)
            return response

        @app.route('/')
        def dump_session_contents():
            return pickle.dumps(dict(flask.session))

        c = app.test_client()
        c.get('/')
        rv = pickle.loads(c.get('/').data)
        self.assert_equal(rv['m'], flask.Markup('Hello!'))
        self.assert_equal(type(rv['m']), flask.Markup)
        self.assert_equal(rv['dt'], now)
        self.assert_equal(rv['u'], the_uuid)
        self.assert_equal(rv['b'], b'\xff')
        self.assert_equal(type(rv['b']), bytes)
        self.assert_equal(rv['t'], (1, 2, 3)) 

def get(self, key):
        try:
            expires, value = self._cache[key]
            if expires > time():
                return pickle.loads(value)
        except (KeyError, pickle.PickleError):
            return None 

def deserialize(pickle_serialized=''):
        """ddserialize"""
        return pickle.loads(pickle_serialized) 

def test_ndarray_pickle():
    np.random.seed(0)
    maxdim = 5
    nrepeat = 10
    for repeat in range(nrepeat):
        for dim in range(1, maxdim):
            a = random_ndarray(dim)
            b = mx.nd.empty(a.shape)
            a[:] = np.random.uniform(-10, 10, a.shape)
            b[:] = np.random.uniform(-10, 10, a.shape)
            a = a + b
            data = pkl.dumps(a)
            a2 = pkl.loads(data)
            assert np.sum(a.asnumpy() != a2.asnumpy()) == 0 

def recv(self):
        """Receive object"""
        buf = self.recv_bytes()
        return pickle.loads(buf) 

def CreateTable(self, table, columns):
        """CreateTable(table, columns) - Create a new table in the database.

        raises TableDBError if it already exists or for other DB errors.
        """
        assert isinstance(columns, list)

        txn = None
        try:
            # checking sanity of the table and column names here on
            # table creation will prevent problems elsewhere.
            if contains_metastrings(table):
                raise ValueError(
                    "bad table name: contains reserved metastrings")
            for column in columns :
                if contains_metastrings(column):
                    raise ValueError(
                        "bad column name: contains reserved metastrings")

            columnlist_key = _columns_key(table)
            if getattr(self.db, "has_key")(columnlist_key):
                raise TableAlreadyExists, "table already exists"

            txn = self.env.txn_begin()
            # store the table's column info
            getattr(self.db, "put_bytes", self.db.put)(columnlist_key,
                    pickle.dumps(columns, 1), txn=txn)

            # add the table name to the tablelist
            tablelist = pickle.loads(getattr(self.db, "get_bytes",
                self.db.get) (_table_names_key, txn=txn, flags=db.DB_RMW))
            tablelist.append(table)
            # delete 1st, in case we opened with DB_DUP
            self.db.delete(_table_names_key, txn=txn)
            getattr(self.db, "put_bytes", self.db.put)(_table_names_key,
                    pickle.dumps(tablelist, 1), txn=txn)

            txn.commit()
            txn = None
        except db.DBError, dberror:
            if txn:
                txn.abort()
            if sys.version_info < (2, 6) :
                raise TableDBError, dberror[1]
            else :
                raise TableDBError, dberror.args[1] 

def CreateOrExtendTable(self, table, columns):
        """CreateOrExtendTable(table, columns)

        Create a new table in the database.

        If a table of this name already exists, extend it to have any
        additional columns present in the given list as well as
        all of its current columns.
        """
        assert isinstance(columns, list)

        try:
            self.CreateTable(table, columns)
        except TableAlreadyExists:
            # the table already existed, add any new columns
            txn = None
            try:
                columnlist_key = _columns_key(table)
                txn = self.env.txn_begin()

                # load the current column list
                oldcolumnlist = pickle.loads(
                    getattr(self.db, "get_bytes",
                        self.db.get)(columnlist_key, txn=txn, flags=db.DB_RMW))
                # create a hash table for fast lookups of column names in the
                # loop below
                oldcolumnhash = {}
                for c in oldcolumnlist:
                    oldcolumnhash[c] = c

                # create a new column list containing both the old and new
                # column names
                newcolumnlist = copy.copy(oldcolumnlist)
                for c in columns:
                    if not c in oldcolumnhash:
                        newcolumnlist.append(c)

                # store the table's new extended column list
                if newcolumnlist != oldcolumnlist :
                    # delete the old one first since we opened with DB_DUP
                    self.db.delete(columnlist_key, txn=txn)
                    getattr(self.db, "put_bytes", self.db.put)(columnlist_key,
                                pickle.dumps(newcolumnlist, 1),
                                txn=txn)

                txn.commit()
                txn = None

                self.__load_column_info(table)
            except db.DBError, dberror:
                if txn:
                    txn.abort()
                if sys.version_info < (2, 6) :
                    raise TableDBError, dberror[1]
                else :
                    raise TableDBError, dberror.args[1] 

def Drop(self, table):
        """Remove an entire table from the database"""
        txn = None
        try:
            txn = self.env.txn_begin()

            # delete the column list
            self.db.delete(_columns_key(table), txn=txn)

            cur = self.db.cursor(txn)

            # delete all keys containing this tables column and row info
            table_key = _search_all_data_key(table)
            while 1:
                try:
                    key, data = cur.set_range(table_key)
                except db.DBNotFoundError:
                    break
                # only delete items in this table
                if key[:len(table_key)] != table_key:
                    break
                cur.delete()

            # delete all rowids used by this table
            table_key = _search_rowid_key(table)
            while 1:
                try:
                    key, data = cur.set_range(table_key)
                except db.DBNotFoundError:
                    break
                # only delete items in this table
                if key[:len(table_key)] != table_key:
                    break
                cur.delete()

            cur.close()

            # delete the tablename from the table name list
            tablelist = pickle.loads(
                getattr(self.db, "get_bytes", self.db.get)(_table_names_key,
                    txn=txn, flags=db.DB_RMW))
            try:
                tablelist.remove(table)
            except ValueError:
                # hmm, it wasn't there, oh well, that's what we want.
                pass
            # delete 1st, incase we opened with DB_DUP
            self.db.delete(_table_names_key, txn=txn)
            getattr(self.db, "put_bytes", self.db.put)(_table_names_key,
                    pickle.dumps(tablelist, 1), txn=txn)

            txn.commit()
            txn = None

            if table in self.__tablecolumns:
                del self.__tablecolumns[table]

        except db.DBError, dberror:
            if txn:
                txn.abort()
            raise TableDBError(dberror.args[1]) 

def test02(self):
        tabname = "test02"
        col0 = 'coolness factor'
        col1 = 'but can it fly?'
        col2 = 'Species'

        import sys
        if sys.version_info[0] < 3 :
            testinfo = [
                {col0: pickle.dumps(8, 1), col1: 'no', col2: 'Penguin'},
                {col0: pickle.dumps(-1, 1), col1: 'no', col2: 'Turkey'},
                {col0: pickle.dumps(9, 1), col1: 'yes', col2: 'SR-71A Blackbird'}
            ]
        else :
            testinfo = [
                {col0: pickle.dumps(8, 1).decode("iso8859-1"),
                    col1: 'no', col2: 'Penguin'},
                {col0: pickle.dumps(-1, 1).decode("iso8859-1"),
                    col1: 'no', col2: 'Turkey'},
                {col0: pickle.dumps(9, 1).decode("iso8859-1"),
                    col1: 'yes', col2: 'SR-71A Blackbird'}
            ]

        try:
            self.tdb.Drop(tabname)
        except dbtables.TableDBError:
            pass
        self.tdb.CreateTable(tabname, [col0, col1, col2])
        for row in testinfo :
            self.tdb.Insert(tabname, row)

        import sys
        if sys.version_info[0] < 3 :
            values = self.tdb.Select(tabname, [col2],
                conditions={col0: lambda x: pickle.loads(x) >= 8})
        else :
            values = self.tdb.Select(tabname, [col2],
                conditions={col0: lambda x:
                    pickle.loads(bytes(x, "iso8859-1")) >= 8})

        self.assertEqual(len(values), 2)
        if values[0]['Species'] == 'Penguin' :
            self.assertEqual(values[1]['Species'], 'SR-71A Blackbird')
        elif values[0]['Species'] == 'SR-71A Blackbird' :
            self.assertEqual(values[1]['Species'], 'Penguin')
        else :
            if verbose:
                print "values= %r" % (values,)
            raise RuntimeError("Wrong values returned!") 

def testPickledDocument(self):
        doc = parseString("<?xml version='1.0' encoding='us-ascii'?>\n"
                    "<!DOCTYPE doc PUBLIC 'http://xml.python.org/public'"
                    " 'http://xml.python.org/system' [\n"
                    "  <!ELEMENT e EMPTY>\n"
                    "  <!ENTITY ent SYSTEM 'http://xml.python.org/entity'>\n"
                    "]><doc attr='value'> text\n"
                    "<?pi sample?> <!-- comment --> <e/> </doc>")
        s = pickle.dumps(doc)
        doc2 = pickle.loads(s)
        stack = [(doc, doc2)]
        while stack:
            n1, n2 = stack.pop()
            self.confirm(n1.nodeType == n2.nodeType
                    and len(n1.childNodes) == len(n2.childNodes)
                    and n1.nodeName == n2.nodeName
                    and not n1.isSameNode(n2)
                    and not n2.isSameNode(n1))
            if n1.nodeType == Node.DOCUMENT_TYPE_NODE:
                len(n1.entities)
                len(n2.entities)
                len(n1.notations)
                len(n2.notations)
                self.confirm(len(n1.entities) == len(n2.entities)
                        and len(n1.notations) == len(n2.notations))
                for i in range(len(n1.notations)):
                    # XXX this loop body doesn't seem to be executed?
                    no1 = n1.notations.item(i)
                    no2 = n1.notations.item(i)
                    self.confirm(no1.name == no2.name
                            and no1.publicId == no2.publicId
                            and no1.systemId == no2.systemId)
                    stack.append((no1, no2))
                for i in range(len(n1.entities)):
                    e1 = n1.entities.item(i)
                    e2 = n2.entities.item(i)
                    self.confirm(e1.notationName == e2.notationName
                            and e1.publicId == e2.publicId
                            and e1.systemId == e2.systemId)
                    stack.append((e1, e2))
            if n1.nodeType != Node.DOCUMENT_NODE:
                self.confirm(n1.ownerDocument.isSameNode(doc)
                        and n2.ownerDocument.isSameNode(doc2))
            for i in range(len(n1.childNodes)):
                stack.append((n1.childNodes[i], n2.childNodes[i])) 

def open_session(self, app, request):
        """
        @param app: Flask app
        @param request: Flask HTTP Request
        @summary: Sets the current session from the request's session cooke. This overrides the default
        Flask implementation, adding AES decryption of the client-side session cookie.
        """

        # Get the session cookie
        session_cookie = request.cookies.get(self.session_cookie_name)
        if not session_cookie:
            return self.session_class()

        # Get the crypto key
        crypto_key = app.config['SESSION_CRYPTO_KEY'] if 'SESSION_CRYPTO_KEY' in app.config else app.crypto_key

        # Split the session cookie : <z|u>.<base64 cipher text>.<base64 mac>.<base64 nonce>
        itup = session_cookie.split(".")
        if len(itup) != 4:
            return self.session_class()  # Session cookie not in the right format

        try:

            # Compressed data?
            if itup[0] == 'z':  # session cookie for compressed data starts with "z."
                is_compressed = True
            else:
                is_compressed = False

            # Decode the cookie parts from base64
            ciphertext = base64.b64decode(bytes(itup[1], 'utf-8'))
            mac = base64.b64decode(bytes(itup[2], 'utf-8'))
            nonce = base64.b64decode(bytes(itup[3], 'utf-8'))

            # Decrypt
            cipher = AES.new(crypto_key, AES.MODE_EAX, nonce)
            data = cipher.decrypt_and_verify(ciphertext, mac)

            # Convert back to a dict and pass that onto the session
            if is_compressed:
                data = zlib.decompress(data)
            session_dict = pickle.loads(data)

            return self.session_class(session_dict)

        except (ValueError, _pickle.UnpicklingError, pickle.UnpicklingError):
            return self.session_class()