Python pickle.dumps() Examples

The following are 30 code examples for showing how to use pickle.dumps(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module pickle , or try the search function .

Example 1
Project: ALF   Author: blackberry   File: SockPuppet.py    License: Apache License 2.0 7 votes vote down vote up
def run_code(self, function, *args, **kwargs):
        log.debug("%s() args:%s kwargs:%s on target", function.func_name, args, kwargs)
        data = {"cmd":self.CODE,
                "code":marshal.dumps(function.func_code),
                "name":function.func_name,
                "args":args,
                "kwargs":kwargs,
                "defaults":function.__defaults__,
                "closure":function.__closure__}
        self.send_data(data)
        log.debug("waiting for code to execute...")
        data = self.recv_data()
        if data["cmd"] == self.EXCEPT:
            log.debug("received exception")
            raise self._process_target_except(data)
        assert data["cmd"] == self.RETURN
        return data["value"] 
Example 2
Project: vergeml   Author: mme   File: cache.py    License: MIT License 6 votes vote down vote up
def _serialize_data(self, data):

        # Default to raw bytes
        type_ = _BYTES

        if isinstance(data, np.ndarray):
        # When the data is a numpy array, use the more compact native
        # numpy format.
            buf = io.BytesIO()
            np.save(buf, data)
            data = buf.getvalue()
            type_ = _NUMPY

        elif not isinstance(data, (bytearray, bytes)):
        # Everything else except byte data is serialized in pickle format.
            data = pickle.dumps(data)
            type_ = _PICKLE

        if self.compress:
        # Optional compression
            data = lz4.frame.compress(data)

        return type_, data 
Example 3
Project: The-chat-room   Author: 11ze   File: vachat.py    License: MIT License 6 votes vote down vote up
def run(self):
        while True:
            try:
                self.sock.connect(self.ADDR)
                break
            except:
                time.sleep(3)
                continue
        print("AUDIO client connected...")
        self.stream = self.p.open(format=FORMAT,
                             channels=CHANNELS,
                             rate=RATE,
                             input=True,
                             frames_per_buffer=CHUNK)
        while self.stream.is_active():
            frames = []
            for i in range(0, int(RATE / CHUNK * RECORD_SECONDS)):
                data = self.stream.read(CHUNK)
                frames.append(data)
            senddata = pickle.dumps(frames)
            try:
                self.sock.sendall(struct.pack("L", len(senddata)) + senddata)
            except:
                break 
Example 4
Project: Dumb-Cogs   Author: irdumbs   File: adventure.py    License: MIT License 6 votes vote down vote up
def t_suspend(self, verb, obj):
        if isinstance(obj, str):
            if os.path.exists(obj):  # pragma: no cover
                self.write('I refuse to overwrite an existing file.')
                return
            savefile = open(obj, 'wb')
        else:
            savefile = obj
        r = self.random_generator  # must replace live object with static state
        self.random_state = r.getstate()
        try:
            del self.random_generator
            savefile.write(zlib.compress(pickle.dumps(self), 9))
        finally:
            self.random_generator = r
            if savefile is not obj:
                savefile.close()
        self.write('Game saved') 
Example 5
Project: pywren-ibm-cloud   Author: pywren   File: handler.py    License: Apache License 2.0 6 votes vote down vote up
def _send_status_os(self):
        """
        Send the status event to the Object Storage
        """
        executor_id = self.response['executor_id']
        job_id = self.response['job_id']
        call_id = self.response['call_id']
        act_id = self.response['activation_id']

        if self.response['type'] == '__init__':
            init_key = create_init_key(JOBS_PREFIX, executor_id, job_id, call_id, act_id)
            self.internal_storage.put_data(init_key, '')

        elif self.response['type'] == '__end__':
            status_key = create_status_key(JOBS_PREFIX, executor_id, job_id, call_id)
            dmpd_response_status = json.dumps(self.response)
            drs = sizeof_fmt(len(dmpd_response_status))
            logger.info("Storing execution stats - Size: {}".format(drs))
            self.internal_storage.put_data(status_key, dmpd_response_status) 
Example 6
Project: jawfish   Author: war-and-code   File: test_case.py    License: MIT License 6 votes vote down vote up
def testPickle(self):
        # Issue 10326

        # Can't use TestCase classes defined in Test class as
        # pickle does not work with inner classes
        test = unittest.TestCase('run')
        for protocol in range(pickle.HIGHEST_PROTOCOL + 1):

            # blew up prior to fix
            pickled_test = pickle.dumps(test, protocol=protocol)
            unpickled_test = pickle.loads(pickled_test)
            self.assertEqual(test, unpickled_test)

            # exercise the TestCase instance in a way that will invoke
            # the type equality lookup mechanism
            unpickled_test.assertEqual(set(), set()) 
Example 7
Project: jawfish   Author: war-and-code   File: handlers.py    License: MIT License 6 votes vote down vote up
def makePickle(self, record):
        """
        Pickles the record in binary format with a length prefix, and
        returns it ready for transmission across the socket.
        """
        ei = record.exc_info
        if ei:
            # just to get traceback text into record.exc_text ...
            dummy = self.format(record)
        # See issue #14436: If msg or args are objects, they may not be
        # available on the receiving end. So we convert the msg % args
        # to a string, save it as msg and zap the args.
        d = dict(record.__dict__)
        d['msg'] = record.getMessage()
        d['args'] = None
        d['exc_info'] = None
        s = pickle.dumps(d, 1)
        slen = struct.pack(">L", len(s))
        return slen + s 
Example 8
Project: dataflow   Author: tensorpack   File: parallel.py    License: Apache License 2.0 6 votes vote down vote up
def run(self):
            enable_death_signal(_warn=self.idx == 0)
            self.ds.reset_state()
            itr = _repeat_iter(lambda: self.ds)

            context = zmq.Context()
            socket = context.socket(zmq.PUSH)
            socket.set_hwm(self.hwm)
            socket.connect(self.conn_name)
            try:
                while True:
                    try:
                        dp = next(itr)
                        socket.send(dumps(dp), copy=False)
                    except Exception:
                        dp = _ExceptionWrapper(sys.exc_info()).pack()
                        socket.send(dumps(dp), copy=False)
                        raise
            # sigint could still propagate here, e.g. when nested
            except KeyboardInterrupt:
                pass
            finally:
                socket.close(0)
                context.destroy(0) 
Example 9
Project: me-ica   Author: ME-ICA   File: test_analyze.py    License: GNU Lesser General Public License v2.1 6 votes vote down vote up
def test_pickle(self):
        # Test that images pickle
        # Image that is not proxied can pickle
        img_klass = self.image_class
        img = img_klass(np.zeros((2,3,4)), None)
        img_str = pickle.dumps(img)
        img2 = pickle.loads(img_str)
        assert_array_equal(img.get_data(), img2.get_data())
        assert_equal(img.get_header(), img2.get_header())
        # Save / reload using bytes IO objects
        for key, value in img.file_map.items():
            value.fileobj = BytesIO()
        img.to_file_map()
        img_prox = img.from_file_map(img.file_map)
        img_str = pickle.dumps(img_prox)
        img2_prox = pickle.loads(img_str)
        assert_array_equal(img.get_data(), img2_prox.get_data()) 
Example 10
Project: pyGSTi   Author: pyGSTio   File: smartcache.py    License: Apache License 2.0 6 votes vote down vote up
def __getstate__(self):
        d = dict(self.__dict__)

        def get_pickleable_dict(cacheDict):
            pickleableCache = dict()
            for k, v in cacheDict.items():
                try:
                    _pickle.dumps(v)
                    pickleableCache[k] = v
                except TypeError as e:
                    if isinstance(v, dict):
                        self.unpickleable.add(str(k[0]) + str(type(v)) + str(e) + str(list(v.keys())))
                    else:
                        self.unpickleable.add(str(k[0]) + str(type(v)) + str(e))  # + str(list(v.__dict__.keys())))
                except _pickle.PicklingError as e:
                    self.unpickleable.add(str(k[0]) + str(type(v)) + str(e))
            return pickleableCache

        d['cache'] = get_pickleable_dict(self.cache)
        d['outargs'] = get_pickleable_dict(self.outargs)
        return d 
Example 11
Project: pyGSTi   Author: pyGSTio   File: test_model.py    License: Apache License 2.0 6 votes vote down vote up
def test_pickle(self):
        # XXX what exactly does this cover and is it needed?  EGN: this tests that the individual pieces (~dicts) within a model can be pickled; it's useful for debuggin b/c often just one of these will break.
        p = pickle.dumps(self.model.preps)
        preps = pickle.loads(p)
        self.assertEqual(list(preps.keys()), list(self.model.preps.keys()))

        p = pickle.dumps(self.model.povms)
        povms = pickle.loads(p)
        self.assertEqual(list(povms.keys()), list(self.model.povms.keys()))

        p = pickle.dumps(self.model.operations)
        gates = pickle.loads(p)
        self.assertEqual(list(gates.keys()), list(self.model.operations.keys()))

        self.model._clean_paramvec()
        p = pickle.dumps(self.model)
        g = pickle.loads(p)
        g._clean_paramvec()
        self.assertAlmostEqual(self.model.frobeniusdist(g), 0.0) 
Example 12
Project: pyGSTi   Author: pyGSTio   File: testHessian.py    License: Apache License 2.0 6 votes vote down vote up
def tets_pickle_ConfidenceRegion(self):
        res = pygsti.obj.Results()
        res.init_dataset(self.ds)
        res.init_circuits(self.gss)
        res.add_estimate(stdxyi.target_model(), stdxyi.target_model(),
                         [self.model]*len(self.maxLengthList), parameters={'objective': 'logl'},
                         estimate_key="default")

        res.add_confidence_region_factory('final iteration estimate', 'final')
        self.assertTrue( res.has_confidence_region_factory('final iteration estimate', 'final'))

        cfctry = res.get_confidence_region_factory('final iteration estimate', 'final')
        cfctry.compute_hessian()
        self.assertTrue( cfctry.has_hessian() )

        cfctry.project_hessian('std')
        ci_std = cfctry.view( 95.0, 'normal', 'std')

        s = pickle.dumps(cfctry)
        cifctry2 = pickle.loads(s)

        s = pickle.dumps(ci_std)
        ci_std2 = pickle.loads(s)

        #TODO: make sure ci_std and ci_std2 are the same 
Example 13
Project: BASS   Author: Cisco-Talos   File: funcdb.py    License: GNU General Public License v2.0 5 votes vote down vote up
def add(self, db, **kwargs):
        db_data = db.data.copy()
        db_data["functions"] = _filter_functions(db.data["functions"], **kwargs)
        data = pickle.dumps(db_data)
        result = requests.post("{:s}/function".format(self.url), files = {"file": BytesIO(data)})
        if result.status_code != 200:
            raise RuntimeError("Request failed with status code {:d}".format(result.status_code)) 
Example 14
Project: BASS   Author: Cisco-Talos   File: funcdb.py    License: GNU General Public License v2.0 5 votes vote down vote up
def find_raw(self, db, **kwargs):
        db_data = db.data.copy()
        db_data["functions"] = _filter_functions(db.data["functions"], **kwargs)
        data = pickle.dumps(db_data)
        result = requests.post("{:s}/function/find/raw".format(self.url), files = {"file": BytesIO(data)})
        if result.status_code == 200:
            return True
        elif result.status_code == 404:
            return False
        else:
            raise RuntimeError("Request failed with status code {:d}".format(result.status_code)) 
Example 15
Project: BASS   Author: Cisco-Talos   File: funcdb.py    License: GNU General Public License v2.0 5 votes vote down vote up
def find_mnem(self, db, **kwargs):
        db_data = db.data.copy()
        db_data["functions"] = _filter_functions(db.data["functions"], **kwargs)
        data = pickle.dumps(db_data)
        result = requests.post("{:s}/function/find/mnem".format(self.url), files = {"file": BytesIO(data)})
        if result.status_code == 200:
            return True
        elif result.status_code == 404:
            return False
        else:
            raise RuntimeError("Request failed with status code {:d}".format(result.status_code)) 
Example 16
Project: ALF   Author: blackberry   File: _qemu.py    License: Apache License 2.0 5 votes vote down vote up
def _remote_gdb_func(cmd, env, timeout, idle_limit):
    return pickle.dumps(_gdb.run_with_gdb(cmd, env=env, timeout=timeout, idle_limit=idle_limit), pickle.HIGHEST_PROTOCOL) 
Example 17
Project: ALF   Author: blackberry   File: _qemu.py    License: Apache License 2.0 5 votes vote down vote up
def _remote_run_func(cmd, env, timeout, memory_limit, idle_limit):
    return pickle.dumps(_common.run(cmd, env=env, timeout=timeout, memory_limit=memory_limit, idle_limit=idle_limit), pickle.HIGHEST_PROTOCOL) 
Example 18
Project: ALF   Author: blackberry   File: SockPuppet.py    License: Apache License 2.0 5 votes vote down vote up
def send_data(self, data):
        is_ack = (data["cmd"] == self.ACK)
        data = pickle.dumps(data, pickle.HIGHEST_PROTOCOL)
        data_len = len(data)
        assert data_len < 0xFFFFFFFF, "Transfer too large!"
        log.debug("-> sending %d bytes", data_len)
        self.conn.sendall(struct.pack("I", data_len))
        self.conn.sendall(data)
        if not is_ack:
            assert self.recv_data()["cmd"] == self.ACK
            log.debug("ACK received") 
Example 19
Project: mmdetection   Author: open-mmlab   File: test.py    License: Apache License 2.0 5 votes vote down vote up
def collect_results_gpu(result_part, size):
    rank, world_size = get_dist_info()
    # dump result part to tensor with pickle
    part_tensor = torch.tensor(
        bytearray(pickle.dumps(result_part)), dtype=torch.uint8, device='cuda')
    # gather all result part tensor shape
    shape_tensor = torch.tensor(part_tensor.shape, device='cuda')
    shape_list = [shape_tensor.clone() for _ in range(world_size)]
    dist.all_gather(shape_list, shape_tensor)
    # padding result part tensor to max length
    shape_max = torch.tensor(shape_list).max()
    part_send = torch.zeros(shape_max, dtype=torch.uint8, device='cuda')
    part_send[:shape_tensor[0]] = part_tensor
    part_recv_list = [
        part_tensor.new_zeros(shape_max) for _ in range(world_size)
    ]
    # gather all result part
    dist.all_gather(part_recv_list, part_send)

    if rank == 0:
        part_list = []
        for recv, shape in zip(part_recv_list, shape_list):
            part_list.append(
                pickle.loads(recv[:shape[0]].cpu().numpy().tobytes()))
        # sort the results
        ordered_results = []
        for res in zip(*part_list):
            ordered_results.extend(list(res))
        # the dataloader may pad some samples
        ordered_results = ordered_results[:size]
        return ordered_results 
Example 20
Project: The-chat-room   Author: 11ze   File: vachat.py    License: MIT License 5 votes vote down vote up
def run(self):
        while True:
            try:
                self.sock.connect(self.ADDR)
                break
            except:
                time.sleep(3)
                continue
        if self.showme:
            cv2.namedWindow('You', cv2.WINDOW_NORMAL)
        print("VEDIO client connected...")
        while self.cap.isOpened():
            ret, frame = self.cap.read()
            if self.showme:
                cv2.imshow('You', frame)
                if cv2.waitKey(1) & 0xFF == 27:
                    self.showme = False
                    cv2.destroyWindow('You')
            sframe = cv2.resize(frame, (0, 0), fx=self.fx, fy=self.fx)
            data = pickle.dumps(sframe)
            zdata = zlib.compress(data, zlib.Z_BEST_COMPRESSION)
            try:
                self.sock.sendall(struct.pack("L", len(zdata)) + zdata)
            except:
                break
            for i in range(self.interval):
                self.cap.read() 
Example 21
Project: fuku-ml   Author: fukuball   File: Utility.py    License: MIT License 5 votes vote down vote up
def serialize(need_serialize=''):
        """serialize"""
        return pickle.dumps(need_serialize, protocol=0) 
Example 22
def test_ndarray_pickle():
    np.random.seed(0)
    maxdim = 5
    nrepeat = 10
    for repeat in range(nrepeat):
        for dim in range(1, maxdim):
            a = random_ndarray(dim)
            b = mx.nd.empty(a.shape)
            a[:] = np.random.uniform(-10, 10, a.shape)
            b[:] = np.random.uniform(-10, 10, a.shape)
            a = a + b
            data = pkl.dumps(a)
            a2 = pkl.loads(data)
            assert np.sum(a.asnumpy() != a2.asnumpy()) == 0 
Example 23
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: optimizer.py    License: Apache License 2.0 5 votes vote down vote up
def get_states(self, dump_optimizer=False):
        """Gets updater states.

        Parameters
        ----------
        dump_optimizer : bool, default False
            Whether to also save the optimizer itself. This would also save optimizer
            information such as learning rate and weight decay schedules.
        """
        return pickle.dumps((self.states, self.optimizer) if dump_optimizer else self.states) 
Example 24
def test_ndarray_pickle():
    maxdim = 5
    for dim in range(1, maxdim):
        a = random_ndarray(dim)
        b = mx.nd.empty(a.shape)
        a[:] = np.random.uniform(-10, 10, a.shape)
        b[:] = np.random.uniform(-10, 10, a.shape)
        a = a + b
        data = pkl.dumps(a)
        a2 = pkl.loads(data)
        assert np.sum(a.asnumpy() != a2.asnumpy()) == 0 
Example 25
def test_symbol_pickle():
    mlist = [models.mlp2(), models.conv()]
    data = pkl.dumps(mlist)
    mlist2 = pkl.loads(data)
    for x, y  in zip(mlist, mlist2):
        assert x.tojson() == y.tojson() 
Example 26
Project: opentracing-python   Author: opentracing   File: binary_propagator.py    License: Apache License 2.0 5 votes vote down vote up
def inject(self, span_context, carrier):
        if type(carrier) is not bytearray:
            raise InvalidCarrierException()

        data = pickle.dumps(span_context)
        carrier.extend(data) 
Example 27
Project: cutout   Author: jojoin   File: rediscache.py    License: MIT License 5 votes vote down vote up
def dump_object(self, value):
        """Dumps an object into a string for redis.  By default it serializes
        integers as regular string and pickle dumps everything else.
        """
        t = type(value)
        if t is int or t is long:
            return str(value)
        return '!' + pickle.dumps(value) 
Example 28
Project: cutout   Author: jojoin   File: memcache.py    License: MIT License 5 votes vote down vote up
def set(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        self._prune()
        self._cache[key] = (time() + timeout, pickle.dumps(value,
            pickle.HIGHEST_PROTOCOL)) 
Example 29
Project: cutout   Author: jojoin   File: memcache.py    License: MIT License 5 votes vote down vote up
def add(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        if len(self._cache) > self._threshold:
            self._prune()
        item = (time() + timeout, pickle.dumps(value,
            pickle.HIGHEST_PROTOCOL))
        self._cache.setdefault(key, item) 
Example 30
Project: clashroyale   Author: cgrok   File: utils.py    License: MIT License 5 votes vote down vote up
def __setitem__(self, key, item):
        with self.connection(True) as con:
            con.execute("insert or replace into `%s` (key,value) values (?,?)" %
                        self.table_name, (key, pickle.dumps(item)))