Python pickle.loads() Examples

The following are 30 code examples for showing how to use pickle.loads(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module pickle , or try the search function .

Example 1
Project: vergeml   Author: mme   File: cache.py    License: MIT License 9 votes vote down vote up
def _deserialize(self, data, type_):

        if self.compress:
        # decompress the data if needed
            data = lz4.frame.decompress(data)

        if type_ == _NUMPY:
        # deserialize numpy arrays
            buf = io.BytesIO(data)
            data = np.load(buf)

        elif type_ == _PICKLE:
        # deserialize other python objects
            data = pickle.loads(data)

        else:
        # Otherwise we just return data as it is (bytes)
            pass

        return data 
Example 2
Project: The-chat-room   Author: 11ze   File: vachat.py    License: MIT License 7 votes vote down vote up
def run(self):
        print("VEDIO server starts...")
        self.sock.bind(self.ADDR)
        self.sock.listen(1)
        conn, addr = self.sock.accept()
        print("remote VEDIO client success connected...")
        data = "".encode("utf-8")
        payload_size = struct.calcsize("L")
        cv2.namedWindow('Remote', cv2.WINDOW_AUTOSIZE)
        while True:
            while len(data) < payload_size:
                data += conn.recv(81920)
            packed_size = data[:payload_size]
            data = data[payload_size:]
            msg_size = struct.unpack("L", packed_size)[0]
            while len(data) < msg_size:
                data += conn.recv(81920)
            zframe_data = data[:msg_size]
            data = data[msg_size:]
            frame_data = zlib.decompress(zframe_data)
            frame = pickle.loads(frame_data)
            cv2.imshow('Remote', frame)
            if cv2.waitKey(1) & 0xFF == 27:
                break 
Example 3
Project: ALF   Author: blackberry   File: SockPuppet.py    License: Apache License 2.0 6 votes vote down vote up
def recv_data(self):
        data_remaining = struct.unpack("I", self.conn.recv(4))[0]
        if not data_remaining:
            log.debug("no data?!")
            return None
        log.debug("<- recving %d bytes", data_remaining)
        data = []
        while data_remaining:
            recv_bytes = data_remaining if data_remaining < self.SOCK_BUF else self.SOCK_BUF
            data.append(self.conn.recv(recv_bytes))
            data_len = len(data[-1])
            if data_len == 0:
                break
            data_remaining -= data_len
        data = pickle.loads("".join(data))
        if data["cmd"] != self.ACK:
            self.send_ack()
        return data 
Example 4
def _controller(self):
        """Return the server controller."""
        def server_controller(cmd_id, cmd_body, _):
            """Server controler."""
            if not self.init_logginig:
                # the reason put the codes here is because we cannot get
                # kvstore.rank earlier
                head = '%(asctime)-15s Server[' + str(
                    self.kvstore.rank) + '] %(message)s'
                logging.basicConfig(level=logging.DEBUG, format=head)
                self.init_logginig = True

            if cmd_id == 0:
                try:
                    optimizer = pickle.loads(cmd_body)
                except:
                    raise
                self.kvstore.set_optimizer(optimizer)
            else:
                print("server %d, unknown command (%d, %s)" % (
                    self.kvstore.rank, cmd_id, cmd_body))
        return server_controller 
Example 5
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_attr.py    License: Apache License 2.0 6 votes vote down vote up
def test_attr_basic():
    with mx.AttrScope(group='4', data='great'):
        data = mx.symbol.Variable('data',
                                  attr={'dtype':'data',
                                        'group': '1',
                                        'force_mirroring': 'True'},
                                  lr_mult=1)
        gdata = mx.symbol.Variable('data2')
    assert gdata.attr('group') == '4'
    assert data.attr('group') == '1'
    assert data.attr('lr_mult') == '1'
    assert data.attr('__lr_mult__') == '1'
    assert data.attr('force_mirroring') == 'True'
    assert data.attr('__force_mirroring__') == 'True'
    data2 = pkl.loads(pkl.dumps(data))
    assert data.attr('dtype') == data2.attr('dtype') 
Example 6
def test_sparse_nd_pickle():
    dim0 = 40
    dim1 = 40
    stypes = ['row_sparse', 'csr']
    densities = [0, 0.5]
    stype_dict = {'row_sparse': RowSparseNDArray, 'csr': CSRNDArray}
    shape = rand_shape_2d(dim0, dim1)
    for stype in stypes:
        for density in densities:
            a, _ = rand_sparse_ndarray(shape, stype, density)
            assert isinstance(a, stype_dict[stype])
            data = pkl.dumps(a)
            b = pkl.loads(data)
            assert isinstance(b, stype_dict[stype])
            assert same(a.asnumpy(), b.asnumpy())


# @kalyc: Getting rid of fixed seed as flakiness could not be reproduced
# tracked at https://github.com/apache/incubator-mxnet/issues/11741 
Example 7
Project: Dumb-Cogs   Author: irdumbs   File: adventure.py    License: MIT License 6 votes vote down vote up
def load(self, ctx, save=None):
        """loads current team's save. defaults to most recent"""
        server = ctx.message.server
        author = ctx.message.author
        channel = ctx.message.channel

        try:
            team = self.splayers[server.id][channel.id][author.id]
        except:
            team = None

        await self.embark.callback(self, ctx, team, save)
        

    # @adventure.command(pass_context=True)
    # async def save(self, ctx, file):
    #     pass

    # if no team and no save, if user doesn't have a save, new game. otherwise new game must specify team and save 
Example 8
Project: pywren-ibm-cloud   Author: pywren   File: jobrunner.py    License: Apache License 2.0 6 votes vote down vote up
def _load_data(self):
        extra_get_args = {}
        if self.data_byte_range is not None:
            range_str = 'bytes={}-{}'.format(*self.data_byte_range)
            extra_get_args['Range'] = range_str

        logger.debug("Getting function data")
        data_download_start_tstamp = time.time()
        data_obj = self.internal_storage.get_data(self.data_key, extra_get_args=extra_get_args)
        logger.debug("Finished getting Function data")
        logger.debug("Unpickle Function data")
        loaded_data = pickle.loads(data_obj)
        logger.debug("Finished unpickle Function data")
        data_download_end_tstamp = time.time()
        self.stats.write('data_download_time', round(data_download_end_tstamp-data_download_start_tstamp, 8))

        return loaded_data 
Example 9
Project: jawfish   Author: war-and-code   File: test_case.py    License: MIT License 6 votes vote down vote up
def testPickle(self):
        # Issue 10326

        # Can't use TestCase classes defined in Test class as
        # pickle does not work with inner classes
        test = unittest.TestCase('run')
        for protocol in range(pickle.HIGHEST_PROTOCOL + 1):

            # blew up prior to fix
            pickled_test = pickle.dumps(test, protocol=protocol)
            unpickled_test = pickle.loads(pickled_test)
            self.assertEqual(test, unpickled_test)

            # exercise the TestCase instance in a way that will invoke
            # the type equality lookup mechanism
            unpickled_test.assertEqual(set(), set()) 
Example 10
Project: me-ica   Author: ME-ICA   File: test_analyze.py    License: GNU Lesser General Public License v2.1 6 votes vote down vote up
def test_pickle(self):
        # Test that images pickle
        # Image that is not proxied can pickle
        img_klass = self.image_class
        img = img_klass(np.zeros((2,3,4)), None)
        img_str = pickle.dumps(img)
        img2 = pickle.loads(img_str)
        assert_array_equal(img.get_data(), img2.get_data())
        assert_equal(img.get_header(), img2.get_header())
        # Save / reload using bytes IO objects
        for key, value in img.file_map.items():
            value.fileobj = BytesIO()
        img.to_file_map()
        img_prox = img.from_file_map(img.file_map)
        img_str = pickle.dumps(img_prox)
        img2_prox = pickle.loads(img_str)
        assert_array_equal(img.get_data(), img2_prox.get_data()) 
Example 11
Project: iSDX   Author: sdn-ixp   File: logServer.py    License: Apache License 2.0 5 votes vote down vote up
def handle_read(self):
        try:
            data = self.recv(self.dlen)
            if len(data) == 0:
                return
        except socket.error as e:
            if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
                return

        self.data += data
        self.dlen -= len(data)
        if self.dlen > 0:
            # don't have complete record yet. wait for more data to read
            return

        if self.rlen == 0:
            self.dlen = self.rlen = struct.unpack('>L', self.data)[0]
            self.data = ''
            # got record length. now read record
            return

        # got complete record
        obj = pickle.loads(self.data)
        record = logging.makeLogRecord(obj)

        # Note: EVERY record gets logged. This is because Logger.handle
        # is normally called AFTER logger-level filtering.
        # Filter (e.g., only WARNING or higher)
        # at the sender to save network bandwidth.
        globalLogger.handle(record)

        # reset for next record
        self.data = ''
        self.rlen = 0
        self.dlen = 4 
Example 12
Project: mmdetection   Author: open-mmlab   File: test.py    License: Apache License 2.0 5 votes vote down vote up
def collect_results_gpu(result_part, size):
    rank, world_size = get_dist_info()
    # dump result part to tensor with pickle
    part_tensor = torch.tensor(
        bytearray(pickle.dumps(result_part)), dtype=torch.uint8, device='cuda')
    # gather all result part tensor shape
    shape_tensor = torch.tensor(part_tensor.shape, device='cuda')
    shape_list = [shape_tensor.clone() for _ in range(world_size)]
    dist.all_gather(shape_list, shape_tensor)
    # padding result part tensor to max length
    shape_max = torch.tensor(shape_list).max()
    part_send = torch.zeros(shape_max, dtype=torch.uint8, device='cuda')
    part_send[:shape_tensor[0]] = part_tensor
    part_recv_list = [
        part_tensor.new_zeros(shape_max) for _ in range(world_size)
    ]
    # gather all result part
    dist.all_gather(part_recv_list, part_send)

    if rank == 0:
        part_list = []
        for recv, shape in zip(part_recv_list, shape_list):
            part_list.append(
                pickle.loads(recv[:shape[0]].cpu().numpy().tobytes()))
        # sort the results
        ordered_results = []
        for res in zip(*part_list):
            ordered_results.extend(list(res))
        # the dataloader may pad some samples
        ordered_results = ordered_results[:size]
        return ordered_results 
Example 13
Project: python-clean-architecture   Author: pcah   File: test_ordered_set.py    License: MIT License 5 votes vote down vote up
def test_pickle():
    set1 = OrderedSet('abracadabra')
    roundtrip = pickle.loads(pickle.dumps(set1))
    assert roundtrip == set1 
Example 14
Project: python-clean-architecture   Author: pcah   File: test_ordered_set.py    License: MIT License 5 votes vote down vote up
def test_empty_pickle():
    empty_oset = OrderedSet()
    empty_roundtrip = pickle.loads(pickle.dumps(empty_oset))
    assert empty_roundtrip == empty_oset 
Example 15
Project: The-chat-room   Author: 11ze   File: vachat.py    License: MIT License 5 votes vote down vote up
def run(self):
        global TERMINATE
        print("AUDIO server starts...")
        self.sock.bind(self.ADDR)
        self.sock.listen(1)
        conn, addr = self.sock.accept()
        print("remote AUDIO client success connected...")
        data = "".encode("utf-8")
        payload_size = struct.calcsize("L")
        self.stream = self.p.open(format=FORMAT,
                                  channels=CHANNELS,
                                  rate=RATE,
                                  output=True,
                                  frames_per_buffer=CHUNK
                                  )
        while True:
            if TERMINATE:
                self.sock.close()
                break
            while len(data) < payload_size:
                data += conn.recv(81920)
            packed_size = data[:payload_size]
            data = data[payload_size:]
            msg_size = struct.unpack("L", packed_size)[0]
            while len(data) < msg_size:
                data += conn.recv(81920)
            frame_data = data[:msg_size]
            data = data[msg_size:]
            frames = pickle.loads(frame_data)
            for frame in frames:
                self.stream.write(frame, CHUNK) 
Example 16
Project: fuku-ml   Author: fukuball   File: Utility.py    License: MIT License 5 votes vote down vote up
def deserialize(pickle_serialized=''):
        """ddserialize"""
        return pickle.loads(pickle_serialized) 
Example 17
def test_ndarray_pickle():
    np.random.seed(0)
    maxdim = 5
    nrepeat = 10
    for repeat in range(nrepeat):
        for dim in range(1, maxdim):
            a = random_ndarray(dim)
            b = mx.nd.empty(a.shape)
            a[:] = np.random.uniform(-10, 10, a.shape)
            b[:] = np.random.uniform(-10, 10, a.shape)
            a = a + b
            data = pkl.dumps(a)
            a2 = pkl.loads(data)
            assert np.sum(a.asnumpy() != a2.asnumpy()) == 0 
Example 18
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: dataloader.py    License: Apache License 2.0 5 votes vote down vote up
def recv(self):
        """Receive object"""
        buf = self.recv_bytes()
        return pickle.loads(buf) 
Example 19
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: optimizer.py    License: Apache License 2.0 5 votes vote down vote up
def set_states(self, states):
        """Sets updater states."""
        states = pickle.loads(states)
        if isinstance(states, tuple) and len(states) == 2:
            self.states, self.optimizer = states
        else:
            self.states = states
        self.states_synced = dict.fromkeys(self.states.keys(), False) 
Example 20
def test_ndarray_pickle():
    maxdim = 5
    for dim in range(1, maxdim):
        a = random_ndarray(dim)
        b = mx.nd.empty(a.shape)
        a[:] = np.random.uniform(-10, 10, a.shape)
        b[:] = np.random.uniform(-10, 10, a.shape)
        a = a + b
        data = pkl.dumps(a)
        a2 = pkl.loads(data)
        assert np.sum(a.asnumpy() != a2.asnumpy()) == 0 
Example 21
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: test_attr.py    License: Apache License 2.0 5 votes vote down vote up
def test_operator():
    data = mx.symbol.Variable('data')
    with mx.AttrScope(__group__='4', __data__='great'):
        fc1 = mx.symbol.Activation(data, act_type='relu')
        with mx.AttrScope(__init_bias__='0.0'):
            fc2 = mx.symbol.FullyConnected(fc1, num_hidden=10, name='fc2')
    assert fc1.attr('__data__') == 'great'
    assert fc2.attr('__data__') == 'great'
    assert fc2.attr('__init_bias__') == '0.0'
    fc2copy = pkl.loads(pkl.dumps(fc2))
    assert fc2copy.tojson() == fc2.tojson()
    fc2weight = fc2.get_internals()['fc2_weight'] 
Example 22
Project: iris   Author: doitintl   File: main.py    License: MIT License 5 votes vote down vote up
def retrieve(key):
    result = memcache.get_multi(['%s.%s' % (key, i) for i in xrange(32)])
    serialized = ''.join(
        [v for k, v in sorted(result.items()) if v is not None])
    try:
        return pickle.loads(serialized)
    except EOFError as e:
        logging.info(e)
        return None 
Example 23
Project: Dumb-Cogs   Author: irdumbs   File: adventure.py    License: MIT License 5 votes vote down vote up
def resume(self, obj):
        """Returns an Adventure game saved to the given file."""
        if isinstance(obj, str):
            savefile = open(obj, 'rb')
        else:
            savefile = obj
        game = pickle.loads(zlib.decompress(savefile.read()))
        if savefile is not obj:
            savefile.close()
        # Reinstate the random number generator.
        game.random_generator = random.Random()
        game.random_generator.setstate(game.random_state)
        del game.random_state
        return game 
Example 24
Project: Dumb-Cogs   Author: irdumbs   File: adventure.py    License: MIT License 5 votes vote down vote up
def save_game(self, server, team, channel, save):
        if team is None:
            raise NoTeam()
        if save is None:
            raise NoSave()

        bp = 'data/adventure/saves'
        team = self._safe_path(team)
        teamf = team.lower()
        save = self._safe_path(save)
        try:
            if not self.game_loops[server.id][teamf][channel.id]['PLAYING']:
                raise NoGameRunning()
        except:
            raise NoGameRunning('{}: {} doesn\'t have a game running in {}'.format(server.id, team, channel.mention))

        # saves
        # try:
        #     del self.saves[server.id][teamf][self.game_loops[server.id][teamf][channel.id]['SAVE']]
        # except:
        #     print('troubles deleting-------------')
        self.saves[server.id][teamf][save.lower()] = channel.id

        self.game_loops[server.id][teamf][channel.id]['SAVE'] = save

        return self.game_loops[server.id][teamf][channel.id]['GAME'].t_suspend('save','{}/{}/{}/{}.save'.format(bp, server.id, teamf, save.lower()))
        


    # caller must specify team or choose default.
    # loads save into channel. if none given, loads most recent save
    # http://stackoverflow.com/questions/18279063/python-find-newest-file-with-mp3-extension-in-directory
    # throws FileNotFoundError if save doesn't exist 
Example 25
Project: opentracing-python   Author: opentracing   File: binary_propagator.py    License: Apache License 2.0 5 votes vote down vote up
def extract(self, carrier):
        if type(carrier) is not bytearray:
            raise InvalidCarrierException()

        try:
            span_context = pickle.loads(carrier)
        except (EOFError, pickle.PickleError):
            raise SpanContextCorruptedException()

        return span_context 
Example 26
Project: lirpg   Author: Hwhitetooth   File: __init__.py    License: MIT License 5 votes vote down vote up
def __setstate__(self, ob):
        import pickle
        self.x = pickle.loads(ob) 
Example 27
Project: A2C   Author: lnpalmer   File: envs.py    License: MIT License 5 votes vote down vote up
def __setstate__(self, ob):
        import pickle
        self.x = pickle.loads(ob) 
Example 28
Project: cutout   Author: jojoin   File: rediscache.py    License: MIT License 5 votes vote down vote up
def load_object(self, value):
        """The reversal of :meth:`dump_object`.  This might be callde with
        None.
        """
        if value is None:
            return None
        if value.startswith('!'):
            return pickle.loads(value[1:])
        try:
            return int(value)
        except ValueError:
            # before 0.8 we did not have serialization.  Still support that.
            return value 
Example 29
Project: cutout   Author: jojoin   File: memcache.py    License: MIT License 5 votes vote down vote up
def get(self, key):
        now = time()
        expires, value = self._cache.get(key, (0, None))
        if expires > time():
            return pickle.loads(value) 
Example 30
Project: HardRLWithYoutube   Author: MaxSobolMark   File: __init__.py    License: MIT License 5 votes vote down vote up
def __setstate__(self, ob):
        import pickle
        self.x = pickle.loads(ob)