Python pickle.loads() Examples
The following are 30 code examples for showing how to use pickle.loads(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.
You may check out the related API usage on the sidebar.
You may also want to check out all available functions/classes of the module
pickle
, or try the search function
.
Example 1
Project: vergeml Author: mme File: cache.py License: MIT License | 9 votes |
def _deserialize(self, data, type_): if self.compress: # decompress the data if needed data = lz4.frame.decompress(data) if type_ == _NUMPY: # deserialize numpy arrays buf = io.BytesIO(data) data = np.load(buf) elif type_ == _PICKLE: # deserialize other python objects data = pickle.loads(data) else: # Otherwise we just return data as it is (bytes) pass return data
Example 2
Project: The-chat-room Author: 11ze File: vachat.py License: MIT License | 7 votes |
def run(self): print("VEDIO server starts...") self.sock.bind(self.ADDR) self.sock.listen(1) conn, addr = self.sock.accept() print("remote VEDIO client success connected...") data = "".encode("utf-8") payload_size = struct.calcsize("L") cv2.namedWindow('Remote', cv2.WINDOW_AUTOSIZE) while True: while len(data) < payload_size: data += conn.recv(81920) packed_size = data[:payload_size] data = data[payload_size:] msg_size = struct.unpack("L", packed_size)[0] while len(data) < msg_size: data += conn.recv(81920) zframe_data = data[:msg_size] data = data[msg_size:] frame_data = zlib.decompress(zframe_data) frame = pickle.loads(frame_data) cv2.imshow('Remote', frame) if cv2.waitKey(1) & 0xFF == 27: break
Example 3
Project: ALF Author: blackberry File: SockPuppet.py License: Apache License 2.0 | 6 votes |
def recv_data(self): data_remaining = struct.unpack("I", self.conn.recv(4))[0] if not data_remaining: log.debug("no data?!") return None log.debug("<- recving %d bytes", data_remaining) data = [] while data_remaining: recv_bytes = data_remaining if data_remaining < self.SOCK_BUF else self.SOCK_BUF data.append(self.conn.recv(recv_bytes)) data_len = len(data[-1]) if data_len == 0: break data_remaining -= data_len data = pickle.loads("".join(data)) if data["cmd"] != self.ACK: self.send_ack() return data
Example 4
Project: dynamic-training-with-apache-mxnet-on-aws Author: awslabs File: kvstore_server.py License: Apache License 2.0 | 6 votes |
def _controller(self): """Return the server controller.""" def server_controller(cmd_id, cmd_body, _): """Server controler.""" if not self.init_logginig: # the reason put the codes here is because we cannot get # kvstore.rank earlier head = '%(asctime)-15s Server[' + str( self.kvstore.rank) + '] %(message)s' logging.basicConfig(level=logging.DEBUG, format=head) self.init_logginig = True if cmd_id == 0: try: optimizer = pickle.loads(cmd_body) except: raise self.kvstore.set_optimizer(optimizer) else: print("server %d, unknown command (%d, %s)" % ( self.kvstore.rank, cmd_id, cmd_body)) return server_controller
Example 5
Project: dynamic-training-with-apache-mxnet-on-aws Author: awslabs File: test_attr.py License: Apache License 2.0 | 6 votes |
def test_attr_basic(): with mx.AttrScope(group='4', data='great'): data = mx.symbol.Variable('data', attr={'dtype':'data', 'group': '1', 'force_mirroring': 'True'}, lr_mult=1) gdata = mx.symbol.Variable('data2') assert gdata.attr('group') == '4' assert data.attr('group') == '1' assert data.attr('lr_mult') == '1' assert data.attr('__lr_mult__') == '1' assert data.attr('force_mirroring') == 'True' assert data.attr('__force_mirroring__') == 'True' data2 = pkl.loads(pkl.dumps(data)) assert data.attr('dtype') == data2.attr('dtype')
Example 6
Project: dynamic-training-with-apache-mxnet-on-aws Author: awslabs File: test_sparse_ndarray.py License: Apache License 2.0 | 6 votes |
def test_sparse_nd_pickle(): dim0 = 40 dim1 = 40 stypes = ['row_sparse', 'csr'] densities = [0, 0.5] stype_dict = {'row_sparse': RowSparseNDArray, 'csr': CSRNDArray} shape = rand_shape_2d(dim0, dim1) for stype in stypes: for density in densities: a, _ = rand_sparse_ndarray(shape, stype, density) assert isinstance(a, stype_dict[stype]) data = pkl.dumps(a) b = pkl.loads(data) assert isinstance(b, stype_dict[stype]) assert same(a.asnumpy(), b.asnumpy()) # @kalyc: Getting rid of fixed seed as flakiness could not be reproduced # tracked at https://github.com/apache/incubator-mxnet/issues/11741
Example 7
Project: Dumb-Cogs Author: irdumbs File: adventure.py License: MIT License | 6 votes |
def load(self, ctx, save=None): """loads current team's save. defaults to most recent""" server = ctx.message.server author = ctx.message.author channel = ctx.message.channel try: team = self.splayers[server.id][channel.id][author.id] except: team = None await self.embark.callback(self, ctx, team, save) # @adventure.command(pass_context=True) # async def save(self, ctx, file): # pass # if no team and no save, if user doesn't have a save, new game. otherwise new game must specify team and save
Example 8
Project: pywren-ibm-cloud Author: pywren File: jobrunner.py License: Apache License 2.0 | 6 votes |
def _load_data(self): extra_get_args = {} if self.data_byte_range is not None: range_str = 'bytes={}-{}'.format(*self.data_byte_range) extra_get_args['Range'] = range_str logger.debug("Getting function data") data_download_start_tstamp = time.time() data_obj = self.internal_storage.get_data(self.data_key, extra_get_args=extra_get_args) logger.debug("Finished getting Function data") logger.debug("Unpickle Function data") loaded_data = pickle.loads(data_obj) logger.debug("Finished unpickle Function data") data_download_end_tstamp = time.time() self.stats.write('data_download_time', round(data_download_end_tstamp-data_download_start_tstamp, 8)) return loaded_data
Example 9
Project: jawfish Author: war-and-code File: test_case.py License: MIT License | 6 votes |
def testPickle(self): # Issue 10326 # Can't use TestCase classes defined in Test class as # pickle does not work with inner classes test = unittest.TestCase('run') for protocol in range(pickle.HIGHEST_PROTOCOL + 1): # blew up prior to fix pickled_test = pickle.dumps(test, protocol=protocol) unpickled_test = pickle.loads(pickled_test) self.assertEqual(test, unpickled_test) # exercise the TestCase instance in a way that will invoke # the type equality lookup mechanism unpickled_test.assertEqual(set(), set())
Example 10
Project: me-ica Author: ME-ICA File: test_analyze.py License: GNU Lesser General Public License v2.1 | 6 votes |
def test_pickle(self): # Test that images pickle # Image that is not proxied can pickle img_klass = self.image_class img = img_klass(np.zeros((2,3,4)), None) img_str = pickle.dumps(img) img2 = pickle.loads(img_str) assert_array_equal(img.get_data(), img2.get_data()) assert_equal(img.get_header(), img2.get_header()) # Save / reload using bytes IO objects for key, value in img.file_map.items(): value.fileobj = BytesIO() img.to_file_map() img_prox = img.from_file_map(img.file_map) img_str = pickle.dumps(img_prox) img2_prox = pickle.loads(img_str) assert_array_equal(img.get_data(), img2_prox.get_data())
Example 11
Project: iSDX Author: sdn-ixp File: logServer.py License: Apache License 2.0 | 5 votes |
def handle_read(self): try: data = self.recv(self.dlen) if len(data) == 0: return except socket.error as e: if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN): return self.data += data self.dlen -= len(data) if self.dlen > 0: # don't have complete record yet. wait for more data to read return if self.rlen == 0: self.dlen = self.rlen = struct.unpack('>L', self.data)[0] self.data = '' # got record length. now read record return # got complete record obj = pickle.loads(self.data) record = logging.makeLogRecord(obj) # Note: EVERY record gets logged. This is because Logger.handle # is normally called AFTER logger-level filtering. # Filter (e.g., only WARNING or higher) # at the sender to save network bandwidth. globalLogger.handle(record) # reset for next record self.data = '' self.rlen = 0 self.dlen = 4
Example 12
Project: mmdetection Author: open-mmlab File: test.py License: Apache License 2.0 | 5 votes |
def collect_results_gpu(result_part, size): rank, world_size = get_dist_info() # dump result part to tensor with pickle part_tensor = torch.tensor( bytearray(pickle.dumps(result_part)), dtype=torch.uint8, device='cuda') # gather all result part tensor shape shape_tensor = torch.tensor(part_tensor.shape, device='cuda') shape_list = [shape_tensor.clone() for _ in range(world_size)] dist.all_gather(shape_list, shape_tensor) # padding result part tensor to max length shape_max = torch.tensor(shape_list).max() part_send = torch.zeros(shape_max, dtype=torch.uint8, device='cuda') part_send[:shape_tensor[0]] = part_tensor part_recv_list = [ part_tensor.new_zeros(shape_max) for _ in range(world_size) ] # gather all result part dist.all_gather(part_recv_list, part_send) if rank == 0: part_list = [] for recv, shape in zip(part_recv_list, shape_list): part_list.append( pickle.loads(recv[:shape[0]].cpu().numpy().tobytes())) # sort the results ordered_results = [] for res in zip(*part_list): ordered_results.extend(list(res)) # the dataloader may pad some samples ordered_results = ordered_results[:size] return ordered_results
Example 13
Project: python-clean-architecture Author: pcah File: test_ordered_set.py License: MIT License | 5 votes |
def test_pickle(): set1 = OrderedSet('abracadabra') roundtrip = pickle.loads(pickle.dumps(set1)) assert roundtrip == set1
Example 14
Project: python-clean-architecture Author: pcah File: test_ordered_set.py License: MIT License | 5 votes |
def test_empty_pickle(): empty_oset = OrderedSet() empty_roundtrip = pickle.loads(pickle.dumps(empty_oset)) assert empty_roundtrip == empty_oset
Example 15
Project: The-chat-room Author: 11ze File: vachat.py License: MIT License | 5 votes |
def run(self): global TERMINATE print("AUDIO server starts...") self.sock.bind(self.ADDR) self.sock.listen(1) conn, addr = self.sock.accept() print("remote AUDIO client success connected...") data = "".encode("utf-8") payload_size = struct.calcsize("L") self.stream = self.p.open(format=FORMAT, channels=CHANNELS, rate=RATE, output=True, frames_per_buffer=CHUNK ) while True: if TERMINATE: self.sock.close() break while len(data) < payload_size: data += conn.recv(81920) packed_size = data[:payload_size] data = data[payload_size:] msg_size = struct.unpack("L", packed_size)[0] while len(data) < msg_size: data += conn.recv(81920) frame_data = data[:msg_size] data = data[msg_size:] frames = pickle.loads(frame_data) for frame in frames: self.stream.write(frame, CHUNK)
Example 16
Project: fuku-ml Author: fukuball File: Utility.py License: MIT License | 5 votes |
def deserialize(pickle_serialized=''): """ddserialize""" return pickle.loads(pickle_serialized)
Example 17
Project: dynamic-training-with-apache-mxnet-on-aws Author: awslabs File: profiler_ndarray.py License: Apache License 2.0 | 5 votes |
def test_ndarray_pickle(): np.random.seed(0) maxdim = 5 nrepeat = 10 for repeat in range(nrepeat): for dim in range(1, maxdim): a = random_ndarray(dim) b = mx.nd.empty(a.shape) a[:] = np.random.uniform(-10, 10, a.shape) b[:] = np.random.uniform(-10, 10, a.shape) a = a + b data = pkl.dumps(a) a2 = pkl.loads(data) assert np.sum(a.asnumpy() != a2.asnumpy()) == 0
Example 18
Project: dynamic-training-with-apache-mxnet-on-aws Author: awslabs File: dataloader.py License: Apache License 2.0 | 5 votes |
def recv(self): """Receive object""" buf = self.recv_bytes() return pickle.loads(buf)
Example 19
Project: dynamic-training-with-apache-mxnet-on-aws Author: awslabs File: optimizer.py License: Apache License 2.0 | 5 votes |
def set_states(self, states): """Sets updater states.""" states = pickle.loads(states) if isinstance(states, tuple) and len(states) == 2: self.states, self.optimizer = states else: self.states = states self.states_synced = dict.fromkeys(self.states.keys(), False)
Example 20
Project: dynamic-training-with-apache-mxnet-on-aws Author: awslabs File: test_ndarray.py License: Apache License 2.0 | 5 votes |
def test_ndarray_pickle(): maxdim = 5 for dim in range(1, maxdim): a = random_ndarray(dim) b = mx.nd.empty(a.shape) a[:] = np.random.uniform(-10, 10, a.shape) b[:] = np.random.uniform(-10, 10, a.shape) a = a + b data = pkl.dumps(a) a2 = pkl.loads(data) assert np.sum(a.asnumpy() != a2.asnumpy()) == 0
Example 21
Project: dynamic-training-with-apache-mxnet-on-aws Author: awslabs File: test_attr.py License: Apache License 2.0 | 5 votes |
def test_operator(): data = mx.symbol.Variable('data') with mx.AttrScope(__group__='4', __data__='great'): fc1 = mx.symbol.Activation(data, act_type='relu') with mx.AttrScope(__init_bias__='0.0'): fc2 = mx.symbol.FullyConnected(fc1, num_hidden=10, name='fc2') assert fc1.attr('__data__') == 'great' assert fc2.attr('__data__') == 'great' assert fc2.attr('__init_bias__') == '0.0' fc2copy = pkl.loads(pkl.dumps(fc2)) assert fc2copy.tojson() == fc2.tojson() fc2weight = fc2.get_internals()['fc2_weight']
Example 22
Project: iris Author: doitintl File: main.py License: MIT License | 5 votes |
def retrieve(key): result = memcache.get_multi(['%s.%s' % (key, i) for i in xrange(32)]) serialized = ''.join( [v for k, v in sorted(result.items()) if v is not None]) try: return pickle.loads(serialized) except EOFError as e: logging.info(e) return None
Example 23
Project: Dumb-Cogs Author: irdumbs File: adventure.py License: MIT License | 5 votes |
def resume(self, obj): """Returns an Adventure game saved to the given file.""" if isinstance(obj, str): savefile = open(obj, 'rb') else: savefile = obj game = pickle.loads(zlib.decompress(savefile.read())) if savefile is not obj: savefile.close() # Reinstate the random number generator. game.random_generator = random.Random() game.random_generator.setstate(game.random_state) del game.random_state return game
Example 24
Project: Dumb-Cogs Author: irdumbs File: adventure.py License: MIT License | 5 votes |
def save_game(self, server, team, channel, save): if team is None: raise NoTeam() if save is None: raise NoSave() bp = 'data/adventure/saves' team = self._safe_path(team) teamf = team.lower() save = self._safe_path(save) try: if not self.game_loops[server.id][teamf][channel.id]['PLAYING']: raise NoGameRunning() except: raise NoGameRunning('{}: {} doesn\'t have a game running in {}'.format(server.id, team, channel.mention)) # saves # try: # del self.saves[server.id][teamf][self.game_loops[server.id][teamf][channel.id]['SAVE']] # except: # print('troubles deleting-------------') self.saves[server.id][teamf][save.lower()] = channel.id self.game_loops[server.id][teamf][channel.id]['SAVE'] = save return self.game_loops[server.id][teamf][channel.id]['GAME'].t_suspend('save','{}/{}/{}/{}.save'.format(bp, server.id, teamf, save.lower())) # caller must specify team or choose default. # loads save into channel. if none given, loads most recent save # http://stackoverflow.com/questions/18279063/python-find-newest-file-with-mp3-extension-in-directory # throws FileNotFoundError if save doesn't exist
Example 25
Project: opentracing-python Author: opentracing File: binary_propagator.py License: Apache License 2.0 | 5 votes |
def extract(self, carrier): if type(carrier) is not bytearray: raise InvalidCarrierException() try: span_context = pickle.loads(carrier) except (EOFError, pickle.PickleError): raise SpanContextCorruptedException() return span_context
Example 26
Project: lirpg Author: Hwhitetooth File: __init__.py License: MIT License | 5 votes |
def __setstate__(self, ob): import pickle self.x = pickle.loads(ob)
Example 27
Project: A2C Author: lnpalmer File: envs.py License: MIT License | 5 votes |
def __setstate__(self, ob): import pickle self.x = pickle.loads(ob)
Example 28
Project: cutout Author: jojoin File: rediscache.py License: MIT License | 5 votes |
def load_object(self, value): """The reversal of :meth:`dump_object`. This might be callde with None. """ if value is None: return None if value.startswith('!'): return pickle.loads(value[1:]) try: return int(value) except ValueError: # before 0.8 we did not have serialization. Still support that. return value
Example 29
Project: cutout Author: jojoin File: memcache.py License: MIT License | 5 votes |
def get(self, key): now = time() expires, value = self._cache.get(key, (0, None)) if expires > time(): return pickle.loads(value)
Example 30
Project: HardRLWithYoutube Author: MaxSobolMark File: __init__.py License: MIT License | 5 votes |
def __setstate__(self, ob): import pickle self.x = pickle.loads(ob)