Python six.moves.cPickle.loads() Examples

def get_project_settings():
    if ENVVAR not in os.environ:
        project = os.environ.get('SCRAPY_PROJECT', 'default')
        init_env(project)

    settings = Settings()
    settings_module_path = os.environ.get(ENVVAR)
    if settings_module_path:
        settings.setmodule(settings_module_path, priority='project')

    # XXX: remove this hack
    pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
    if pickled_settings:
        settings.setdict(pickle.loads(pickled_settings), priority='project')

    # XXX: deprecate and remove this functionality
    env_overrides = {k[7:]: v for k, v in os.environ.items() if
                     k.startswith('SCRAPY_')}
    if env_overrides:
        settings.setdict(env_overrides, priority='project')

    return settings 

def _read_data(self, spider, request):
        key = self._request_key(request)
        try:
            ts = self.db.Get(key + b'_time')
        except KeyError:
            return  # not found or invalid entry

        if 0 < self.expiration_secs < time() - float(ts):
            return  # expired

        try:
            data = self.db.Get(key + b'_data')
        except KeyError:
            return  # invalid entry
        else:
            return pickle.loads(data) 

def test_xrange():
    yield assert_equal, list(xrange(10)), list(_xrange(10))
    yield assert_equal, list(xrange(10, 15)), list(_xrange(10, 15))
    yield assert_equal, list(xrange(10, 20, 2)), list(_xrange(10, 20, 2))
    yield assert_equal, list(xrange(5, 1, -1)), list(_xrange(5, 1, -1))
    yield (assert_equal, list(xrange(5, 55, 3)),
           list(cPickle.loads(cPickle.dumps(_xrange(5, 55, 3)))))
    yield assert_equal, _xrange(5).index(4), 4
    yield assert_equal, _xrange(5, 9).index(6), 1
    yield assert_equal, _xrange(8, 24, 3).index(11), 1
    yield assert_equal, _xrange(25, 4, -5).index(25), 0
    yield assert_equal, _xrange(28, 7, -7).index(14), 2
    yield assert_raises, ValueError, _xrange(2, 9, 2).index, 3
    yield assert_raises, ValueError, _xrange(2, 20, 2).index, 9
    yield assert_equal, _xrange(5).count(5), 0
    yield assert_equal, _xrange(5).count(4), 1
    yield assert_equal, _xrange(4, 9).count(4), 1
    yield assert_equal, _xrange(3, 9, 2).count(4), 0
    yield assert_equal, _xrange(3, 9, 2).count(5), 1
    yield assert_equal, _xrange(3, 9, 2).count(20), 0
    yield assert_equal, _xrange(9, 3).count(5), 0
    yield assert_equal, _xrange(3, 10, -1).count(5), 0
    yield assert_equal, _xrange(10, 3, -1).count(5), 1
    yield assert_equal, _xrange(10, 0, -2).count(6), 1
    yield assert_equal, _xrange(10, -1, -3).count(7), 1 

def best_models(self, nb_models, model, data, max_evals):
        trials_list = self.compute_trials(model, data, max_evals)
        num_trials = sum(len(trials) for trials in trials_list)
        if num_trials < nb_models:
            nb_models = len(trials_list)
        scores = []
        for trials in trials_list:
            scores = scores + [trial.get('result').get('loss')
                               for trial in trials]
        cut_off = sorted(scores, reverse=True)[nb_models - 1]
        model_list = []
        for trials in trials_list:
            for trial in trials:
                if trial.get('result').get('loss') >= cut_off:
                    model = model_from_yaml(trial.get('result').get('model'))
                    model.set_weights(pickle.loads(
                        trial.get('result').get('weights')))
                    model_list.append(model)
        return model_list 

def verify_tee(n, original, seed):
    try:
        state = random.getstate()
        iterators = list(tee(original, n=n))
        results = [[] for _ in range(n)]
        exhausted = [False] * n
        while not all(exhausted):
            # Upper argument of random.randint is inclusive. Argh.
            i = random.randint(0, n - 1)
            if not exhausted[i]:
                if len(results[i]) == len(original):
                    assert_raises(StopIteration, next, iterators[i])
                    assert results[i] == original
                    exhausted[i] = True
                else:
                    if random.randint(0, 1):
                        iterators[i] = cPickle.loads(
                            cPickle.dumps(iterators[i]))
                    elem = next(iterators[i])
                    results[i].append(elem)
    finally:
        random.setstate(state) 

def minimize(self, model, data, max_evals, notebook_name=None):
        global best_model_yaml, best_model_weights

        trials_list = self.compute_trials(
            model, data, max_evals, notebook_name)

        best_val = 1e7
        for trials in trials_list:
            for trial in trials:
                val = trial.get('result').get('loss')
                if val < best_val:
                    best_val = val
                    best_model_yaml = trial.get('result').get('model')
                    best_model_weights = trial.get('result').get('weights')

        best_model = model_from_yaml(best_model_yaml)
        best_model.set_weights(pickle.loads(best_model_weights))

        return best_model 

def test_pickling(self, backend_config):
        x_data, = self.generate_inputs()

        link = self.create_link(self.generate_params())
        link.to_device(backend_config.device)

        x = chainer.Variable(x_data)
        x.to_device(backend_config.device)

        y = link(x)
        y_data1 = y.data
        del x, y
        pickled = pickle.dumps(link, -1)
        del link
        link = pickle.loads(pickled)
        x = chainer.Variable(x_data)
        x.to_device(backend_config.device)
        y = link(x)
        y_data2 = y.data

        testing.assert_allclose(y_data1, y_data2, atol=0, rtol=0) 

def test_pickling(self, backend_config):
        x_data, = self.generate_inputs()

        link = self.create_link(self.generate_params())
        link.to_device(backend_config.device)

        x = chainer.Variable(x_data)
        x.to_device(backend_config.device)

        y = link(x)
        y_data1 = y.data
        del x, y
        pickled = pickle.dumps(link, -1)
        del link
        link = pickle.loads(pickled)
        x = chainer.Variable(x_data)
        x.to_device(backend_config.device)
        y = link(x)
        y_data2 = y.data

        testing.assert_allclose(y_data1, y_data2, atol=0, rtol=0) 

def test_pickling(self, backend_config):
        x_data, = self.generate_inputs()

        link = self.create_link(self.generate_params())
        link.to_device(backend_config.device)

        x = chainer.Variable(x_data)
        x.to_device(backend_config.device)

        y = link(x)
        y_data1 = y.data
        del x, y
        pickled = pickle.dumps(link, -1)
        del link
        link = pickle.loads(pickled)
        x = chainer.Variable(x_data)
        x.to_device(backend_config.device)
        y = link(x)
        y_data2 = y.data

        testing.assert_allclose(y_data1, y_data2, atol=0, rtol=0) 

def load(fname):
    """Load an embedding dump generated by `save`"""

    content = _open(fname).read()
    if PY2:
      state = pickle.loads(content)
    else:
      state = pickle.loads(content, encoding='latin1')
    voc, vec = state
    if len(voc) == 2:
      words, counts = voc
      word_count = dict(zip(words, counts))
      vocab = CountedVocabulary(word_count=word_count)
    else:
      vocab = OrderedVocabulary(voc)
    return Embedding(vocabulary=vocab, vectors=vec) 

def check_pickling(self, x_data):
        x = chainer.Variable(x_data)
        y = self.link(x)
        y_data1 = y.data

        del x, y

        pickled = pickle.dumps(self.link, -1)
        del self.link
        self.link = pickle.loads(pickled)

        x = chainer.Variable(x_data)
        y = self.link(x)
        y_data2 = y.data

        testing.assert_allclose(y_data1, y_data2, atol=0, rtol=0) 

def check_pickling(self, x_data):
        x = chainer.Variable(x_data)
        y = self.link(x)
        y_data1 = y.data

        del x, y

        pickled = pickle.dumps(self.link, -1)
        del self.link
        self.link = pickle.loads(pickled)

        x = chainer.Variable(x_data)
        y = self.link(x)
        y_data2 = y.data

        testing.assert_allclose(y_data1, y_data2, atol=0, rtol=0) 

def get_project_settings():
    if ENVVAR not in os.environ:
        project = os.environ.get('SCRAPY_PROJECT', 'default')
        init_env(project)

    settings = Settings()
    settings_module_path = os.environ.get(ENVVAR)
    if settings_module_path:
        settings.setmodule(settings_module_path, priority='project')

    # XXX: remove this hack
    pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
    if pickled_settings:
        settings.setdict(pickle.loads(pickled_settings), priority='project')

    # XXX: deprecate and remove this functionality
    env_overrides = {k[7:]: v for k, v in os.environ.items() if
                     k.startswith('SCRAPY_')}
    if env_overrides:
        settings.setdict(env_overrides, priority='project')

    return settings 

def test_pickling(self):
        try:
            features = numpy.arange(360, dtype='uint16').reshape((10, 36))
            h5file = h5py.File('file.hdf5', mode='w')
            h5file['features'] = features
            split_dict = {'train': {'features': (0, 10, None, '.')}}
            h5file.attrs['split'] = H5PYDataset.create_split_array(split_dict)
            dataset = cPickle.loads(
                cPickle.dumps(H5PYDataset(h5file, which_sets=('train',))))
            # Make sure _out_of_memory_{open,close} accesses
            # external_file_handle rather than _external_file_handle
            dataset._out_of_memory_open()
            dataset._out_of_memory_close()
            assert dataset.data_sources is None
        finally:
            os.remove('file.hdf5') 

def execute_request(self, request):
        # noinspection PyBroadException
        try:
            # Use pickle to read the binary data.
            data_object = pickle.loads(request)
        except Exception:  # pickle.loads is document as raising any type of exception, so have to catch them all.
            self.__logger.warn(
                "Could not parse incoming metric line from graphite pickle server, ignoring",
                error_code="graphite_monitor/badUnpickle",
            )
            return

        try:
            # The format should be [[ metric [ timestamp, value]] ... ]
            for (metric, datapoint) in data_object:
                value = float(datapoint[1])
                orig_timestamp = float(datapoint[0])
                self.__logger.emit_value(
                    metric, value, extra_fields={"orig_time": orig_timestamp}
                )
        except ValueError:
            self.__logger.warn(
                "Could not parse incoming metric line from graphite pickle server, ignoring",
                error_code="graphite_monitor/badPickleLine",
            ) 

def predict(self, data_root, model_root, test_root, test_div, out_path, readable=False):
        meta_path = os.path.join(data_root, 'meta')
        meta = cPickle.loads(open(meta_path, 'rb').read())

        model_fname = os.path.join(model_root, 'model.h5')
        self.logger.info('# of classes(train): %s' % len(meta['y_vocab']))
        model = load_model(model_fname,
                           custom_objects={'top1_acc': top1_acc})

        test_path = os.path.join(test_root, 'data.h5py')
        test_data = h5py.File(test_path, 'r')

        test = test_data[test_div]
        batch_size = opt.batch_size
        pred_y = []
        test_gen = ThreadsafeIter(self.get_sample_generator(test, batch_size, raise_stop_event=True))
        total_test_samples = test['uni'].shape[0]
        with tqdm.tqdm(total=total_test_samples) as pbar:
            for chunk in test_gen:
                total_test_samples = test['uni'].shape[0]
                X, _ = chunk
                _pred_y = model.predict(X)
                pred_y.extend([np.argmax(y) for y in _pred_y])
                pbar.update(X[0].shape[0])
        self.write_prediction_result(test, pred_y, meta, out_path, readable=readable) 

def _read_data(self, spider, request):
        key = self._request_key(request)
        db = self.db
        tkey = '%s_time' % key
        if tkey not in db:
            return  # not found

        ts = db[tkey]
        if 0 < self.expiration_secs < time() - float(ts):
            return  # expired

        return pickle.loads(db['%s_data' % key]) 

def test_load(self):
        instance = cPickle.loads(cPickle.dumps(DummyClass()))
        assert_equal(instance.bulky_attr, list(range(100)))
        assert instance.non_picklable is not None 

def test_canPickleAndUnpickleISOTXS(self):
        pikAA = cPickle.loads(cPickle.dumps(self.isotxsAA))
        self.assertTrue(xsLibraries.compare(pikAA, self.isotxsAA)) 

def test_canPickleAndUnpickleGAMISO(self):
        pikAA = cPickle.loads(cPickle.dumps(self.gamisoAA))
        self.assertTrue(xsLibraries.compare(pikAA, self.gamisoAA)) 

def _read_data(self, spider, request):
        key = self._request_key(request)
        db = self.db
        tkey = '%s_time' % key
        if tkey not in db:
            return  # not found

        ts = db[tkey]
        if 0 < self.expiration_secs < time() - float(ts):
            return  # expired

        return pickle.loads(db['%s_data' % key]) 

def deserialize(arr):
    """Unserialize a Python object from an array of float32 values fetched from
    a workspace. See serialize().
    """
    return pickle.loads(arr.astype(np.uint8).tobytes()) 

def launchTask(self, driver, task):
        logger.info('Launch task')
        proc_id = int(task['task_id']['value'])
        self.reply_status(driver, proc_id, 'TASK_RUNNING')
        params = pickle.loads(decode_data(task['data']))
        a = params['a']
        kw = params['kw']
        handlers = params['handlers']
        hostname = params['hostname']

        for i, key in enumerate(['stdin', 'stdout', 'stderr']):
            kw[key] = s = socket.socket()
            logger.info('Connect %s:%s for %s' % (hostname, handlers[i], key))
            s.connect((hostname, handlers[i]))

        kw.pop('close_fds', None)

        try:
            p = subprocess.Popen(*a, close_fds=True, **kw)
        except Exception:
            exc_type, exc_value, tb = sys.exc_info()
            # Save the traceback and attach it to the exception object
            exc_lines = traceback.format_exception(exc_type,
                                                   exc_value,
                                                   tb)
            exc_value.child_traceback = ''.join(exc_lines)
            self.reply_status(driver, proc_id, 'TASK_FAILED',
                              data=(None, exc_value))
            logger.exception('Exec failed')
            return
        finally:
            kw['stdin'].close()
            kw['stdout'].close()
            kw['stderr'].close()

        with self.cond:
            self.procs[proc_id] = p
            self.pid_to_proc[p.pid] = proc_id
            self.cond.notify() 

def test_data_stream_pickling(self):
        stream = DataStream(H5PYDataset(self.h5file, which_sets=('train',)),
                            iteration_scheme=SequentialScheme(100, 10))
        cPickle.loads(cPickle.dumps(stream))
        stream.close() 

def test_pickling(self):
        dataset = cPickle.loads(cPickle.dumps(self.dataset))
        assert_equal(len(dataset.nodes), 1) 

def test_pickling(self):
        self.stream = cPickle.loads(cPickle.dumps(self.stream))
        server_data = self.stream.get_epoch_iterator()
        expected_data = get_stream().get_epoch_iterator()
        for _, s, e in zip(range(3), server_data, expected_data):
            for data in zip(s, e):
                assert_allclose(*data, rtol=1e-5)
        assert_raises(StopIteration, next, server_data) 

def test_role_serialization():
    """Test that roles compare equal before and after serialization."""
    roles = [blocks.roles.INPUT,
             blocks.roles.OUTPUT,
             blocks.roles.COST,
             blocks.roles.PARAMETER,
             blocks.roles.AUXILIARY,
             blocks.roles.WEIGHT,
             blocks.roles.BIAS,
             blocks.roles.FILTER]

    for role in roles:
        deserialized = cPickle.loads(cPickle.dumps(role))
        assert deserialized == role 

def test_application_serialization():
    assert id(cPickle.loads(cPickle.dumps(Linear.apply))) == id(Linear.apply) 

def test_with_extra_dims_is_serializable():
    brick = LinearWithExtraDims(
        input_dim=3, output_dim=4,
        weights_init=Constant(1), biases_init=Constant(0))
    brick.initialize()
    cPickle.loads(cPickle.dumps(brick)) 

def test_training_resumption():
    def do_test(with_serialization):
        data_stream = IterableDataset(range(10)).get_example_stream()
        main_loop = MainLoop(
            MockAlgorithm(), data_stream,
            extensions=[WriteBatchExtension(),
                        FinishAfter(after_n_batches=14)])
        main_loop.run()
        assert main_loop.log.status['iterations_done'] == 14

        if with_serialization:
            main_loop = cPickle.loads(cPickle.dumps(main_loop))

        finish_after = unpack(
            [ext for ext in main_loop.extensions
             if isinstance(ext, FinishAfter)], singleton=True)
        finish_after.add_condition(
            ["after_batch"],
            predicate=lambda log: log.status['iterations_done'] == 27)
        main_loop.run()
        assert main_loop.log.status['iterations_done'] == 27
        assert main_loop.log.status['epochs_done'] == 2
        for i in range(27):
            assert main_loop.log[i + 1]['batch'] == {"data": i % 10}

    do_test(False)
    do_test(True)