Python pickle.dump() Examples

def gt_roidb(self):
        """
        Return the database of ground-truth regions of interest.
        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = pickle.load(fid)
            print('{} gt roidb loaded from {}'.format(self.name, cache_file))
            return roidb

        gt_roidb = [self._load_imagenet_annotation(index)
                    for index in self.image_index]
        with open(cache_file, 'wb') as fid:
            pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
        print('wrote gt roidb to {}'.format(cache_file))

        return gt_roidb 

def gt_roidb(self):
    """
    Return the database of ground-truth regions of interest.

    This function loads/saves from/to a cache file to speed up future calls.
    """
    cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
    if os.path.exists(cache_file):
      with open(cache_file, 'rb') as fid:
        try:
          roidb = pickle.load(fid)
        except:
          roidb = pickle.load(fid, encoding='bytes')
      print('{} gt roidb loaded from {}'.format(self.name, cache_file))
      return roidb

    gt_roidb = [self._load_pascal_annotation(index)
                for index in self.image_index]
    with open(cache_file, 'wb') as fid:
      pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
    print('wrote gt roidb to {}'.format(cache_file))

    return gt_roidb 

def save_eval_file(opt, stats, eval_type="losses", split="dev", ext="pickle"):
    if cfg.test_save:
        name = "{}/{}.{}".format(utils.make_name(
            opt, prefix="garbage/{}/".format(eval_type),
            is_dir=True, eval_=True), split, ext)
    else:
        name = "{}/{}.{}".format(utils.make_name(
            opt, prefix="results/{}/".format(eval_type),
            is_dir=True, eval_=True), split, ext)
    print("Saving {} {} to {}".format(split, eval_type, name))

    if ext == "pickle":
        with open(name, "wb") as f:
            pickle.dump(stats, f)
    elif ext == "txt":
        with open(name, "w") as f:
            f.write(stats)
    elif ext == "json":
        with open(name, "w") as f:
            json.dump(stats, f)
    else:
        raise 

def _write_coco_results_file(self, all_boxes, res_file):
    # [{"image_id": 42,
    #   "category_id": 18,
    #   "bbox": [258.15,41.29,348.26,243.78],
    #   "score": 0.236}, ...]
    results = []
    for cls_ind, cls in enumerate(self.classes):
      if cls == '__background__':
        continue
      print('Collecting {} results ({:d}/{:d})'.format(cls, cls_ind,
                                                       self.num_classes - 1))
      coco_cat_id = self._class_to_coco_cat_id[cls]
      results.extend(self._coco_results_one_category(all_boxes[cls_ind],
                                                     coco_cat_id))
    print('Writing results json to {}'.format(res_file))
    with open(res_file, 'w') as fid:
      json.dump(results, fid) 

def gt_roidb(self):
        """
        Return the database of ground-truth regions of interest.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = pickle.load(fid)
            print('{} gt roidb loaded from {}'.format(self.name, cache_file))
            return roidb

        gt_roidb = [self._load_pascal_annotation(index)
                    for index in self.image_index]
        with open(cache_file, 'wb') as fid:
            pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
        print('wrote gt roidb to {}'.format(cache_file))

        return gt_roidb 

def train(self):
        while (self.epoch < self.option.max_epoch and not self.early_stopped):
            self.one_epoch_train()
            self.one_epoch_valid()
            self.one_epoch_test()
            self.epoch += 1
            model_path = self.saver.save(self.sess, 
                                         self.option.model_path,
                                         global_step=self.epoch)
            print("Model saved at %s" % model_path)
            
            if self.early_stop():
                self.early_stopped = True
                print("Early stopped at epoch %d" % (self.epoch))
        
        all_test_in_top = [np.mean(x[1]) for x in self.test_stats]
        best_test_epoch = np.argmax(all_test_in_top)
        best_test = all_test_in_top[best_test_epoch]
        
        msg = "Best test in top: %0.4f at epoch %d." % (best_test, best_test_epoch + 1)       
        print(msg)
        self.log_file.write(msg + "\n")
        pickle.dump([self.train_stats, self.valid_stats, self.test_stats],
                    open(os.path.join(self.option.this_expsdir, "results.pckl"), "w")) 

def saveToDisk(self):
        """Saves the data to disk."""
        out.info('Saving to disk (%s)\n' % (self.filename))

        # Make sure they want to save
        if(not self.attrSaveable()):
            return

        # Get whatever the data is
        pyld = self.exportAttr(self.getAttr())

        # Write the file to disk, truncate if it exists
        try:
            with open(self.filename, 'wb') as output:
                pickle.dump(pyld, output)
                os.fsync(output.fileno())
        except Exception as e:
            out.err('Error writing to disk %s\n' % (str(e)))

        try:
            with open(self.filename + ".yaml", "w") as output:
                yaml.dump(pyld, output)
        except Exception as error:
            out.err("Error writing yaml file: {}".format(error)) 

def _write_coco_results_file(self, all_boxes, res_file):
    # [{"image_id": 42,
    #   "category_id": 18,
    #   "bbox": [258.15,41.29,348.26,243.78],
    #   "score": 0.236}, ...]
    results = []
    for cls_ind, cls in enumerate(self.classes):
      if cls == '__background__':
        continue
      print('Collecting {} results ({:d}/{:d})'.format(cls, cls_ind,
                                                       self.num_classes - 1))
      coco_cat_id = self._class_to_coco_cat_id[cls]
      results.extend(self._coco_results_one_category(all_boxes[cls_ind],
                                                     coco_cat_id))
    print('Writing results json to {}'.format(res_file))
    with open(res_file, 'w') as fid:
      json.dump(results, fid) 

def gt_roidb(self):
    """
    Return the database of ground-truth regions of interest.

    This function loads/saves from/to a cache file to speed up future calls.
    """
    cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
    if os.path.exists(cache_file):
      with open(cache_file, 'rb') as fid:
        try:
          roidb = pickle.load(fid)
        except:
          roidb = pickle.load(fid, encoding='bytes')
      print('{} gt roidb loaded from {}'.format(self.name, cache_file))
      return roidb

    gt_roidb = [self._load_pascal_labels(index)
                for index in self.image_index]
    with open(cache_file, 'wb') as fid:
      pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
    print('wrote gt roidb to {}'.format(cache_file))

    return gt_roidb 

def cache_data(self, name, data):
        """Save ``data`` to cache under ``name``.

        If ``data`` is ``None``, the corresponding cache file will be
        deleted.

        :param name: name of datastore
        :param data: data to store. This may be any object supported by
                the cache serializer

        """
        serializer = manager.serializer(self.cache_serializer)

        cache_path = self.cachefile('%s.%s' % (name, self.cache_serializer))

        if data is None:
            if os.path.exists(cache_path):
                os.unlink(cache_path)
                self.logger.debug('deleted cache file: %s', cache_path)
            return

        with atomic_writer(cache_path, 'wb') as file_obj:
            serializer.dump(data, file_obj)

        self.logger.debug('cached data: %s', cache_path) 

def gt_roidb(self):
        """
        Return the database of ground-truth regions of interest.

        This function loads/saves from/to a cache file to speed up future calls.
        """        
        cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
        if os.path.exists(cache_file):
            fid = gzip.open(cache_file,'rb')
            roidb = pickle.load(fid)
            fid.close()
            print('{} gt roidb loaded from {}'.format(self.name, cache_file))
            return roidb

        gt_roidb = [self._load_vg_annotation(index)
                    for index in self.image_index]
        fid = gzip.open(cache_file,'wb')
        pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
        fid.close()
        print('wrote gt roidb to {}'.format(cache_file))
        return gt_roidb 

def saveSeed(randomStates, saveDir):
    """
    Takes a list of dictionaries of random generator states of different modules
    and saves them in a .pkl format.
    
    Inputs:
        randomStates (list): The length of this list is equal to the number of
            modules whose states want to be saved (torch, numpy, etc.). Each
            element in this list is a dictionary. The dictionary has three keys:
            'module' with the name of the module in string format ('numpy' or
            'torch', for example), 'state' with the saved generator state and,
            if corresponds, 'seed' with the specific seed for the generator
            (note that torch has both state and seed, but numpy only has state)
        saveDir (path): where to save the seed, it will be saved under the 
            filename 'randomSeedUsed.pkl'
    """
    pathToSeed = os.path.join(saveDir, 'randomSeedUsed.pkl')
    with open(pathToSeed, 'wb') as seedFile:
        pickle.dump({'randomStates': randomStates}, seedFile) 

def main(cache_dir):
    files_list = list(os.listdir(cache_dir))
    for file in files_list:
        full_filename = os.path.join(cache_dir, file)
        if os.path.isfile(full_filename):
            print("Processing {}".format(full_filename))
            m, stored_kwargs = pickle.load(open(full_filename, 'rb'))
            updated_kwargs = util.get_compatible_kwargs(model.Model, stored_kwargs)

            model_hash = util.object_hash(updated_kwargs)
            print("New hash -> " + model_hash)
            model_filename = os.path.join(cache_dir, "model_{}.p".format(model_hash))
            sys.setrecursionlimit(100000)
            pickle.dump((m,updated_kwargs), open(model_filename,'wb'), protocol=pickle.HIGHEST_PROTOCOL)

            os.remove(full_filename) 

def register(self, name, serializer):
        """Register ``serializer`` object under ``name``.

        Raises :class:`AttributeError` if ``serializer`` in invalid.

        .. note::

            ``name`` will be used as the file extension of the saved files.

        :param name: Name to register ``serializer`` under
        :type name: ``unicode`` or ``str``
        :param serializer: object with ``load()`` and ``dump()``
            methods

        """
        # Basic validation
        getattr(serializer, 'load')
        getattr(serializer, 'dump')

        self._serializers[name] = serializer 

def saveState(dataHub):
    import pickle as pickle
    import gzip

    pickle.dump(dataHub, gzip.open(dataHub.args.save_state, "wb"))
    logging.warn("^"*20 + " saving state to pickle and exiting " + "^"*20) 

def save(self) -> None:
        """Save the study to disk."""
        filename = '{}.study'.format(self.name)
        if self.datadir is not None:
            filename = os.path.join(self.datadir, filename)
            if not os.path.isdir(self.datadir):
                os.mkdir(self.datadir)
        with open(filename, 'wb') as f:
            pickle.dump((type(self), self._init_kwargs(), self.trial_results),
                        f) 

def save_pkl(pkl_file, file_path, file_name, min_v=None, max_v=None):
    pk.dump(pkl_file, open(os.path.join(file_path, file_name), "wb"))
    print(file_name,' is dumped in: ', file_path)
    if((min_v is not None) and (max_v is not None)):
        pk.dump(min_v, open(os.path.join(file_path,file_name+"min"), "wb"))
        print(file_name+".min",' is dumped in: ', file_path)
        pk.dump(max_v, open(os.path.join(file_path,file_name+"max"), "wb"))
        print(file_name+".max",' is dumped in: ', file_path) 

def save_policy(self, path):
        """Pickles the current policy for later inspection.
        """
        with open(path, 'wb') as f:
            pickle.dump(self.policy, f) 

def save_pkl(pkl_file, file_path, file_name, min_v=None, max_v=None):
    pk.dump(pkl_file, open(os.path.join(file_path, file_name), "wb"))
    print(file_name,' is dumped in: ', file_path)
    if((min_v is not None) and (max_v is not None)):
        pk.dump(min_v, open(os.path.join(file_path,file_name+"min"), "wb"))
        print(file_name+".min",' is dumped in: ', file_path)
        pk.dump(max_v, open(os.path.join(file_path,file_name+"max"), "wb"))
        print(file_name+".max",' is dumped in: ', file_path) 

def load_all_kegg_entries(self, filename="kegg_gene.dat"):
        if os.path.isfile(filename):
            import pickle
            results = pickle.load(open(filename, "r"))
            return results
        # TODO:
        # donwload from a URL  
        print("could not find kegg data. fetching data from website if possible")
        # Fetches the KEGG results using multicore to send several requests at the same time
        found = self.alldata.keys()
        names = [x for x in self.names if x not in found]
        print("Fetching %s enties" % len(names))


        mc = test_func(names)
        self.mcresults = mc.results.copy()
        # here are the entries to be used as keys
        try:
            entries = ["hsa:"+x['entry'].split()[0] for x in self.mcresults if x]

            for entry, result in zip(entries, self.mcresults):
                self.alldata[entry] = result
        except:
            print("something wrng happened while scaning mcresults")

        #import pickle
        #pickle.dump(results, open("kegg_gene.dat","w")) 

def save_obj(filename, obj):
    with open(filename, 'wb') as file:
        pickle.dump(obj, file, protocol=pickle.HIGHEST_PROTOCOL)
    logger.info('saved : {}\t{}'.format(filename, type(obj))) 

def pickle_tf_vars():  
    """
    Saves tensorflow variables
    Requires them to be initialized first, also a default session must exist
    """
    _dict = {v.name : v.eval() for v in tf.global_variables()}
    with open(osp.join(G.output_dir, "vars.pkl"), 'wb') as f:
        pickle.dump(_dict, f) 

def bod2pascal(self):
        pascalLabel_path = os.path.join(self.basepath, r'pascalLabel')
        for basename in self.namelist:
            objects = parse_bod_poly(os.path.join(self.labelpath, basename + '.txt'))
            tree_root = ET.Element('annotation')
            folder = ET.SubElement(tree_root, 'secondjpg')
            filename = ET.SubElement(tree_root, basename)
            size = ET.SubElement(tree_root, 'size')
            width = ET.SubElement(size, 'width')
            height = ET.SubElement(size, 'height')
            ## TODO: read imagesize from img or info
            imgname = os.path.join(self.basepath, 'images', basename + '.jpg')
            # img = cv2.imread(imgname)
            width.text = str(1024)
            height.text = str(1024)
            for obj in objects:
                object = ET.SubElement(tree_root, 'object')
                ET.dump(tree_root)
                name = ET.SubElement(object, 'name')
                name.text = datamap[obj['name']]
                difficult = ET.SubElement(object, 'difficult')
                print('difficult:', obj['difficult'])
                difficult.text = str(obj['difficult'])
                print('type difficult.text:', type(difficult.text))
                bndbox = ET.SubElement(object, 'bndbox')
                xmin = ET.SubElement(bndbox, 'xmin')
                xmax = ET.SubElement(bndbox, 'xmax')
                ymin = ET.SubElement(bndbox, 'ymin')
                ymax = ET.SubElement(bndbox, 'ymax')
                poly = obj['poly']
                bbox = dots4ToRec4(poly)
                xmin.text = str(bbox[0])
                ymin.text = str(bbox[1])
                xmax.text = str(bbox[2])
                ymax.text = str(bbox[3])
            tree = ET.ElementTree(tree_root)
            tree.write(os.path.join(pascalLabel_path, basename + '.xml')) 

def getcategory(
        basepath,
        label,
        ):
    classedict = {}
    def initdic():
        for clsname in classname_15:
            wordname = datamap_15[clsname]
            classedict[wordname] = []
    initdic()
    picklepath = os.path.join(basepath, 'pickle')
    pickledir = os.path.join(picklepath, 'category-file.pickle')
    if not os.path.isfile(pickledir):
        labelpath = os.path.join(basepath, label)
        filelist = GetFileFromThisRootDir(labelpath)
        for fullname in filelist:
            name = mybasename(fullname)
            objects = parse_bod_poly(fullname)
            for obj in objects:
                #wordname = datamap[obj['name']]
                wordname = obj['name']
                if name not in classedict[wordname]:
                    classedict[wordname].append(name)

        with open(pickledir, 'wb') as f:
            pickle.dump(classedict, f, pickle.HIGHEST_PROTOCOL)
    else:
        with open(pickledir, 'rb') as f:
            classedict = pickle.load(f)
    return classedict 

def execute_at_exit(self):
        if self.save_best is not None:
            log.out.info("Saving numpynet object to a pickle")
            with open(self.save_best, "wb") as output_handle:
                pickle.dump(best_self, output_handle, pickle.HIGHEST_PROTOCOL) 

def _get_cached(self, cache_item, fn):
        cache_path = os.path.join(self._root_path, 'cache', '{}_{}.pkl'.format(self._name, cache_item))
        if os.path.exists(cache_path):
            logger.info('loading cache {}'.format(cache_path))
            with open(cache_path, 'rb') as fid:
                cached = pickle.load(fid)
            return cached
        else:
            logger.info('computing cache {}'.format(cache_path))
            cached = fn()
            logger.info('saving cache {}'.format(cache_path))
            with open(cache_path, 'wb') as fid:
                pickle.dump(cached, fid, pickle.HIGHEST_PROTOCOL)
            return cached 

def evaluate_detections(self, detections, **kwargs):
        cache_path = os.path.join(self._root_path, 'cache', '{}_{}.pkl'.format(self._name, 'detections'))
        logger.info('saving cache {}'.format(cache_path))
        with open(cache_path, 'wb') as fid:
            pickle.dump(detections, fid, pickle.HIGHEST_PROTOCOL)
        self._evaluate_detections(detections, **kwargs) 

def save(self, fname):
        args_save = {key: v.asnumpy() for key, v in self.args.items()}
        with open(fname, 'wb') as fout:
            pickle.dump(args_save, fout) 

def save_obj(obj, name):
    with open(name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL) 

def save(self, fname):
        args_save = {key: v.asnumpy() for key, v in self.args.items()}
        with open(fname, 'wb') as fout:
            pickle.dump(args_save, fout)