Python pickle.HIGHEST_PROTOCOL() Examples

The following are code examples for showing how to use pickle.HIGHEST_PROTOCOL(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: Collaborative-Learning-for-Weakly-Supervised-Object-Detection   Author: Sunarker   File: pascal_voc.py    MIT License 6 votes vote down vote up
def gt_roidb(self):
    """
    Return the database of ground-truth regions of interest.

    This function loads/saves from/to a cache file to speed up future calls.
    """
    cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
    if os.path.exists(cache_file):
      with open(cache_file, 'rb') as fid:
        try:
          roidb = pickle.load(fid)
        except:
          roidb = pickle.load(fid, encoding='bytes')
      print('{} gt roidb loaded from {}'.format(self.name, cache_file))
      return roidb

    gt_roidb = [self._load_pascal_labels(index)
                for index in self.image_index]
    with open(cache_file, 'wb') as fid:
      pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
    print('wrote gt roidb to {}'.format(cache_file))

    return gt_roidb 
Example 2
Project: Collaborative-Learning-for-Weakly-Supervised-Object-Detection   Author: Sunarker   File: coco.py    MIT License 6 votes vote down vote up
def gt_roidb(self):
    """
    Return the database of ground-truth regions of interest.
    This function loads/saves from/to a cache file to speed up future calls.
    """
    cache_file = osp.join(self.cache_path, self.name + '_gt_roidb.pkl')
    if osp.exists(cache_file):
      with open(cache_file, 'rb') as fid:
        roidb = pickle.load(fid)
      print('{} gt roidb loaded from {}'.format(self.name, cache_file))
      return roidb

    gt_roidb = [self._load_coco_annotation(index)
                for index in self._image_index]

    with open(cache_file, 'wb') as fid:
      pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
    print('wrote gt roidb to {}'.format(cache_file))
    return gt_roidb 
Example 3
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: cache.py    Apache License 2.0 6 votes vote down vote up
def set(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        filename = self._get_filename(key)
        self._prune()
        try:
            fd, tmp = tempfile.mkstemp(suffix=self._fs_transaction_suffix,
                                       dir=self._path)
            with os.fdopen(fd, 'wb') as f:
                pickle.dump(int(time() + timeout), f, 1)
                pickle.dump(value, f, pickle.HIGHEST_PROTOCOL)
            rename(tmp, filename)
            os.chmod(filename, self._mode)
        except (IOError, OSError):
            return False
        else:
            return True 
Example 4
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: cache.py    Apache License 2.0 6 votes vote down vote up
def set(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        filename = self._get_filename(key)
        self._prune()
        try:
            fd, tmp = tempfile.mkstemp(suffix=self._fs_transaction_suffix,
                                       dir=self._path)
            with os.fdopen(fd, 'wb') as f:
                pickle.dump(int(time() + timeout), f, 1)
                pickle.dump(value, f, pickle.HIGHEST_PROTOCOL)
            rename(tmp, filename)
            os.chmod(filename, self._mode)
        except (IOError, OSError):
            return False
        else:
            return True 
Example 5
Project: FasterRCNN_TF_Py3   Author: upojzsb   File: pascal_voc.py    MIT License 6 votes vote down vote up
def gt_roidb(self):
        """
        Return the database of ground-truth regions of interest.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                try:
                    roidb = pickle.load(fid)
                except:
                    roidb = pickle.load(fid, encoding='bytes')
            print('{} gt roidb loaded from {}'.format(self.name, cache_file))
            return roidb

        gt_roidb = [self._load_pascal_annotation(index)
                    for index in self.image_index]
        with open(cache_file, 'wb') as fid:
            pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
        print('wrote gt roidb to {}'.format(cache_file))

        return gt_roidb 
Example 6
Project: FasterRCNN_TF_Py3   Author: upojzsb   File: coco.py    MIT License 6 votes vote down vote up
def gt_roidb(self):
        """
        Return the database of ground-truth regions of interest.
        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = osp.join(self.cache_path, self.name + '_gt_roidb.pkl')
        if osp.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = pickle.load(fid)
            print('{} gt roidb loaded from {}'.format(self.name, cache_file))
            return roidb

        gt_roidb = [self._load_coco_annotation(index)
                    for index in self._image_index]

        with open(cache_file, 'wb') as fid:
            pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
        print('wrote gt roidb to {}'.format(cache_file))
        return gt_roidb 
Example 7
Project: flasky   Author: RoseOu   File: cache.py    MIT License 6 votes vote down vote up
def set(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        filename = self._get_filename(key)
        self._prune()
        try:
            fd, tmp = tempfile.mkstemp(suffix=self._fs_transaction_suffix,
                                       dir=self._path)
            with os.fdopen(fd, 'wb') as f:
                pickle.dump(int(time() + timeout), f, 1)
                pickle.dump(value, f, pickle.HIGHEST_PROTOCOL)
            rename(tmp, filename)
            os.chmod(filename, self._mode)
        except (IOError, OSError):
            return False
        else:
            return True 
Example 8
Project: HAPI   Author: MAfarrag   File: WeirdFn.py    MIT License 6 votes vote down vote up
def save_obj(obj, saved_name ):
    """
    ===============================================================
        save_obj(obj, saved_name )
    ===============================================================
    this function is used to save any python object to your hard desk
    
    Inputs:
    ----------
        1-obj:
            
        2-saved_name:
            ['String'] name of the object 
    Outputs:    
    ----------
        the object will be saved to the given path/current working directory
        with the given name
    Example:
        data={"key1":[1,2,3,5],"key2":[6,2,9,7]}
        save_obj(data,path+'/flow_acc_table')
    """
    with open( saved_name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL) 
Example 9
Project: HAPI   Author: MAfarrag   File: WeirdFn.py    MIT License 6 votes vote down vote up
def save_obj(obj, saved_name ):
    """
    ===============================================================
        save_obj(obj, saved_name )
    ===============================================================
    this function is used to save any python object to your hard desk
    
    Inputs:
    ----------
        1-obj:
            
        2-saved_name:
            ['String'] name of the object 
    Outputs:    
    ----------
        the object will be saved to the given path/current working directory
        with the given name
    Example:
        data={"key1":[1,2,3,5],"key2":[6,2,9,7]}
        save_obj(data,path+'/flow_acc_table')
    """
    with open( saved_name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL) 
Example 10
Project: HAPI   Author: MAfarrag   File: WeirdFn.py    MIT License 6 votes vote down vote up
def save_obj(obj, saved_name ):
    """
    ===============================================================
        save_obj(obj, saved_name )
    ===============================================================
    this function is used to save any python object to your hard desk
    
    Inputs:
    ----------
        1-obj:
            
        2-saved_name:
            ['String'] name of the object 
    Outputs:    
    ----------
        the object will be saved to the given path/current working directory
        with the given name
    Example:
        data={"key1":[1,2,3,5],"key2":[6,2,9,7]}
        save_obj(data,path+'/flow_acc_table')
    """
    with open( saved_name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL) 
Example 11
Project: cascade-rcnn_Pytorch   Author: guoruoqian   File: pascal_voc_rbg.py    MIT License 6 votes vote down vote up
def gt_roidb(self):
    """
    Return the database of ground-truth regions of interest.

    This function loads/saves from/to a cache file to speed up future calls.
    """
    cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
    if os.path.exists(cache_file):
      with open(cache_file, 'rb') as fid:
        try:
          roidb = pickle.load(fid)
        except:
          roidb = pickle.load(fid, encoding='bytes')
      print('{} gt roidb loaded from {}'.format(self.name, cache_file))
      return roidb

    gt_roidb = [self._load_pascal_annotation(index)
                for index in self.image_index]
    with open(cache_file, 'wb') as fid:
      pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
    print('wrote gt roidb to {}'.format(cache_file))

    return gt_roidb 
Example 12
Project: cascade-rcnn_Pytorch   Author: guoruoqian   File: pascal_voc.py    MIT License 6 votes vote down vote up
def gt_roidb(self):
        """
        Return the database of ground-truth regions of interest.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = pickle.load(fid)
            print('{} gt roidb loaded from {}'.format(self.name, cache_file))
            return roidb

        gt_roidb = [self._load_pascal_annotation(index)
                    for index in self.image_index]
        with open(cache_file, 'wb') as fid:
            pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
        print('wrote gt roidb to {}'.format(cache_file))

        return gt_roidb 
Example 13
Project: cascade-rcnn_Pytorch   Author: guoruoqian   File: coco.py    MIT License 6 votes vote down vote up
def gt_roidb(self):
    """
    Return the database of ground-truth regions of interest.
    This function loads/saves from/to a cache file to speed up future calls.
    """
    cache_file = osp.join(self.cache_path, self.name + '_gt_roidb.pkl')
    if osp.exists(cache_file):
      with open(cache_file, 'rb') as fid:
        roidb = pickle.load(fid)
      print('{} gt roidb loaded from {}'.format(self.name, cache_file))
      return roidb

    gt_roidb = [self._load_coco_annotation(index)
                for index in self._image_index]

    with open(cache_file, 'wb') as fid:
      pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
    print('wrote gt roidb to {}'.format(cache_file))
    return gt_roidb 
Example 14
Project: cascade-rcnn_Pytorch   Author: guoruoqian   File: vg.py    MIT License 6 votes vote down vote up
def gt_roidb(self):
        """
        Return the database of ground-truth regions of interest.

        This function loads/saves from/to a cache file to speed up future calls.
        """        
        cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
        if os.path.exists(cache_file):
            fid = gzip.open(cache_file,'rb')
            roidb = pickle.load(fid)
            fid.close()
            print('{} gt roidb loaded from {}'.format(self.name, cache_file))
            return roidb

        gt_roidb = [self._load_vg_annotation(index)
                    for index in self.image_index]
        fid = gzip.open(cache_file,'wb')
        pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
        fid.close()
        print('wrote gt roidb to {}'.format(cache_file))
        return gt_roidb 
Example 15
Project: pywren-ibm-cloud   Author: pywren   File: cloudpickle.py    Apache License 2.0 6 votes vote down vote up
def dumps(obj, protocol=None):
    """Serialize obj as a string of bytes allocated in memory

    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.

    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    file = StringIO()
    try:
        cp = CloudPickler(file, protocol=protocol)
        cp.dump(obj)
        return file.getvalue()
    finally:
        file.close()


# including pickles unloading functions in this namespace 
Example 16
Project: esjspy   Author: Dothion   File: base.py    GNU General Public License v3.0 5 votes vote down vote up
def _dump(identifier: str, some_obj: Any, cache_dir: Path = Path(CACHE_DIR), use_cache=True) -> None:
    if not use_cache:
        return
    md5 = _gen_md5(identifier)
    cache_dir = cache_dir / md5[0:2] / md5[2:4]
    if not cache_dir.exists():
        cache_dir.mkdir(parents=True)
    cache_path = cache_dir / md5
    with cache_path.open('wb') as f:
        pickle.dump(some_obj, f, protocol=pickle.HIGHEST_PROTOCOL)
    logger.debug('保存了 %s 的内容到缓存。' % identifier if LANGUAGE in _simplified else
                 '保存了 %s 的內容到 cache。' % identifier) 
Example 17
Project: pyblish-win   Author: pyblish   File: test_runner.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_pickle_unpickle(self):
        # Issue #7197: a TextTestRunner should be (un)pickleable. This is
        # required by test_multiprocessing under Windows (in verbose mode).
        from StringIO import StringIO as PickleableIO
        # cStringIO objects are not pickleable, but StringIO objects are.
        stream = PickleableIO("foo")
        runner = unittest.TextTestRunner(stream)
        for protocol in range(pickle.HIGHEST_PROTOCOL + 1):
            s = pickle.dumps(runner, protocol=protocol)
            obj = pickle.loads(s)
            # StringIO objects never compare equal, a cheap test instead.
            self.assertEqual(obj.stream.getvalue(), stream.getvalue()) 
Example 18
Project: pyblish-win   Author: pyblish   File: test_case.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def testPickle(self):
        # Issue 10326

        # Can't use TestCase classes defined in Test class as
        # pickle does not work with inner classes
        test = unittest.TestCase('run')
        for protocol in range(pickle.HIGHEST_PROTOCOL + 1):

            # blew up prior to fix
            pickled_test = pickle.dumps(test, protocol=protocol)

            unpickled_test = pickle.loads(pickled_test)
            self.assertEqual(test, unpickled_test) 
Example 19
Project: pyblish-win   Author: pyblish   File: test_xrange.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_pickling(self):
        testcases = [(13,), (0, 11), (-22, 10), (20, 3, -1),
                     (13, 21, 3), (-2, 2, 2)]
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            for t in testcases:
                r = xrange(*t)
                self.assertEqual(list(pickle.loads(pickle.dumps(r, proto))),
                                 list(r))

        M = min(sys.maxint, sys.maxsize)
        large_testcases = testcases + [
            (0, M, 1),
            (M, 0, -1),
            (0, M, M - 1),
            (M // 2, M, 1),
            (0, -M, -1),
            (0, -M, 1 - M),
            (-M, M, 2),
            (-M, M, 1024),
            (-M, M, 10585),
            (M, -M, -2),
            (M, -M, -1024),
            (M, -M, -10585),
            ]
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            for t in large_testcases:
                r = xrange(*t)
                r_out = pickle.loads(pickle.dumps(r, proto))
                self.assert_xranges_equivalent(r_out, r) 
Example 20
Project: pyblish-win   Author: pyblish   File: test_bytes.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_pickling(self):
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            for b in b"", b"a", b"abc", b"\xffab\x80", b"\0\0\377\0\0":
                b = self.type2test(b)
                ps = pickle.dumps(b, proto)
                q = pickle.loads(ps)
                self.assertEqual(b, q) 
Example 21
Project: pyblish-win   Author: pyblish   File: test_bytes.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_pickle(self):
        a = ByteArraySubclass(b"abcd")
        a.x = 10
        a.y = ByteArraySubclass(b"efgh")
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            b = pickle.loads(pickle.dumps(a, proto))
            self.assertNotEqual(id(a), id(b))
            self.assertEqual(a, b)
            self.assertEqual(a.x, b.x)
            self.assertEqual(a.y, b.y)
            self.assertEqual(type(a), type(b))
            self.assertEqual(type(a.y), type(b.y)) 
Example 22
Project: pyblish-win   Author: pyblish   File: test_set.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_pickling(self):
        for i in range(pickle.HIGHEST_PROTOCOL + 1):
            p = pickle.dumps(self.s, i)
            dup = pickle.loads(p)
            self.assertEqual(self.s, dup, "%s != %s" % (self.s, dup))
            if type(self.s) not in (set, frozenset):
                self.s.x = 10
                p = pickle.dumps(self.s)
                dup = pickle.loads(p)
                self.assertEqual(self.s.x, dup.x) 
Example 23
Project: pyblish-win   Author: pyblish   File: pickletester.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_highest_protocol(self):
        # Of course this needs to be changed when HIGHEST_PROTOCOL changes.
        self.assertEqual(self.module.HIGHEST_PROTOCOL, 2) 
Example 24
Project: pyblish-win   Author: pyblish   File: test_re.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def pickle_test(self, pickle):
        oldpat = re.compile('a(?:b|(c|e){1,2}?|d)+?(.)')
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            pickled = pickle.dumps(oldpat, proto)
            newpat = pickle.loads(pickled)
            self.assertEqual(newpat, oldpat) 
Example 25
Project: pyblish-win   Author: pyblish   File: test_memoryio.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_pickling(self):
        buf = self.buftype("1234567890")
        memio = self.ioclass(buf)
        memio.foo = 42
        memio.seek(2)

        class PickleTestMemIO(self.ioclass):
            def __init__(me, initvalue, foo):
                self.ioclass.__init__(me, initvalue)
                me.foo = foo
            # __getnewargs__ is undefined on purpose. This checks that PEP 307
            # is used to provide pickling support.

        # Pickle expects the class to be on the module level. Here we use a
        # little hack to allow the PickleTestMemIO class to derive from
        # self.ioclass without having to define all combinations explicitly on
        # the module-level.
        import __main__
        PickleTestMemIO.__module__ = '__main__'
        __main__.PickleTestMemIO = PickleTestMemIO
        submemio = PickleTestMemIO(buf, 80)
        submemio.seek(2)

        # We only support pickle protocol 2 and onward since we use extended
        # __reduce__ API of PEP 307 to provide pickling support.
        for proto in range(2, pickle.HIGHEST_PROTOCOL):
            for obj in (memio, submemio):
                obj2 = pickle.loads(pickle.dumps(obj, protocol=proto))
                self.assertEqual(obj.getvalue(), obj2.getvalue())
                self.assertEqual(obj.__class__, obj2.__class__)
                self.assertEqual(obj.foo, obj2.foo)
                self.assertEqual(obj.tell(), obj2.tell())
                obj.close()
                self.assertRaises(ValueError, pickle.dumps, obj, proto)
        del __main__.PickleTestMemIO 
Example 26
Project: pyblish-win   Author: pyblish   File: forking.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def __init__(self, process_obj):
            # create pipe for communication with child
            rfd, wfd = os.pipe()

            # get handle for read end of the pipe and make it inheritable
            rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True)
            os.close(rfd)

            # start process
            cmd = get_command_line() + [rhandle]
            cmd = ' '.join('"%s"' % x for x in cmd)
            hp, ht, pid, tid = _subprocess.CreateProcess(
                _python_exe, cmd, None, None, 1, 0, None, None, None
                )
            ht.Close()
            close(rhandle)

            # set attributes of self
            self.pid = pid
            self.returncode = None
            self._handle = hp

            # send information to child
            prep_data = get_preparation_data(process_obj._name)
            to_child = os.fdopen(wfd, 'wb')
            Popen._tls.process_handle = int(hp)
            try:
                dump(prep_data, to_child, HIGHEST_PROTOCOL)
                dump(process_obj, to_child, HIGHEST_PROTOCOL)
            finally:
                del Popen._tls.process_handle
                to_child.close() 
Example 27
Project: Collaborative-Learning-for-Weakly-Supervised-Object-Detection   Author: Sunarker   File: train_val.py    MIT License 5 votes vote down vote up
def snapshot(self, iter):
    net = self.net

    if not os.path.exists(self.output_dir):
      os.makedirs(self.output_dir)

    # Store the model snapshot
    filename = cfg.TRAIN.SNAPSHOT_PREFIX + '_iter_{:d}'.format(iter) + '.pth'
    filename = os.path.join(self.output_dir, filename)
    torch.save(self.net.state_dict(), filename)
    print('Wrote snapshot to: {:s}'.format(filename))
    
    
    if iter % 10000 == 0:
        shutil.copyfile(filename, filename + '.{:d}_cache'.format(iter))
    
    # Also store some meta information, random state, etc.
    nfilename = cfg.TRAIN.SNAPSHOT_PREFIX + '_iter_{:d}'.format(iter) + '.pkl'
    nfilename = os.path.join(self.output_dir, nfilename)
    # current state of numpy random
    st0 = np.random.get_state()
    # current position in the database
    cur = self.data_layer._cur
    # current shuffled indexes of the database
    perm = self.data_layer._perm
    # current position in the validation database
    cur_val = self.data_layer_val._cur
    # current shuffled indexes of the validation database
    perm_val = self.data_layer_val._perm

    # Dump the meta info
    with open(nfilename, 'wb') as fid:
      pickle.dump(st0, fid, pickle.HIGHEST_PROTOCOL)
      pickle.dump(cur, fid, pickle.HIGHEST_PROTOCOL)
      pickle.dump(perm, fid, pickle.HIGHEST_PROTOCOL)
      pickle.dump(cur_val, fid, pickle.HIGHEST_PROTOCOL)
      pickle.dump(perm_val, fid, pickle.HIGHEST_PROTOCOL)
      pickle.dump(iter, fid, pickle.HIGHEST_PROTOCOL)

    return filename, nfilename 
Example 28
Project: Collaborative-Learning-for-Weakly-Supervised-Object-Detection   Author: Sunarker   File: pascal_voc.py    MIT License 5 votes vote down vote up
def selective_search_roidb(self):
    """
    Return the database of selective search regions of interest.
    Ground-truth ROIs are also included.

    This function loads/saves from/to a cache file to speed up future calls.
    """
    cache_file = os.path.join(self.cache_path,
                              self.name + '_selective_search_roidb.pkl')

    if os.path.exists(cache_file):
        with open(cache_file, 'rb') as fid:
            roidb = pickle.load(fid)
        print('{} ss roidb loaded from {}'.format(self.name, cache_file))
        return roidb

    if int(self._year) == 2007 or self._image_set != 'test':
        gt_roidb = self.gt_roidb()
        # ss_roidb = self._load_selective_search_roidb(gt_roidb)
        # roidb = datasets.imdb.merge_roidbs(gt_roidb, ss_roidb)
        roidb = self._load_selective_search_roidb(gt_roidb)
    else:
        roidb = self._load_selective_search_roidb(None)
    with open(cache_file, 'wb') as fid:
        pickle.dump(roidb, fid, pickle.HIGHEST_PROTOCOL)
    print('wrote ss roidb to {}'.format(cache_file))

    return roidb 
Example 29
Project: Collaborative-Learning-for-Weakly-Supervised-Object-Detection   Author: Sunarker   File: coco.py    MIT License 5 votes vote down vote up
def _do_detection_eval(self, res_file, output_dir):
    ann_type = 'bbox'
    coco_dt = self._COCO.loadRes(res_file)
    coco_eval = COCOeval(self._COCO, coco_dt)
    coco_eval.params.useSegm = (ann_type == 'segm')
    coco_eval.evaluate()
    coco_eval.accumulate()
    self._print_detection_eval_metrics(coco_eval)
    eval_file = osp.join(output_dir, 'detection_results.pkl')
    with open(eval_file, 'wb') as fid:
      pickle.dump(coco_eval, fid, pickle.HIGHEST_PROTOCOL)
    print('Wrote COCO eval results to: {}'.format(eval_file)) 
Example 30
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: sessions.py    Apache License 2.0 5 votes vote down vote up
def save(self, session):
        fn = self.get_session_filename(session.sid)
        fd, tmp = tempfile.mkstemp(suffix=_fs_transaction_suffix,
                                   dir=self.path)
        f = os.fdopen(fd, 'wb')
        try:
            dump(dict(session), f, HIGHEST_PROTOCOL)
        finally:
            f.close()
        try:
            rename(tmp, fn)
            os.chmod(fn, self.mode)
        except (IOError, OSError):
            pass 
Example 31
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: cache.py    Apache License 2.0 5 votes vote down vote up
def set(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        self._prune()
        self._cache[key] = (time() + timeout, pickle.dumps(value,
            pickle.HIGHEST_PROTOCOL))
        return True 
Example 32
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: cache.py    Apache License 2.0 5 votes vote down vote up
def add(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        self._prune()
        item = (time() + timeout, pickle.dumps(value,
            pickle.HIGHEST_PROTOCOL))
        if key in self._cache:
            return False
        self._cache.setdefault(key, item)
        return True 
Example 33
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: sessions.py    Apache License 2.0 5 votes vote down vote up
def save(self, session):
        fn = self.get_session_filename(session.sid)
        fd, tmp = tempfile.mkstemp(suffix=_fs_transaction_suffix,
                                   dir=self.path)
        f = os.fdopen(fd, 'wb')
        try:
            dump(dict(session), f, HIGHEST_PROTOCOL)
        finally:
            f.close()
        try:
            rename(tmp, fn)
            os.chmod(fn, self.mode)
        except (IOError, OSError):
            pass 
Example 34
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: cache.py    Apache License 2.0 5 votes vote down vote up
def add(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        self._prune()
        item = (time() + timeout, pickle.dumps(value,
            pickle.HIGHEST_PROTOCOL))
        if key in self._cache:
            return False
        self._cache.setdefault(key, item)
        return True 
Example 35
Project: OpenAPS   Author: medicinexlab   File: savedata.py    MIT License 5 votes vote down vote up
def save_data(data_item, name):
    """
    This function saves the data as a pickle file.
    """

    with open('./savedata/' + name + '.pickle', 'wb') as handle:
        pickle.dump(data_item, handle, protocol=pickle.HIGHEST_PROTOCOL) 
Example 36
Project: FasterRCNN_TF_Py3   Author: upojzsb   File: coco.py    MIT License 5 votes vote down vote up
def _do_detection_eval(self, res_file, output_dir):
        ann_type = 'bbox'
        coco_dt = self._COCO.loadRes(res_file)
        coco_eval = COCOeval(self._COCO, coco_dt)
        coco_eval.params.useSegm = (ann_type == 'segm')
        coco_eval.evaluate()
        coco_eval.accumulate()
        self._print_detection_eval_metrics(coco_eval)
        eval_file = osp.join(output_dir, 'detection_results.pkl')
        with open(eval_file, 'wb') as fid:
            pickle.dump(coco_eval, fid, pickle.HIGHEST_PROTOCOL)
        print('Wrote COCO eval results to: {}'.format(eval_file)) 
Example 37
Project: FasterRCNN_TF_Py3   Author: upojzsb   File: train.py    MIT License 5 votes vote down vote up
def snapshot(self, sess, iter):
        net = self.net

        if not os.path.exists(self.output_dir):
            os.makedirs(self.output_dir)

        # Store the model snapshot
        filename = 'vgg16_faster_rcnn_iter_{:d}'.format(iter) + '.ckpt'
        filename = os.path.join(self.output_dir, filename)
        self.saver.save(sess, filename)
        print('Wrote snapshot to: {:s}'.format(filename))

        # Also store some meta information, random state, etc.
        nfilename = 'vgg16_faster_rcnn_iter_{:d}'.format(iter) + '.pkl'
        nfilename = os.path.join(self.output_dir, nfilename)
        # current state of numpy random
        st0 = np.random.get_state()
        # current position in the database
        cur = self.data_layer._cur
        # current shuffled indeces of the database
        perm = self.data_layer._perm

        # Dump the meta info
        with open(nfilename, 'wb') as fid:
            pickle.dump(st0, fid, pickle.HIGHEST_PROTOCOL)
            pickle.dump(cur, fid, pickle.HIGHEST_PROTOCOL)
            pickle.dump(perm, fid, pickle.HIGHEST_PROTOCOL)
            pickle.dump(iter, fid, pickle.HIGHEST_PROTOCOL)

        return filename, nfilename 
Example 38
Project: zmirror   Author: aploium   File: cache_system.py    MIT License 5 votes vote down vote up
def put_obj(self, key, obj, expires=DEFAULT_EXPIRE, obj_size=0, last_modified=None, info_dict=None):
        """
        将一个对象存入缓存
        :param key: key
        :param last_modified: str  format: "Mon, 18 Nov 2013 09:02:42 GMT"
        :param obj_size: too big object should not be cached
        :param expires: seconds to expire
        :param info_dict: custom dict contains information, stored in memory, so can access quickly
        :type key: str
        :type last_modified: str
        :type info_dict: dict or None
        :type obj: Any
        """
        if expires <= 0 or obj_size > self.max_size_byte:
            return False

        self.delete(key)

        temp_file = tempfile.NamedTemporaryFile(prefix="zmirror_", suffix=".tmp", delete=False)
        pickle.dump(obj, temp_file, protocol=pickle.HIGHEST_PROTOCOL)

        cache_item = (
            temp_file.name,  # 0 cache file path
            info_dict,  # 1 custom dict contains information
            int(time.time()),  # 2 added time (unix time)
            expires,  # 3 expires second
            _time_str_to_unix(last_modified),  # 4 last modified, unix time
        )
        temp_file.close()
        self.items_dict[key] = cache_item
        return True 
Example 39
Project: zmirror   Author: aploium   File: zmirror.py    MIT License 5 votes vote down vote up
def dump_zmirror_snapshot(folder="error_dump", msg=None, our_response=None):
    """
    dump当前状态到文件
    :param folder: 文件夹名
    :type folder: str
    :param our_response: Flask返回对象, 可选
    :type our_response: Response
    :param msg: 额外的信息
    :type msg: str
    :return: dump下来的文件绝对路径
    :rtype: Union[str, None]
    """
    import pickle
    try:
        if not os.path.exists(zmirror_root(folder)):
            os.mkdir(zmirror_root(folder))
        _time_str = datetime.now().strftime('snapshot_%Y-%m-%d_%H-%M-%S')

        import config

        snapshot = {
            "time": datetime.now(),
            "parse": parse.dump(),
            "msg": msg,
            "traceback": traceback.format_exc(),
            "config": attributes(config, to_dict=True),
            "FlaskRequest": attributes(request, to_dict=True),
        }
        if our_response is not None:
            our_response.freeze()
        snapshot["OurResponse"] = our_response

        dump_file_path = os.path.abspath(os.path.join(zmirror_root(folder), _time_str + '.dump'))

        with open(dump_file_path, 'wb') as fp:
            pickle.dump(snapshot, fp, pickle.HIGHEST_PROTOCOL)
        return dump_file_path
    except:
        return None 
Example 40
Project: disentangling_conditional_gans   Author: zalandoresearch   File: misc.py    MIT License 5 votes vote down vote up
def save_pkl(obj, filename):
    with open(filename, 'wb') as file:
        pickle.dump(obj, file, protocol=pickle.HIGHEST_PROTOCOL)

#----------------------------------------------------------------------------
# Image utils. 
Example 41
Project: VSE-C   Author: ExplorerFreda   File: vocab.py    MIT License 5 votes vote down vote up
def main(data_path, data_name):
    vocab = build_vocab(data_path, data_name, jsons=annotations, threshold=4)
    with open('./vocab/%s_vocab.pkl' % data_name, 'wb') as f:
        pickle.dump(vocab, f, pickle.HIGHEST_PROTOCOL)
    print(("Saved vocabulary file to ", './vocab/%s_vocab.pkl' % data_name)) 
Example 42
Project: flasky   Author: RoseOu   File: sessions.py    MIT License 5 votes vote down vote up
def save(self, session):
        fn = self.get_session_filename(session.sid)
        fd, tmp = tempfile.mkstemp(suffix=_fs_transaction_suffix,
                                   dir=self.path)
        f = os.fdopen(fd, 'wb')
        try:
            dump(dict(session), f, HIGHEST_PROTOCOL)
        finally:
            f.close()
        try:
            rename(tmp, fn)
            os.chmod(fn, self.mode)
        except (IOError, OSError):
            pass 
Example 43
Project: flasky   Author: RoseOu   File: cache.py    MIT License 5 votes vote down vote up
def set(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        self._prune()
        self._cache[key] = (time() + timeout, pickle.dumps(value,
            pickle.HIGHEST_PROTOCOL))
        return True 
Example 44
Project: flasky   Author: RoseOu   File: cache.py    MIT License 5 votes vote down vote up
def add(self, key, value, timeout=None):
        if timeout is None:
            timeout = self.default_timeout
        self._prune()
        item = (time() + timeout, pickle.dumps(value,
            pickle.HIGHEST_PROTOCOL))
        if key in self._cache:
            return False
        self._cache.setdefault(key, item)
        return True 
Example 45
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: ner.py    Apache License 2.0 5 votes vote down vote up
def save_obj(obj, name):
    with open(name + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL) 
Example 46
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: imdb.py    Apache License 2.0 5 votes vote down vote up
def evaluate_detections(self, detections, **kwargs):
        cache_path = os.path.join(self._root_path, 'cache', '{}_{}.pkl'.format(self._name, 'detections'))
        logger.info('saving cache {}'.format(cache_path))
        with open(cache_path, 'wb') as fid:
            pickle.dump(detections, fid, pickle.HIGHEST_PROTOCOL)
        self._evaluate_detections(detections, **kwargs) 
Example 47
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: imdb.py    Apache License 2.0 5 votes vote down vote up
def _get_cached(self, cache_item, fn):
        cache_path = os.path.join(self._root_path, 'cache', '{}_{}.pkl'.format(self._name, cache_item))
        if os.path.exists(cache_path):
            logger.info('loading cache {}'.format(cache_path))
            with open(cache_path, 'rb') as fid:
                cached = pickle.load(fid)
            return cached
        else:
            logger.info('computing cache {}'.format(cache_path))
            cached = fn()
            logger.info('saving cache {}'.format(cache_path))
            with open(cache_path, 'wb') as fid:
                pickle.dump(cached, fid, pickle.HIGHEST_PROTOCOL)
            return cached 
Example 48
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: dataloader.py    Apache License 2.0 5 votes vote down vote up
def send(self, obj):
        """Send object"""
        buf = io.BytesIO()
        ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj)
        self.send_bytes(buf.getvalue()) 
Example 49
Project: FCOS_GluonCV   Author: DetectionTeamUCAS   File: dataloader.py    Apache License 2.0 5 votes vote down vote up
def _worker_fn(samples, transform_fn, batchify_fn):
    """Function for processing data in worker process."""
    # it is required that each worker process has to fork a new MXIndexedRecordIO handle
    # preserving dataset as global variable can save tons of overhead and is safe in new process
    global _worker_dataset
    t_dataset = _worker_dataset.transform(transform_fn)
    batch = batchify_fn([t_dataset[i] for i in samples])
    buf = io.BytesIO()
    ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(batch)
    return buf.getvalue() 
Example 50
Project: plugin.program.steam.streaming   Author: Drakulix   File: comm.py    MIT License 5 votes vote down vote up
def _write(_socket, obj):
    data = pickle.dumps(obj, pickle.HIGHEST_PROTOCOL)
    _socket.send(struct.pack("=i", len(data)))
    _socket.send(struct.pack("="+str(len(data))+"s", data))

#read object