Python pickle.Pickler() Examples

The following are code examples for showing how to use pickle.Pickler(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: RSSMOSPipeline   Author: mattyowl   File: fixRefModel.py    GNU General Public License v3.0 6 votes vote down vote up
def makeModelArcSpectrum(data, approxCoordsDict, outFileName, yRow, sigmaCut = 3.0, thresholdSigma = 5.0, 
                         featureMinPix = 30):
    """Make reference model arc spectrum. This has wavelengths of features identified in a table. We also
    save the middle row of the spectrum.
    
    """
    
    # Detect and tag features with known wavelengths in reference spectrum
    featureTable, segMap=detectLines(data)
    featureTable=tagWavelengthFeatures(featureTable, approxCoordsDict)
    data_centreRow=data[yRow]
    
    # Save reference model as a pickled dictionary
    refModelDict={'featureTable': featureTable, 'arc_centreRow': data_centreRow}
    pickleFile=file(outFileName, "wb")
    pickler=pickle.Pickler(pickleFile)
    pickler.dump(refModelDict)
    pickleFile.close()
    
#------------------------------------------------------------------------------------------------------------- 
Example 2
Project: FalltoSkyBot   Author: Sakiut   File: casino.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def post(server, user, data, money):
	if server.name in data.keys():
		if user.name in data[server.name].keys():
			data[server.name][user.name] = money
		else:
			uniqueData = {user.name: 500}
			data[server.name].update(uniqueData)
	else:
		uniqueData = {server.name:{user.name: 500}}
		data.update(uniqueData)

	f = open(fileName, "wb")
	p = pickle.Pickler(f)
	p.dump(data)
	f.close()

	return data

######################################################################################################################## 
Example 3
Project: FalltoSkyBot   Author: Sakiut   File: moderation.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def warn(server, user, data):
	if server.name in data.keys():
		if user.name in data[server.name].keys():
			data[server.name][user.name] += 1
		else:
			uniqueData = {user.name: 1}
			data[server.name].update(uniqueData)
	else:
		uniqueData = {server.name:{user.name: 1}}
		data.update(uniqueData)

	f = open(fileName, "wb")
	p = pickle.Pickler(f)
	p.dump(data)
	f.close()

	return data

##################################################################################################################################################### 
Example 4
Project: NiujiaoDebugger   Author: MrSrc   File: test_pickle.py    GNU General Public License v3.0 6 votes vote down vote up
def test_pickler(self):
            basesize = support.calcobjsize('6P2n3i2n3iP')
            p = _pickle.Pickler(io.BytesIO())
            self.assertEqual(object.__sizeof__(p), basesize)
            MT_size = struct.calcsize('3nP0n')
            ME_size = struct.calcsize('Pn0P')
            check = self.check_sizeof
            check(p, basesize +
                MT_size + 8 * ME_size +  # Minimal memo table size.
                sys.getsizeof(b'x'*4096))  # Minimal write buffer size.
            for i in range(6):
                p.dump(chr(i))
            check(p, basesize +
                MT_size + 32 * ME_size +  # Size of memo table required to
                                          # save references to 6 objects.
                0)  # Write buffer is cleared after every dump(). 
Example 5
Project: D-VAE   Author: muhanzhang   File: pkl_utils.py    MIT License 6 votes vote down vote up
def save(self, obj):
        # Remove the tag.trace attribute from Variable and Apply nodes
        if isinstance(obj, theano.gof.utils.scratchpad):
            for tag in self.tag_to_remove:
                if hasattr(obj, tag):
                    del obj.__dict__[tag]
        # Remove manually-added docstring of Elemwise ops
        elif (isinstance(obj, theano.tensor.Elemwise)):
            if '__doc__' in obj.__dict__:
                del obj.__dict__['__doc__']

        return Pickler.save(self, obj)


# Make an unpickler that tries encoding byte streams before raising TypeError.
# This is useful with python 3, in order to unpickle files created with
# python 2.
# This code is taken from Pandas, https://github.com/pydata/pandas,
# under the same 3-clause BSD license. 
Example 6
Project: estimators   Author: fridiculous   File: hashing.py    MIT License 6 votes vote down vote up
def save(self, obj):
        if isinstance(obj, (types.MethodType, type({}.pop))):
            # the Pickler cannot pickle instance methods; here we decompose
            # them into components that make them uniquely identifiable
            if hasattr(obj, '__func__'):
                func_name = obj.__func__.__name__
            else:
                func_name = obj.__name__
            inst = obj.__self__
            if type(inst) == type(pickle):
                obj = _MyHash(func_name, inst.__name__)
            elif inst is None:
                # type(None) or type(module) do not pickle
                obj = _MyHash(func_name, inst)
            else:
                cls = obj.__self__.__class__
                obj = _MyHash(func_name, inst, cls)
        Pickler.save(self, obj) 
Example 7
Project: estimators   Author: fridiculous   File: hashing.py    MIT License 6 votes vote down vote up
def save_global(self, obj, name=None, pack=struct.pack):
        # We have to override this method in order to deal with objects
        # defined interactively in IPython that are not injected in
        # __main__
        kwargs = dict(name=name, pack=pack)
        if sys.version_info >= (3, 4):
            del kwargs['pack']
        try:
            Pickler.save_global(self, obj, **kwargs)
        except pickle.PicklingError:
            Pickler.save_global(self, obj, **kwargs)
            module = getattr(obj, "__module__", None)
            if module == '__main__':
                my_name = name
                if my_name is None:
                    my_name = obj.__name__
                mod = sys.modules[module]
                if not hasattr(mod, my_name):
                    # IPython doesn't inject the variables define
                    # interactively in __main__
                    setattr(mod, my_name, obj) 
Example 8
Project: mlens   Author: flennerhag   File: hashing.py    MIT License 6 votes vote down vote up
def save(self, obj):
        if isinstance(obj, (types.MethodType, type({}.pop))):
            # the Pickler cannot pickle instance methods; here we decompose
            # them into components that make them uniquely identifiable
            if hasattr(obj, '__func__'):
                func_name = obj.__func__.__name__
            else:
                func_name = obj.__name__
            inst = obj.__self__
            if type(inst) == type(pickle):
                obj = _MyHash(func_name, inst.__name__)
            elif inst is None:
                # type(None) or type(module) do not pickle
                obj = _MyHash(func_name, inst)
            else:
                cls = obj.__self__.__class__
                obj = _MyHash(func_name, inst, cls)
        Pickler.save(self, obj) 
Example 9
Project: mlens   Author: flennerhag   File: hashing.py    MIT License 6 votes vote down vote up
def save_global(self, obj, name=None, pack=struct.pack):
        # We have to override this method in order to deal with objects
        # defined interactively in IPython that are not injected in
        # __main__
        kwargs = dict(name=name, pack=pack)
        if sys.version_info >= (3, 4):
            del kwargs['pack']
        try:
            Pickler.save_global(self, obj, **kwargs)
        except pickle.PicklingError:
            Pickler.save_global(self, obj, **kwargs)
            module = getattr(obj, "__module__", None)
            if module == '__main__':
                my_name = name
                if my_name is None:
                    my_name = obj.__name__
                mod = sys.modules[module]
                if not hasattr(mod, my_name):
                    # IPython doesn't inject the variables define
                    # interactively in __main__
                    setattr(mod, my_name, obj) 
Example 10
Project: cqp-sdk-for-py37-native   Author: crud-boy   File: test_pickle.py    GNU General Public License v2.0 6 votes vote down vote up
def test_pickler(self):
            basesize = support.calcobjsize('6P2n3i2n3iP')
            p = _pickle.Pickler(io.BytesIO())
            self.assertEqual(object.__sizeof__(p), basesize)
            MT_size = struct.calcsize('3nP0n')
            ME_size = struct.calcsize('Pn0P')
            check = self.check_sizeof
            check(p, basesize +
                MT_size + 8 * ME_size +  # Minimal memo table size.
                sys.getsizeof(b'x'*4096))  # Minimal write buffer size.
            for i in range(6):
                p.dump(chr(i))
            check(p, basesize +
                MT_size + 32 * ME_size +  # Size of memo table required to
                                          # save references to 6 objects.
                0)  # Write buffer is cleared after every dump(). 
Example 11
Project: poker   Author: surgebiswas   File: pkl_utils.py    MIT License 6 votes vote down vote up
def save(self, obj):
        # Remove the tag.trace attribute from Variable and Apply nodes
        if isinstance(obj, theano.gof.utils.scratchpad):
            for tag in self.tag_to_remove:
                if hasattr(obj, tag):
                    del obj.__dict__[tag]
        # Remove manually-added docstring of Elemwise ops
        elif (isinstance(obj, theano.tensor.Elemwise)):
            if '__doc__' in obj.__dict__:
                del obj.__dict__['__doc__']

        return Pickler.save(self, obj)


# Make an unpickler that tries encoding byte streams before raising TypeError.
# This is useful with python 3, in order to unpickle files created with
# python 2.
# This code is taken from Pandas, https://github.com/pydata/pandas,
# under the same 3-clause BSD license. 
Example 12
Project: RPGOne   Author: RTHMaK   File: hashing.py    Apache License 2.0 6 votes vote down vote up
def save(self, obj):
        if isinstance(obj, (types.MethodType, type({}.pop))):
            # the Pickler cannot pickle instance methods; here we decompose
            # them into components that make them uniquely identifiable
            if hasattr(obj, '__func__'):
                func_name = obj.__func__.__name__
            else:
                func_name = obj.__name__
            inst = obj.__self__
            if type(inst) == type(pickle):
                obj = _MyHash(func_name, inst.__name__)
            elif inst is None:
                # type(None) or type(module) do not pickle
                obj = _MyHash(func_name, inst)
            else:
                cls = obj.__self__.__class__
                obj = _MyHash(func_name, inst, cls)
        Pickler.save(self, obj) 
Example 13
Project: RPGOne   Author: RTHMaK   File: hashing.py    Apache License 2.0 6 votes vote down vote up
def save_global(self, obj, name=None, pack=struct.pack):
        # We have to override this method in order to deal with objects
        # defined interactively in IPython that are not injected in
        # __main__
        kwargs = dict(name=name, pack=pack)
        if sys.version_info >= (3, 4):
            del kwargs['pack']
        try:
            Pickler.save_global(self, obj, **kwargs)
        except pickle.PicklingError:
            Pickler.save_global(self, obj, **kwargs)
            module = getattr(obj, "__module__", None)
            if module == '__main__':
                my_name = name
                if my_name is None:
                    my_name = obj.__name__
                mod = sys.modules[module]
                if not hasattr(mod, my_name):
                    # IPython doesn't inject the variables define
                    # interactively in __main__
                    setattr(mod, my_name, obj) 
Example 14
Project: attention-lvcsr   Author: rizar   File: pkl_utils.py    MIT License 6 votes vote down vote up
def save(self, obj):
        # Remove the tag.trace attribute from Variable and Apply nodes
        if isinstance(obj, theano.gof.utils.scratchpad):
            for tag in self.tag_to_remove:
                if hasattr(obj, tag):
                    del obj.__dict__[tag]
        # Remove manually-added docstring of Elemwise ops
        elif (isinstance(obj, theano.tensor.Elemwise)):
            if '__doc__' in obj.__dict__:
                del obj.__dict__['__doc__']

        return Pickler.save(self, obj)


# Make an unpickler that tries encoding byte streams before raising TypeError.
# This is useful with python 3, in order to unpickle files created with
# python 2.
# This code is taken from Pandas, https://github.com/pydata/pandas,
# under the same 3-clause BSD license. 
Example 15
Project: pyblish-win   Author: pyblish   File: shelve.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 
Example 16
Project: pyblish-win   Author: pyblish   File: test_pickle.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def dumps(self, arg, proto=0, fast=0):
        f = StringIO()
        p = pickle.Pickler(f, proto)
        if fast:
            p.fast = fast
        p.dump(arg)
        f.seek(0)
        return f.read() 
Example 17
Project: pyblish-win   Author: pyblish   File: test_pickle.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def dumps(self, arg, proto=0, fast=0):
        class PersPickler(pickle.Pickler):
            def persistent_id(subself, obj):
                return self.persistent_id(obj)
        f = StringIO()
        p = PersPickler(f, proto)
        if fast:
            p.fast = fast
        p.dump(arg)
        f.seek(0)
        return f.read() 
Example 18
Project: NiujiaoDebugger   Author: MrSrc   File: shelve.py    GNU General Public License v3.0 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = BytesIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key.encode(self.keyencoding)] = f.getvalue() 
Example 19
Project: NiujiaoDebugger   Author: MrSrc   File: test_pickle.py    GNU General Public License v3.0 5 votes vote down vote up
def test_pickler_reference_cycle(self):
        def check(Pickler):
            for proto in range(pickle.HIGHEST_PROTOCOL + 1):
                f = io.BytesIO()
                pickler = Pickler(f, proto)
                pickler.dump('abc')
                self.assertEqual(self.loads(f.getvalue()), 'abc')
            pickler = Pickler(io.BytesIO())
            self.assertEqual(pickler.persistent_id('def'), 'def')
            r = weakref.ref(pickler)
            del pickler
            self.assertIsNone(r())

        class PersPickler(self.pickler):
            def persistent_id(subself, obj):
                return obj
        check(PersPickler)

        class PersPickler(self.pickler):
            @classmethod
            def persistent_id(cls, obj):
                return obj
        check(PersPickler)

        class PersPickler(self.pickler):
            @staticmethod
            def persistent_id(obj):
                return obj
        check(PersPickler) 
Example 20
Project: 2018-1-OSS-E7   Author: 18-1-SKKU-OSS   File: CNCList.py    MIT License 5 votes vote down vote up
def copy(self, event=None):
		sio = StringIO()
		pickler = pickle.Pickler(sio)
		#sio.write(_PLOT_CLIP)
		for block,line in self.getCleanSelection():
			if line is None:
				pickler.dump(self.gcode.blocks[block].dump())
			else:
				pickler.dump(self.gcode.blocks[block][line])
		self.clipboard_clear()
		self.clipboard_append(sio.getvalue())
		return "break"

	# ---------------------------------------------------------------------- 
Example 21
Project: rainmachine-developer-resources   Author: sprinkler   File: rmParserUserData.py    GNU General Public License v3.0 5 votes vote down vote up
def RMUserData_adaptToSQLite(userData):
    if userData == None:
        return None

    outputStream = StringIO()
    pickler = pickle.Pickler(outputStream)
    pickler.dump(userData)
    return outputStream.getvalue() 
Example 22
Project: rainmachine-developer-resources   Author: sprinkler   File: rmParserParams.py    GNU General Public License v3.0 5 votes vote down vote up
def RMParserParams_adaptToSQLite(params):
    if params == None:
        return None

    outputStream = StringIO()
    pickler = pickle.Pickler(outputStream)
    pickler.dump(params)
    return outputStream.getvalue() 
Example 23
Project: ironpython2   Author: IronLanguages   File: shelve.py    Apache License 2.0 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 
Example 24
Project: ironpython2   Author: IronLanguages   File: test_pickle.py    Apache License 2.0 5 votes vote down vote up
def dumps(self, arg, proto=0, fast=0):
        f = StringIO()
        p = pickle.Pickler(f, proto)
        if fast:
            p.fast = fast
        p.dump(arg)
        f.seek(0)
        return f.read() 
Example 25
Project: ironpython2   Author: IronLanguages   File: test_pickle.py    Apache License 2.0 5 votes vote down vote up
def dumps(self, arg, proto=0, fast=0):
        class PersPickler(pickle.Pickler):
            def persistent_id(subself, obj):
                return self.persistent_id(obj)
        f = StringIO()
        p = PersPickler(f, proto)
        if fast:
            p.fast = fast
        p.dump(arg)
        f.seek(0)
        return f.read() 
Example 26
Project: ironpython2   Author: IronLanguages   File: test_cPickle.py    Apache License 2.0 5 votes vote down vote up
def test_pers_load(self):
        for binary in [True, False]:
            src = StringIO()
            p = cPickle.Pickler(src)
            p.persistent_id = persistent_id
            p.binary = binary

            value = MyData('abc')
            p.dump(value)

            up = cPickle.Unpickler(StringIO(src.getvalue()))
            up.persistent_load = persistent_load
            res = up.load()

            self.assertEqual(res.value, value.value)

            # errors
            src = StringIO()
            p = cPickle.Pickler(src)
            p.persistent_id = persistent_id
            p.binary = binary

            value = MyData('abc')
            p.dump(value)

            up = cPickle.Unpickler(StringIO(src.getvalue()))

            # exceptions vary betwee cPickle & Pickle
            try:
                up.load()
                self.assertUnreachable()
            except Exception, e:
                pass 
Example 27
Project: D-VAE   Author: muhanzhang   File: pkl_utils.py    MIT License 5 votes vote down vote up
def __init__(self, file, protocol=0, extra_tag_to_remove=None):
        # Can't use super as Pickler isn't a new style class
        Pickler.__init__(self, file, protocol)
        self.tag_to_remove = ['trace', 'test_value']
        if extra_tag_to_remove:
            self.tag_to_remove.extend(extra_tag_to_remove) 
Example 28
Project: open-recipe   Author: dspray95   File: gencache.py    The Unlicense 5 votes vote down vote up
def _SaveDicts():
	if is_readonly:
		raise RuntimeError("Trying to write to a readonly gencache ('%s')!" \
		                    % win32com.__gen_path__)
	f = open(os.path.join(GetGeneratePath(), "dicts.dat"), "wb")
	try:
		p = pickle.Pickler(f)
		p.dump(pickleVersion)
		p.dump(clsidToTypelib)
	finally:
		f.close() 
Example 29
Project: Blockly-rduino-communication   Author: technologiescollege   File: shelve.py    GNU General Public License v3.0 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = BytesIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key.encode(self.keyencoding)] = f.getvalue() 
Example 30
Project: IronHydra   Author: microdee   File: shelve.py    MIT License 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 
Example 31
Project: estimators   Author: fridiculous   File: hashing.py    MIT License 5 votes vote down vote up
def __init__(self, hash_name='md5'):
        self.stream = io.BytesIO()
        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        protocol = (pickle.DEFAULT_PROTOCOL if PY3_OR_LATER
                    else pickle.HIGHEST_PROTOCOL)
        Pickler.__init__(self, self.stream, protocol=protocol)
        # Initialise the hash obj
        self._hash = hashlib.new(hash_name) 
Example 32
Project: estimators   Author: fridiculous   File: hashing.py    MIT License 5 votes vote down vote up
def memoize(self, obj):
        # We want hashing to be sensitive to value instead of reference.
        # For example we want ['aa', 'aa'] and ['aa', 'aaZ'[:2]]
        # to hash to the same value and that's why we disable memoization
        # for strings
        if isinstance(obj, _bytes_or_unicode):
            return
        Pickler.memoize(self, obj)

    # The dispatch table of the pickler is not accessible in Python
    # 3, as these lines are only bugware for IPython, we skip them. 
Example 33
Project: estimators   Author: fridiculous   File: hashing.py    MIT License 5 votes vote down vote up
def _batch_setitems(self, items):
        # forces order of keys in dict to ensure consistent hash.
        try:
            # Trying first to compare dict assuming the type of keys is
            # consistent and orderable.
            # This fails on python 3 when keys are unorderable
            # but we keep it in a try as it's faster.
            Pickler._batch_setitems(self, iter(sorted(items)))
        except TypeError:
            # If keys are unorderable, sorting them using their hash. This is
            # slower but works in any case.
            Pickler._batch_setitems(self, iter(sorted((hash(k), v)
                                                      for k, v in items))) 
Example 34
Project: deepjets   Author: deepjets   File: tasksystem.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def __init__(self, *args, **kwargs):
        if not "protocol" in kwargs:
            kwargs["protocol"] = pickle.HIGHEST_PROTOCOL
        pickle.Pickler.__init__(self, *args, **kwargs) 
Example 35
Project: deepjets   Author: deepjets   File: tasksystem.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def checkExec():
        if "--forkExecProc" in sys.argv:
            try:
                import better_exchook
            except ImportError:
                pass    # Doesn't matter.
            else:
                better_exchook.install()
            argidx = sys.argv.index("--forkExecProc")
            writeFileNo = int(sys.argv[argidx + 1])
            readFileNo = int(sys.argv[argidx + 2])
            readend = os.fdopen(readFileNo, "r")
            writeend = os.fdopen(writeFileNo, "w")
            unpickler = Unpickler(readend)
            name = unpickler.load()
            if ExecingProcess.Verbose: print("ExecingProcess child %s (pid %i)" % (name, os.getpid()))
            try:
                target = unpickler.load()
                args = unpickler.load()
            except EOFError:
                print("Error: unpickle incomplete")
                raise SystemExit
            ret = target(*args)
            # IOError is probably broken pipe. That probably means that the parent died.
            try: Pickler(writeend).dump(ret)
            except IOError: pass
            try: readend.close()
            except IOError: pass
            try: writeend.close()
            except IOError: pass
            if ExecingProcess.Verbose: print("ExecingProcess child %s (pid %i) finished" % (name, os.getpid()))
            raise SystemExit 
Example 36
Project: deepjets   Author: deepjets   File: tasksystem.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def send(self, value):
        self._check_closed()
        self._check_writable()
        buf = StringIO()
        Pickler(buf).dump(value)
        self.conn.send_bytes(buf.getvalue()) 
Example 37
Project: mlens   Author: flennerhag   File: hashing.py    MIT License 5 votes vote down vote up
def __init__(self, hash_name='md5'):
        self.stream = io.BytesIO()
        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        protocol = (pickle.DEFAULT_PROTOCOL if PY3_OR_LATER
                    else pickle.HIGHEST_PROTOCOL)
        Pickler.__init__(self, self.stream, protocol=protocol)
        # Initialise the hash obj
        self._hash = hashlib.new(hash_name) 
Example 38
Project: mlens   Author: flennerhag   File: hashing.py    MIT License 5 votes vote down vote up
def memoize(self, obj):
        # We want hashing to be sensitive to value instead of reference.
        # For example we want ['aa', 'aa'] and ['aa', 'aaZ'[:2]]
        # to hash to the same value and that's why we disable memoization
        # for strings
        if isinstance(obj, _bytes_or_unicode):
            return
        Pickler.memoize(self, obj)

    # The dispatch table of the pickler is not accessible in Python
    # 3, as these lines are only bugware for IPython, we skip them. 
Example 39
Project: mlens   Author: flennerhag   File: hashing.py    MIT License 5 votes vote down vote up
def _batch_setitems(self, items):
        # forces order of keys in dict to ensure consistent hash.
        try:
            # Trying first to compare dict assuming the type of keys is
            # consistent and orderable.
            # This fails on python 3 when keys are unorderable
            # but we keep it in a try as it's faster.
            Pickler._batch_setitems(self, iter(sorted(items)))
        except TypeError:
            # If keys are unorderable, sorting them using their hash. This is
            # slower but works in any case.
            Pickler._batch_setitems(self, iter(sorted((hash(k), v)
                                                      for k, v in items))) 
Example 40
Project: cqp-sdk-for-py37-native   Author: crud-boy   File: shelve.py    GNU General Public License v2.0 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = BytesIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key.encode(self.keyencoding)] = f.getvalue() 
Example 41
Project: cqp-sdk-for-py37-native   Author: crud-boy   File: test_pickle.py    GNU General Public License v2.0 5 votes vote down vote up
def test_pickler_reference_cycle(self):
        def check(Pickler):
            for proto in range(pickle.HIGHEST_PROTOCOL + 1):
                f = io.BytesIO()
                pickler = Pickler(f, proto)
                pickler.dump('abc')
                self.assertEqual(self.loads(f.getvalue()), 'abc')
            pickler = Pickler(io.BytesIO())
            self.assertEqual(pickler.persistent_id('def'), 'def')
            r = weakref.ref(pickler)
            del pickler
            self.assertIsNone(r())

        class PersPickler(self.pickler):
            def persistent_id(subself, obj):
                return obj
        check(PersPickler)

        class PersPickler(self.pickler):
            @classmethod
            def persistent_id(cls, obj):
                return obj
        check(PersPickler)

        class PersPickler(self.pickler):
            @staticmethod
            def persistent_id(obj):
                return obj
        check(PersPickler) 
Example 42
Project: aws-lambda-runtime-pypy   Author: uscheller   File: shelve.py    Apache License 2.0 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = BytesIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key.encode(self.keyencoding)] = f.getvalue() 
Example 43
Project: Computable   Author: ktraunmueller   File: shelve.py    MIT License 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 
Example 44
Project: oss-ftp   Author: aliyun   File: shelve.py    MIT License 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 
Example 45
Project: oss-ftp   Author: aliyun   File: test_pickle.py    MIT License 5 votes vote down vote up
def dumps(self, arg, proto=0, fast=0):
        f = StringIO()
        p = pickle.Pickler(f, proto)
        if fast:
            p.fast = fast
        p.dump(arg)
        f.seek(0)
        return f.read() 
Example 46
Project: oss-ftp   Author: aliyun   File: test_pickle.py    MIT License 5 votes vote down vote up
def dumps(self, arg, proto=0, fast=0):
        class PersPickler(pickle.Pickler):
            def persistent_id(subself, obj):
                return self.persistent_id(obj)
        f = StringIO()
        p = PersPickler(f, proto)
        if fast:
            p.fast = fast
        p.dump(arg)
        f.seek(0)
        return f.read() 
Example 47
Project: poker   Author: surgebiswas   File: pkl_utils.py    MIT License 5 votes vote down vote up
def __init__(self, file, protocol=0, extra_tag_to_remove=None):
        # Can't use super as Pickler isn't a new style class
        Pickler.__init__(self, file, protocol)
        self.tag_to_remove = ['trace', 'test_value']
        if extra_tag_to_remove:
            self.tag_to_remove.extend(extra_tag_to_remove) 
Example 48
Project: goldminer   Author: adael   File: filesave.py    MIT License 5 votes vote down vote up
def save_world(world):
    with open(filename, 'wb') as fout:
        pickler = pickle.Pickler(fout, -1)
        pickler.dump(world) 
Example 49
Project: MachineLearningSamples-ImageClassificationUsingCntk   Author: Azure-Samples   File: utilities_general_v2.py    MIT License 5 votes vote down vote up
def writePickle(outputFile, data):
    p = pickle.Pickler(open(outputFile,"wb"))
    p.fast = True
    p.dump(data) 
Example 50
Project: setup   Author: mindbender-studio   File: shelve.py    MIT License 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = BytesIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key.encode(self.keyencoding)] = f.getvalue() 
Example 51
Project: autokit   Author: tadejs   File: data_manager.py    MIT License 5 votes vote down vote up
def loadData (self, filename, verbose=True, replace_missing=True):
        ''' Get the data from a text file in one of 3 formats: matrix, sparse, binary_sparse'''
        if verbose:  print(("========= Reading " + filename))
        start = time.time()
        if self.use_pickle and os.path.exists (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")):
            with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "r") as pickle_file:
                vprint (verbose, "Loading pickle file : " + os.path.join(self.tmp_dir, os.path.basename(filename) + ".pickle"))
                return pickle.load(pickle_file)
        if 'format' not in list(self.info.keys()):
            self.getFormatData(filename)
        if 'feat_num' not in list(self.info.keys()):
            self.getNbrFeatures(filename)
            
        data_func = {'dense':autokit.data_io.data, 'sparse':autokit.data_io.data_sparse, 'sparse_binary':autokit.data_io.data_binary_sparse}
        
        data = data_func[self.info['format']](filename, self.info['feat_num'])
  
        # INPORTANT: when we replace missing values we double the number of variables
  
        if self.info['format']=='dense' and replace_missing and np.any(list(map(np.isnan,data))):
            vprint (verbose, "Replace missing values by 0 (slow, sorry)")
            data = autokit.data_converter.replace_missing(data)
        if self.use_pickle:
            with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "wb") as pickle_file:
                vprint (verbose, "Saving pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
                p = pickle.Pickler(pickle_file) 
                p.fast = True 
                p.dump(data)
        end = time.time()
        if verbose:  print(( "[+] Success in %5.2f sec" % (end - start)))
        return data 
Example 52
Project: autokit   Author: tadejs   File: data_manager.py    MIT License 5 votes vote down vote up
def loadLabel (self, filename, verbose=True):
        ''' Get the solution/truth values'''
        if verbose:  print(("========= Reading " + filename))
        start = time.time()
        if self.use_pickle and os.path.exists (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")):
            with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "r") as pickle_file:
                vprint (verbose, "Loading pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
                return pickle.load(pickle_file)
        if 'task' not in list(self.info.keys()):
            self.getTypeProblem(filename)
    
           # IG: Here change to accommodate the new multiclass label format
        if self.info['task'] == 'multilabel.classification':
            label = autokit.data_io.data(filename)
        elif self.info['task'] == 'multiclass.classification':
            label = autokit.data_converter.convert_to_num(autokit.data_io.data(filename))              
        else:
            label = np.ravel(autokit.data_io.data(filename)) # get a column vector
            #label = np.array([np.ravel(data_io.data(filename))]).transpose() # get a column vector
   
        if self.use_pickle:
            with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "wb") as pickle_file:
                vprint (verbose, "Saving pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
                p = pickle.Pickler(pickle_file) 
                p.fast = True 
                p.dump(label)
        end = time.time()
        if verbose:  print(( "[+] Success in %5.2f sec" % (end - start)))
        return label 
Example 53
Project: automl-challenge-2018   Author: jungtaekkim   File: data_manager.py    MIT License 5 votes vote down vote up
def loadData (self, filename, verbose=True, replace_missing=True):
		''' Get the data from a text file in one of 3 formats: matrix, sparse, sparse_binary'''
		if verbose:  print("========= Reading " + filename)
		start = time.time()
		if self.use_pickle and os.path.exists (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")):
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "r") as pickle_file:
				vprint (verbose, "Loading pickle file : " + os.path.join(self.tmp_dir, os.path.basename(filename) + ".pickle"))
				return pickle.load(pickle_file)
		if 'format' not in self.info.keys():
			self.getFormatData(filename)
		if 'feat_num' not in self.info.keys():
			self.getNbrFeatures(filename)
			
		data_func = {'dense':data_io.data, 'sparse':data_io.data_sparse, 'sparse_binary':data_io.data_binary_sparse}
		
		data = data_func[self.info['format']](filename, self.info['feat_num'])
  
		# INPORTANT: when we replace missing values we double the number of variables
  
		if self.info['format']=='dense' and replace_missing and np.any(map(np.isnan,data)):
			vprint (verbose, "Replace missing values by 0 (slow, sorry)")
			data = data_converter.replace_missing(data)
		if self.use_pickle:
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "wb") as pickle_file:
				vprint (verbose, "Saving pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
				p = pickle.Pickler(pickle_file) 
				p.fast = True 
				p.dump(data)
		end = time.time()
		if verbose:  print( "[+] Success in %5.2f sec" % (end - start))
		return data 
Example 54
Project: automl-challenge-2018   Author: jungtaekkim   File: data_manager.py    MIT License 5 votes vote down vote up
def loadLabel (self, filename, verbose=True):
		''' Get the solution/truth values'''
		if verbose:  print("========= Reading " + filename)
		start = time.time()
		if self.use_pickle and os.path.exists (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")):
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "r") as pickle_file:
				vprint (verbose, "Loading pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
				return pickle.load(pickle_file)
		if 'task' not in self.info.keys():
			self.getTypeProblem(filename)
	
           # IG: Here change to accommodate the new multiclass label format
		if self.info['task'] == 'multilabel.classification':
			label = data_io.data(filename)
		elif self.info['task'] == 'multiclass.classification':
			label = data_converter.convert_to_num(data_io.data(filename))
		else:
			label = np.ravel(data_io.data(filename)) # get a column vector
			#label = np.array([np.ravel(data_io.data(filename))]).transpose() # get a column vector
   
		if self.use_pickle:
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "wb") as pickle_file:
				vprint (verbose, "Saving pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
				p = pickle.Pickler(pickle_file) 
				p.fast = True 
				p.dump(label)
		end = time.time()
		if verbose:  print( "[+] Success in %5.2f sec" % (end - start))
		return label 
Example 55
Project: cloud-regionsrv-client   Author: SUSE-Enceladus   File: registerutils.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def store_smt_data(smt_data_file_path, smt):
    """Store the given SMT server information to the given file"""
    smt_data = open(smt_data_file_path, 'wb')
    os.fchmod(smt_data.fileno(), stat.S_IREAD | stat.S_IWRITE)
    p = pickle.Pickler(smt_data)
    p.dump(smt)
    smt_data.close()


# ---------------------------------------------------------------------------- 
Example 56
Project: pytocl   Author: moltob   File: analysis.py    MIT License 5 votes vote down vote up
def __init__(self):
        dirname = 'drivelogs'
        timestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
        fname = 'drivelog-{}.pickle'.format(timestr)
        fpath = os.path.abspath(os.path.join(dirname, fname))
        _logger.info('Logging data snapshots to {}.'.format(fpath))

        os.makedirs(dirname, exist_ok=True)
        self.file = open(fpath, 'wb')
        self.pickler = pickle.Pickler(self.file)
        self.numlogged = 0 
Example 57
Project: godot-zeronet-plugin   Author: zam-org   File: shelve.py    GNU General Public License v2.0 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 
Example 58
Project: godot-zeronet-plugin   Author: zam-org   File: shelve.py    GNU General Public License v2.0 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 
Example 59
Project: godot-zeronet-plugin   Author: zam-org   File: shelve.py    GNU General Public License v2.0 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 
Example 60
Project: godot-zeronet-plugin   Author: zam-org   File: shelve.py    GNU General Public License v2.0 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 
Example 61
Project: godot-zeronet-plugin   Author: zam-org   File: shelve.py    GNU General Public License v2.0 5 votes vote down vote up
def __setitem__(self, key, value):
        if self.writeback:
            self.cache[key] = value
        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)
        self.dict[key] = f.getvalue() 
Example 62
Project: roof-Classification   Author: TempestVanSchaik   File: utilities_general_v2.py    MIT License 5 votes vote down vote up
def writePickle(outputFile, data):
    p = pickle.Pickler(open(outputFile,"wb"))
    p.fast = True
    p.dump(data) 
Example 63
Project: RPGOne   Author: RTHMaK   File: hashing.py    Apache License 2.0 5 votes vote down vote up
def __init__(self, hash_name='md5'):
        self.stream = io.BytesIO()
        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        protocol = (pickle.DEFAULT_PROTOCOL if PY3_OR_LATER
                    else pickle.HIGHEST_PROTOCOL)
        Pickler.__init__(self, self.stream, protocol=protocol)
        # Initialise the hash obj
        self._hash = hashlib.new(hash_name) 
Example 64
Project: RPGOne   Author: RTHMaK   File: hashing.py    Apache License 2.0 5 votes vote down vote up
def memoize(self, obj):
        # We want hashing to be sensitive to value instead of reference.
        # For example we want ['aa', 'aa'] and ['aa', 'aaZ'[:2]]
        # to hash to the same value and that's why we disable memoization
        # for strings
        if isinstance(obj, _bytes_or_unicode):
            return
        Pickler.memoize(self, obj)

    # The dispatch table of the pickler is not accessible in Python
    # 3, as these lines are only bugware for IPython, we skip them. 
Example 65
Project: RPGOne   Author: RTHMaK   File: hashing.py    Apache License 2.0 5 votes vote down vote up
def _batch_setitems(self, items):
        # forces order of keys in dict to ensure consistent hash.
        try:
            # Trying first to compare dict assuming the type of keys is
            # consistent and orderable.
            # This fails on python 3 when keys are unorderable
            # but we keep it in a try as it's faster.
            Pickler._batch_setitems(self, iter(sorted(items)))
        except TypeError:
            # If keys are unorderable, sorting them using their hash. This is
            # slower but works in any case.
            Pickler._batch_setitems(self, iter(sorted((hash(k), v)
                                                      for k, v in items))) 
Example 66
Project: AutoML4   Author: djajetic   File: data_manager.py    MIT License 5 votes vote down vote up
def loadData (self, filename, verbose=True, replace_missing=True):
		''' Get the data from a text file in one of 3 formats: matrix, sparse, binary_sparse'''
		if verbose:  print("========= Reading " + filename)
		start = time.time()
		if self.use_pickle and os.path.exists (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")):
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "r") as pickle_file:
				vprint (verbose, "Loading pickle file : " + os.path.join(self.tmp_dir, os.path.basename(filename) + ".pickle"))
				return pickle.load(pickle_file)
		if 'format' not in self.info.keys():
			self.getFormatData(filename)
		if 'feat_num' not in self.info.keys():
			self.getNbrFeatures(filename)
			
		data_func = {'dense':data_io.data, 'sparse':data_io.data_sparse, 'sparse_binary':data_io.data_binary_sparse}
		
		data = data_func[self.info['format']](filename, self.info['feat_num'])
  
		# INPORTANT: when we replace missing values we double the number of variables
  
		if self.info['format']=='dense' and replace_missing and np.any(map(np.isnan,data)):
			vprint (verbose, "Replace missing values by 0 (slow, sorry)")
			data = data_converter.replace_missing(data)
		if self.use_pickle:
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "wb") as pickle_file:
				vprint (verbose, "Saving pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
				p = pickle.Pickler(pickle_file) 
				p.fast = True 
				p.dump(data)
		end = time.time()
		if verbose:  print( "[+] Success in %5.2f sec" % (end - start))
		return data 
Example 67
Project: AutoML4   Author: djajetic   File: data_manager.py    MIT License 5 votes vote down vote up
def loadLabel (self, filename, verbose=True):
		''' Get the solution/truth values'''
		if verbose:  print("========= Reading " + filename)
		start = time.time()
		if self.use_pickle and os.path.exists (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle")):
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "r") as pickle_file:
				vprint (verbose, "Loading pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
				return pickle.load(pickle_file)
		if 'task' not in self.info.keys():
			self.getTypeProblem(filename)
	
           # IG: Here change to accommodate the new multiclass label format
		if self.info['task'] == 'multilabel.classification':
			label = data_io.data(filename)
		elif self.info['task'] == 'multiclass.classification':
			label = data_converter.convert_to_num(data_io.data(filename))              
		else:
			label = np.ravel(data_io.data(filename)) # get a column vector
			#label = np.array([np.ravel(data_io.data(filename))]).transpose() # get a column vector
   
		if self.use_pickle:
			with open (os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"), "wb") as pickle_file:
				vprint (verbose, "Saving pickle file : " + os.path.join (self.tmp_dir, os.path.basename(filename) + ".pickle"))
				p = pickle.Pickler(pickle_file) 
				p.fast = True 
				p.dump(label)
		end = time.time()
		if verbose:  print( "[+] Success in %5.2f sec" % (end - start))
		return label 
Example 68
Project: attention-lvcsr   Author: rizar   File: pkl_utils.py    MIT License 5 votes vote down vote up
def __init__(self, file, protocol=0, extra_tag_to_remove=None):
        # Can't use super as Pickler isn't a new style class
        Pickler.__init__(self, file, protocol)
        self.tag_to_remove = ['trace', 'test_value']
        if extra_tag_to_remove:
            self.tag_to_remove.extend(extra_tag_to_remove) 
Example 69
Project: attention-lvcsr   Author: rizar   File: serialization.py    MIT License 5 votes vote down vote up
def dump_and_add_to_dump(object_, file_, parameters=None, to_add=None,
                         use_cpickle=False, protocol=DEFAULT_PROTOCOL,
                         **kwargs):
    r"""Calls both `dump` and `add_to_dump` to serialze several objects.

    This function is used to serialize several at the same time, using
    persistent ID. Its main advantage is that it can be used with
    `secure_dump`.

    Parameters
    ----------
    object_ : object
        The object to pickle. If None, only the parameters passed to the
        `parameters` argument will be saved.
    file_ : file
        The destination for saving.
    parameters : list, optional
        Shared variables whose internal numpy arrays should be saved
        separately in the `_parameters` field of the tar file.
    to_add : dict of objects
        A {'name': object} dictionnary of additional objects to save in
        the tar archive. Its keys will be used as name in the tar file.
    use_cpickle : bool
        Use cPickle instead of pickle. Setting it to true will disable the
        warning message if you try to pickle objects from the main module,
        so be sure that there is no warning before turning this flag
        on. Default: False.
    protocol : int, optional
        The pickling protocol to use. Unlike Python's built-in pickle, the
        default is set to `2` instead of 0 for Python 2. The Python 3
        default (level 3) is maintained.
    \*\*kwargs
        Keyword arguments to be passed to `pickle.Pickler`.

    """
    dump(object_, file_, parameters=parameters, use_cpickle=use_cpickle,
         protocol=protocol, **kwargs)
    if to_add is not None:
        for name, obj in six.iteritems(to_add):
            add_to_dump(obj, file_, name, parameters=parameters,
                        use_cpickle=use_cpickle, protocol=protocol, **kwargs) 
Example 70
Project: ironpython2   Author: IronLanguages   File: test_cPickle.py    Apache License 2.0 4 votes vote down vote up
def test_persistent_load(self):
        class MyData(object):
            def __init__(self, value):
                self.value = value

        def persistent_id(obj):
            if hasattr(obj, 'value'):
                return 'MyData: %s' % obj.value
            return None

        def persistent_load(id):
            return MyData(id[8:])


        for binary in [True, False]:
            src = StringIO()
            p = cPickle.Pickler(src)
            p.persistent_id = persistent_id
            p.binary = binary

            value = MyData('abc')
            p.dump(value)

            up = cPickle.Unpickler(StringIO(src.getvalue()))
            up.persistent_load = persistent_load
            res = up.load()

            self.assertEqual(res.value, value.value)

            # errors
            src = StringIO()
            p = cPickle.Pickler(src)
            p.persistent_id = persistent_id
            p.binary = binary

            value = MyData('abc')
            p.dump(value)

            up = cPickle.Unpickler(StringIO(src.getvalue()))

            # exceptions vary between cPickle & Pickle
            try:
                up.load()
                self.assertUnreachable()
            except Exception, e:
                pass 
Example 71
Project: D-VAE   Author: muhanzhang   File: pkl_utils.py    MIT License 4 votes vote down vote up
def dump(obj, file_handler, protocol=DEFAULT_PROTOCOL,
         persistent_id=PersistentSharedVariableID):
    """Pickles an object to a zip file using external persistence.

    :param obj: The object to pickle.
    :type obj: object

    :param file_handler: The file handle to save the object to.
    :type file_handler: file

    :param protocol: The pickling protocol to use. Unlike Python's built-in
        pickle, the default is set to `2` instead of 0 for Python 2. The
        Python 3 default (level 3) is maintained.
    :type protocol: int, optional

    :param persistent_id: The callable that persists certain objects in the
        object hierarchy to separate files inside of the zip file. For example,
        :class:`PersistentNdarrayID` saves any :class:`numpy.ndarray` to a
        separate NPY file inside of the zip file.
    :type persistent_id: callable

    .. versionadded:: 0.8

    .. note::
        The final file is simply a zipped file containing at least one file,
        `pkl`, which contains the pickled object. It can contain any other
        number of external objects. Note that the zip files are compatible with
        NumPy's :func:`numpy.load` function.

    >>> import theano
    >>> foo_1 = theano.shared(0, name='foo')
    >>> foo_2 = theano.shared(1, name='foo')
    >>> with open('model.zip', 'wb') as f:
    ...     dump((foo_1, foo_2, numpy.array(2)), f)
    >>> numpy.load('model.zip').keys()
    ['foo', 'foo_2', 'array_0', 'pkl']
    >>> numpy.load('model.zip')['foo']
    array(0)
    >>> with open('model.zip', 'rb') as f:
    ...     foo_1, foo_2, array = load(f)
    >>> array
    array(2)

    """
    with closing(zipfile.ZipFile(file_handler, 'w', zipfile.ZIP_DEFLATED,
                                 allowZip64=True)) as zip_file:
        def func(f):
            p = pickle.Pickler(f, protocol=protocol)
            p.persistent_id = persistent_id(zip_file)
            p.dump(obj)
        zipadd(func, zip_file, 'pkl') 
Example 72
Project: estimators   Author: fridiculous   File: hashing.py    MIT License 4 votes vote down vote up
def save(self, obj):
        """ Subclass the save method, to hash ndarray subclass, rather
            than pickling them. Off course, this is a total abuse of
            the Pickler class.
        """
        if isinstance(obj, self.np.ndarray) and not obj.dtype.hasobject:
            # Compute a hash of the object
            # The update function of the hash requires a c_contiguous buffer.
            if obj.shape == ():
                # 0d arrays need to be flattened because viewing them as bytes
                # raises a ValueError exception.
                obj_c_contiguous = obj.flatten()
            elif obj.flags.c_contiguous:
                obj_c_contiguous = obj
            elif obj.flags.f_contiguous:
                obj_c_contiguous = obj.T
            else:
                # Cater for non-single-segment arrays: this creates a
                # copy, and thus aleviates this issue.
                # XXX: There might be a more efficient way of doing this
                obj_c_contiguous = obj.flatten()

            # memoryview is not supported for some dtypes, e.g. datetime64, see
            # https://github.com/numpy/numpy/issues/4983. The
            # workaround is to view the array as bytes before
            # taking the memoryview.
            self._hash.update(
                self._getbuffer(obj_c_contiguous.view(self.np.uint8)))

            # We store the class, to be able to distinguish between
            # Objects with the same binary content, but different
            # classes.
            if self.coerce_mmap and isinstance(obj, self.np.memmap):
                # We don't make the difference between memmap and
                # normal ndarrays, to be able to reload previously
                # computed results with memmap.
                klass = self.np.ndarray
            else:
                klass = obj.__class__
            # We also return the dtype and the shape, to distinguish
            # different views on the same data with different dtypes.

            # The object will be pickled by the pickler hashed at the end.
            obj = (klass, ('HASHED', obj.dtype, obj.shape, obj.strides))
        elif isinstance(obj, self.np.dtype):
            # Atomic dtype objects are interned by their default constructor:
            # np.dtype('f8') is np.dtype('f8')
            # This interning is not maintained by a
            # pickle.loads + pickle.dumps cycle, because __reduce__
            # uses copy=True in the dtype constructor. This
            # non-deterministic behavior causes the internal memoizer
            # of the hasher to generate different hash values
            # depending on the history of the dtype object.
            # To prevent the hash from being sensitive to this, we use
            # .descr which is a full (and never interned) description of
            # the array dtype according to the numpy doc.
            klass = obj.__class__
            obj = (klass, ('HASHED', obj.descr))
        Hasher.save(self, obj) 
Example 73
Project: deepjets   Author: deepjets   File: tasksystem.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def start(self):
        assert self.pid is None
        assert self.exit_status is None
        def pipeOpen():
            readend, writeend = os.pipe()
            readend = os.fdopen(readend, "r")
            writeend = os.fdopen(writeend, "w")
            return readend, writeend
        self.pipe_c2p = pipeOpen()
        self.pipe_p2c = pipeOpen()
        self.parent_pid = os.getpid()
        pid = os.fork()
        if pid == 0: # child
            try:
                sys.stdin.close()    # Force no tty stdin.
                self.pipe_c2p[0].close()
                self.pipe_p2c[1].close()
                py_mod_file = os.path.splitext(__file__)[0] + ".py"
                assert os.path.exists(py_mod_file)
                binary = "python"
                full_binary_paths = [os.path.join(path, binary)
                                                         for path in os.environ["PATH"].split(os.pathsep)]
                full_binary_paths = filter(lambda path: os.access(path, os.X_OK), full_binary_paths)
                assert full_binary_paths, "%r not found in PATH %r" % (binary, os.environ["PATH"])
                args = [full_binary_paths[0],
                                py_mod_file,
                                "--forkExecProc",
                                str(self.pipe_c2p[1].fileno()),
                                str(self.pipe_p2c[0].fileno())]
                if self.env_update:
                    os.environ.update(self.env_update)
                os.execv(args[0], args)    # Does not return if successful.
            except BaseException:
                print("ExecingProcess: Error at initialization.")
                sys.excepthook(*sys.exc_info())
                sys.exit(1)
            finally:
                sys.exit()
        else: # parent
            self.pipe_c2p[1].close()
            self.pipe_p2c[0].close()
            self.pid = pid
            self.pickler = Pickler(self.pipe_p2c[1])
            self.pickler.dump(self.name)
            self.pickler.dump(self.target)
            self.pickler.dump(self.args)
            self.pipe_p2c[1].flush() 
Example 74
Project: mlens   Author: flennerhag   File: hashing.py    MIT License 4 votes vote down vote up
def save(self, obj):
        """ Subclass the save method, to hash ndarray subclass, rather
            than pickling them. Off course, this is a total abuse of
            the Pickler class.
        """
        if isinstance(obj, self.np.ndarray) and not obj.dtype.hasobject:
            # Compute a hash of the object
            # The update function of the hash requires a c_contiguous buffer.
            if obj.shape == ():
                # 0d arrays need to be flattened because viewing them as bytes
                # raises a ValueError exception.
                obj_c_contiguous = obj.flatten()
            elif obj.flags.c_contiguous:
                obj_c_contiguous = obj
            elif obj.flags.f_contiguous:
                obj_c_contiguous = obj.T
            else:
                # Cater for non-single-segment arrays: this creates a
                # copy, and thus aleviates this issue.
                # XXX: There might be a more efficient way of doing this
                obj_c_contiguous = obj.flatten()

            # memoryview is not supported for some dtypes, e.g. datetime64, see
            # https://github.com/numpy/numpy/issues/4983. The
            # workaround is to view the array as bytes before
            # taking the memoryview.
            self._hash.update(
                self._getbuffer(obj_c_contiguous.view(self.np.uint8)))

            # We store the class, to be able to distinguish between
            # Objects with the same binary content, but different
            # classes.
            if self.coerce_mmap and isinstance(obj, self.np.memmap):
                # We don't make the difference between memmap and
                # normal ndarrays, to be able to reload previously
                # computed results with memmap.
                klass = self.np.ndarray
            else:
                klass = obj.__class__
            # We also return the dtype and the shape, to distinguish
            # different views on the same data with different dtypes.

            # The object will be pickled by the pickler hashed at the end.
            obj = (klass, ('HASHED', obj.dtype, obj.shape, obj.strides))
        elif isinstance(obj, self.np.dtype):
            # Atomic dtype objects are interned by their default constructor:
            # np.dtype('f8') is np.dtype('f8')
            # This interning is not maintained by a
            # pickle.loads + pickle.dumps cycle, because __reduce__
            # uses copy=True in the dtype constructor. This
            # non-deterministic behavior causes the internal memoizer
            # of the hasher to generate different hash values
            # depending on the history of the dtype object.
            # To prevent the hash from being sensitive to this, we use
            # .descr which is a full (and never interned) description of
            # the array dtype according to the numpy doc.
            klass = obj.__class__
            obj = (klass, ('HASHED', obj.descr))
        Hasher.save(self, obj) 
Example 75
Project: Computable   Author: ktraunmueller   File: session.py    MIT License 4 votes vote down vote up
def save(variables=None, file=SAVEFILE, dictionary=None, verbose=False):

    """saves variables from a numpy session to a file.  Variables
    which won't pickle are "proxied" if possible.

    'variables'       a string of comma seperated variables: e.g. "a,b,c"
                      Defaults to dictionary.keys().

    'file'            a filename or file object for the session file.

    'dictionary'      the dictionary in which to look up the variables.
                      Defaults to the caller's globals()

    'verbose'         print additional debug output when True.
    """

    global VERBOSE
    VERBOSE = verbose

    _update_proxy_types()

    if isinstance(file, str):
        file = open(file, "wb")

    if dictionary is None:
        dictionary = _callers_globals()

    if variables is None:
        keys = list(dictionary.keys())
    else:
        keys = variables.split(",")

    source_modules = _callers_modules() + list(sys.modules.keys())

    p = pickle.Pickler(file, protocol=2)

    _verbose("variables:", keys)
    for k in keys:
        v = dictionary[k]
        _verbose("saving", k, type(v))
        try:  # Try to write an ordinary pickle
            p.dump(v)
            _verbose("pickled", k)
        except (pickle.PicklingError, TypeError, SystemError):
            # Use proxies for stuff that won't pickle
            if isinstance(v, type(sys)): # module
                proxy = _ModuleProxy(v.__name__, save=True)
            else:
                try:
                    module, name = _locate(source_modules, v)
                except ObjectNotFound:
                    _errout("warning: couldn't find object", k,
                            "in any module... skipping.")
                    continue
                else:
                    proxy = _ObjectProxy(module, name, type(v), save=True)
            p.dump(proxy)
    o = _SaveSession(keys, save=True)
    p.dump(o)
    file.close() 
Example 76
Project: poker   Author: surgebiswas   File: pkl_utils.py    MIT License 4 votes vote down vote up
def dump(obj, file_handler, protocol=DEFAULT_PROTOCOL,
         persistent_id=PersistentSharedVariableID):
    """Pickles an object to a zip file using external persistence.

    :param obj: The object to pickle.
    :type obj: object

    :param file_handler: The file handle to save the object to.
    :type file_handler: file

    :param protocol: The pickling protocol to use. Unlike Python's built-in
        pickle, the default is set to `2` instead of 0 for Python 2. The
        Python 3 default (level 3) is maintained.
    :type protocol: int, optional

    :param persistent_id: The callable that persists certain objects in the
        object hierarchy to separate files inside of the zip file. For example,
        :class:`PersistentNdarrayID` saves any :class:`numpy.ndarray` to a
        separate NPY file inside of the zip file.
    :type persistent_id: callable

    .. versionadded:: 0.8

    .. note::
        The final file is simply a zipped file containing at least one file,
        `pkl`, which contains the pickled object. It can contain any other
        number of external objects. Note that the zip files are compatible with
        NumPy's :func:`numpy.load` function.

    >>> import theano
    >>> foo_1 = theano.shared(0, name='foo')
    >>> foo_2 = theano.shared(1, name='foo')
    >>> with open('model.zip', 'wb') as f:
    ...     dump((foo_1, foo_2, numpy.array(2)), f)
    >>> numpy.load('model.zip').keys()
    ['foo', 'foo_2', 'array_0', 'pkl']
    >>> numpy.load('model.zip')['foo']
    array(0)
    >>> with open('model.zip', 'rb') as f:
    ...     foo_1, foo_2, array = load(f)
    >>> array
    array(2)

    """
    with closing(zipfile.ZipFile(file_handler, 'w', zipfile.ZIP_DEFLATED,
                                 allowZip64=True)) as zip_file:
        def func(f):
            p = pickle.Pickler(f, protocol=protocol)
            p.persistent_id = persistent_id(zip_file)
            p.dump(obj)
        zipadd(func, zip_file, 'pkl') 
Example 77
Project: RPGOne   Author: RTHMaK   File: hashing.py    Apache License 2.0 4 votes vote down vote up
def save(self, obj):
        """ Subclass the save method, to hash ndarray subclass, rather
            than pickling them. Off course, this is a total abuse of
            the Pickler class.
        """
        if isinstance(obj, self.np.ndarray) and not obj.dtype.hasobject:
            # Compute a hash of the object
            # The update function of the hash requires a c_contiguous buffer.
            if obj.shape == ():
                # 0d arrays need to be flattened because viewing them as bytes
                # raises a ValueError exception.
                obj_c_contiguous = obj.flatten()
            elif obj.flags.c_contiguous:
                obj_c_contiguous = obj
            elif obj.flags.f_contiguous:
                obj_c_contiguous = obj.T
            else:
                # Cater for non-single-segment arrays: this creates a
                # copy, and thus aleviates this issue.
                # XXX: There might be a more efficient way of doing this
                obj_c_contiguous = obj.flatten()

            # memoryview is not supported for some dtypes, e.g. datetime64, see
            # https://github.com/numpy/numpy/issues/4983. The
            # workaround is to view the array as bytes before
            # taking the memoryview.
            self._hash.update(
                self._getbuffer(obj_c_contiguous.view(self.np.uint8)))

            # We store the class, to be able to distinguish between
            # Objects with the same binary content, but different
            # classes.
            if self.coerce_mmap and isinstance(obj, self.np.memmap):
                # We don't make the difference between memmap and
                # normal ndarrays, to be able to reload previously
                # computed results with memmap.
                klass = self.np.ndarray
            else:
                klass = obj.__class__
            # We also return the dtype and the shape, to distinguish
            # different views on the same data with different dtypes.

            # The object will be pickled by the pickler hashed at the end.
            obj = (klass, ('HASHED', obj.dtype, obj.shape, obj.strides))
        elif isinstance(obj, self.np.dtype):
            # Atomic dtype objects are interned by their default constructor:
            # np.dtype('f8') is np.dtype('f8')
            # This interning is not maintained by a
            # pickle.loads + pickle.dumps cycle, because __reduce__
            # uses copy=True in the dtype constructor. This
            # non-deterministic behavior causes the internal memoizer
            # of the hasher to generate different hash values
            # depending on the history of the dtype object.
            # To prevent the hash from being sensitive to this, we use
            # .descr which is a full (and never interned) description of
            # the array dtype according to the numpy doc.
            klass = obj.__class__
            obj = (klass, ('HASHED', obj.descr))
        Hasher.save(self, obj) 
Example 78
Project: eve-metrics   Author: ccpgames   File: memcache.py    MIT License 4 votes vote down vote up
def __init__(self, servers, debug=0, pickleProtocol=0,
                 pickler=pickle.Pickler, unpickler=pickle.Unpickler,
                 pload=None, pid=None,
                 server_max_key_length=SERVER_MAX_KEY_LENGTH,
                 server_max_value_length=SERVER_MAX_VALUE_LENGTH,
                 dead_retry=_DEAD_RETRY, socket_timeout=_SOCKET_TIMEOUT,
                 cache_cas = False):
        """
        Create a new Client object with the given list of servers.

        @param servers: C{servers} is passed to L{set_servers}.
        @param debug: whether to display error messages when a server can't be
        contacted.
        @param pickleProtocol: number to mandate protocol used by (c)Pickle.
        @param pickler: optional override of default Pickler to allow subclassing.
        @param unpickler: optional override of default Unpickler to allow subclassing.
        @param pload: optional persistent_load function to call on pickle loading.
        Useful for cPickle since subclassing isn't allowed.
        @param pid: optional persistent_id function to call on pickle storing.
        Useful for cPickle since subclassing isn't allowed.
        @param dead_retry: number of seconds before retrying a blacklisted
        server. Default to 30 s.
        @param socket_timeout: timeout in seconds for all calls to a server. Defaults
        to 3 seconds.
        @param cache_cas: (default False) If true, cas operations will be
        cached.  WARNING: This cache is not expired internally, if you have
        a long-running process you will need to expire it manually via
        "client.reset_cas(), or the cache can grow unlimited.
        @param server_max_key_length: (default SERVER_MAX_KEY_LENGTH)
        Data that is larger than this will not be sent to the server.
        @param server_max_value_length: (default SERVER_MAX_VALUE_LENGTH)
        Data that is larger than this will not be sent to the server.
        """
        local.__init__(self)
        self.debug = debug
        self.dead_retry = dead_retry
        self.socket_timeout = socket_timeout
        self.set_servers(servers)
        self.stats = {}
        self.cache_cas = cache_cas
        self.reset_cas()

        # Allow users to modify pickling/unpickling behavior
        self.pickleProtocol = pickleProtocol
        self.pickler = pickler
        self.unpickler = unpickler
        self.persistent_load = pload
        self.persistent_id = pid
        self.server_max_key_length = server_max_key_length
        self.server_max_value_length = server_max_value_length

        #  figure out the pickler style
        file = StringIO()
        try:
            pickler = self.pickler(file, protocol = self.pickleProtocol)
            self.picklerIsKeyword = True
        except TypeError:
            self.picklerIsKeyword = False 
Example 79
Project: attention-lvcsr   Author: rizar   File: pkl_utils.py    MIT License 4 votes vote down vote up
def dump(obj, file_handler, protocol=DEFAULT_PROTOCOL,
         persistent_id=PersistentSharedVariableID):
    """Pickles an object to a zip file using external persistence.

    :param obj: The object to pickle.
    :type obj: object

    :param file_handler: The file handle to save the object to.
    :type file_handler: file

    :param protocol: The pickling protocol to use. Unlike Python's built-in
        pickle, the default is set to `2` instead of 0 for Python 2. The
        Python 3 default (level 3) is maintained.
    :type protocol: int, optional

    :param persistent_id: The callable that persists certain objects in the
        object hierarchy to separate files inside of the zip file. For example,
        :class:`PersistentNdarrayID` saves any :class:`numpy.ndarray` to a
        separate NPY file inside of the zip file.
    :type persistent_id: callable

    .. versionadded:: 0.8

    .. note::
        The final file is simply a zipped file containing at least one file,
        `pkl`, which contains the pickled object. It can contain any other
        number of external objects. Note that the zip files are compatible with
        NumPy's :func:`numpy.load` function.

    >>> import theano
    >>> foo_1 = theano.shared(0, name='foo')
    >>> foo_2 = theano.shared(1, name='foo')
    >>> with open('model.zip', 'wb') as f:
    ...     dump((foo_1, foo_2, numpy.array(2)), f)
    >>> numpy.load('model.zip').keys()
    ['foo', 'foo_2', 'array_0', 'pkl']
    >>> numpy.load('model.zip')['foo']
    array(0)
    >>> with open('model.zip', 'rb') as f:
    ...     foo_1, foo_2, array = load(f)
    >>> array
    array(2)

    """
    with closing(zipfile.ZipFile(file_handler, 'w', zipfile.ZIP_DEFLATED,
                                 allowZip64=True)) as zip_file:
        def func(f):
            p = pickle.Pickler(f, protocol=protocol)
            p.persistent_id = persistent_id(zip_file)
            p.dump(obj)
        zipadd(func, zip_file, 'pkl') 
Example 80
Project: attention-lvcsr   Author: rizar   File: serialization.py    MIT License 4 votes vote down vote up
def dump(object_, file_, parameters=None, use_cpickle=False,
         protocol=DEFAULT_PROTOCOL, **kwargs):
    r"""Pickles an object, optionally saving its parameters separately.

    Parameters
    ----------
    object_ : object
        The object to pickle. If None, only the parameters passed to the
        `parameters` argument will be saved.
    file_ : file
        The destination for saving.
    parameters : list, optional
        Shared variables whose internal numpy arrays should be saved
        separately in the `_parameters` field of the tar file.
    pickle_object : bool
        If False, `object_` will not be serialized, only its parameters.
        This flag can be used when `object_` is not serializable, but one
        still want to save its parameters. Default: True
    use_cpickle : bool
        Use cPickle instead of pickle. Setting it to true will disable the
        warning message if you try to pickle objects from the main module,
        so be sure that there is no warning before turning this flag
        on. Default: False.
    protocol : int, optional
        The pickling protocol to use. Unlike Python's built-in pickle, the
        default is set to `2` instead of 0 for Python 2. The Python 3
        default (level 3) is maintained.
    \*\*kwargs
        Keyword arguments to be passed to `pickle.Pickler`.

    """
    if use_cpickle:
        pickler = cPickle.Pickler
    else:
        pickler = _PicklerWithWarning
    with closing(tarfile.TarFile(fileobj=file_, mode='w')) as tar_file:
        external_objects = {}

        def _save_parameters(f):
            renamer = _Renamer()
            named_parameters = {renamer(p): p for p in parameters}
            numpy.savez(f, **{n: p.get_value()
                              for n, p in named_parameters.items()})
            for n, p in named_parameters.items():
                array_ = p.container.storage[0]
                external_objects[id(array_)] = _mangle_parameter_name(
                    type(array_), n)
        if parameters:
            _taradd(_save_parameters, tar_file, '_parameters')
        if object_ is not None:
            save_object = _SaveObject(pickler, object_, external_objects,
                                      protocol, **kwargs)
            _taradd(save_object, tar_file, '_pkl')