Python pickle.DEFAULT_PROTOCOL Examples

The following are 30 code examples for showing how to use pickle.DEFAULT_PROTOCOL(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module pickle , or try the search function .

Example 1
Project: pywren-ibm-cloud   Author: pywren   File: cloudpickle.py    License: Apache License 2.0 6 votes vote down vote up
def dumps(obj, protocol=None):
    """Serialize obj as a string of bytes allocated in memory

    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.

    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    file = StringIO()
    try:
        cp = CloudPickler(file, protocol=protocol)
        cp.dump(obj)
        return file.getvalue()
    finally:
        file.close()


# including pickles unloading functions in this namespace 
Example 2
Project: mlens   Author: flennerhag   File: numpy_pickle.py    License: MIT License 6 votes vote down vote up
def __init__(self, fp, protocol=None):
        self.file_handle = fp
        self.buffered = isinstance(self.file_handle, BinaryZlibFile)

        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        if protocol is None:
            protocol = (pickle.DEFAULT_PROTOCOL if PY3_OR_LATER
                        else pickle.HIGHEST_PROTOCOL)

        Pickler.__init__(self, self.file_handle, protocol=protocol)
        # delayed import of numpy, to avoid tight coupling
        try:
            import numpy as np
        except ImportError:
            np = None
        self.np = np 
Example 3
Project: FATE   Author: FederatedAI   File: cloudpickle.py    License: Apache License 2.0 6 votes vote down vote up
def dumps(obj, protocol=None):
    """Serialize obj as a string of bytes allocated in memory
    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.
    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    file = StringIO()
    try:
        cp = CloudPickler(file, protocol=protocol)
        cp.dump(obj)
        return file.getvalue()
    finally:
        file.close()


# including pickles unloading functions in this namespace 
Example 4
Project: BentoML   Author: bentoml   File: cloudpickle.py    License: Apache License 2.0 6 votes vote down vote up
def dumps(obj, protocol=None):
    """Serialize obj as a string of bytes allocated in memory

    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.

    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    file = StringIO()
    try:
        cp = CloudPickler(file, protocol=protocol)
        cp.dump(obj)
        return file.getvalue()
    finally:
        file.close()


# including pickles unloading functions in this namespace 
Example 5
Project: ray   Author: ray-project   File: cloudpickle.py    License: Apache License 2.0 6 votes vote down vote up
def dumps(obj, protocol=None):
    """Serialize obj as a string of bytes allocated in memory

    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.

    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    file = BytesIO()
    try:
        cp = CloudPickler(file, protocol=protocol)
        cp.dump(obj)
        return file.getvalue()
    finally:
        file.close()


# including pickles unloading functions in this namespace 
Example 6
Project: ray   Author: ray-project   File: cloudpickle_fast.py    License: Apache License 2.0 6 votes vote down vote up
def dumps(obj, protocol=None, buffer_callback=None):
    """Serialize obj as a string of bytes allocated in memory

    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.

    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    with io.BytesIO() as file:
        cp = CloudPickler(file, protocol=protocol, buffer_callback=buffer_callback)
        cp.dump(obj)
        return file.getvalue()


# COLLECTION OF OBJECTS __getnewargs__-LIKE METHODS
# ------------------------------------------------- 
Example 7
Project: bugbug   Author: mozilla   File: repository.py    License: Mozilla Public License 2.0 6 votes vote down vote up
def __init__(self, save):
        self.save = save

        try:
            self.db_experiences = shelve.Shelf(
                LMDBDict("data/commit_experiences.lmdb", readonly=not save),
                protocol=pickle.DEFAULT_PROTOCOL,
                writeback=save,
            )
        except lmdb.Error as e:
            if not save and "No such file or directory" in str(e):
                self.db_experiences = {}
            else:
                raise

        if not save:
            self.mem_experiences = {} 
Example 8
Project: Splunking-Crime   Author: nccgroup   File: numpy_pickle.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def __init__(self, fp, protocol=None):
        self.file_handle = fp
        self.buffered = isinstance(self.file_handle, BinaryZlibFile)

        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        if protocol is None:
            protocol = (pickle.DEFAULT_PROTOCOL if PY3_OR_LATER
                        else pickle.HIGHEST_PROTOCOL)

        Pickler.__init__(self, self.file_handle, protocol=protocol)
        # delayed import of numpy, to avoid tight coupling
        try:
            import numpy as np
        except ImportError:
            np = None
        self.np = np 
Example 9
Project: abu   Author: bbfamily   File: numpy_pickle.py    License: GNU General Public License v3.0 6 votes vote down vote up
def __init__(self, fp, protocol=None):
        self.file_handle = fp
        self.buffered = isinstance(self.file_handle, BinaryZlibFile)

        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        if protocol is None:
            protocol = (pickle.DEFAULT_PROTOCOL if PY3_OR_LATER
                        else pickle.HIGHEST_PROTOCOL)

        Pickler.__init__(self, self.file_handle, protocol=protocol)
        # delayed import of numpy, to avoid tight coupling
        try:
            import numpy as np
        except ImportError:
            np = None
        self.np = np 
Example 10
Project: eggroll   Author: WeBankFinTech   File: cloudpickle.py    License: Apache License 2.0 6 votes vote down vote up
def dumps(obj, protocol=None):
  """Serialize obj as a string of bytes allocated in memory
  protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
  pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
  between processes running the same Python version.
  Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
  compatibility with older versions of Python.
  """
  file = StringIO()
  try:
    cp = CloudPickler(file, protocol=protocol)
    cp.dump(obj)
    return file.getvalue()
  finally:
    file.close()


# including pickles unloading functions in this namespace 
Example 11
Project: SqueezeMeta   Author: jtamames   File: numpy_pickle.py    License: GNU General Public License v3.0 6 votes vote down vote up
def __init__(self, filename, compress=0, cache_size=10, protocol=None):
        self._filename = filename
        self._filenames = [filename, ]
        self.cache_size = cache_size
        self.compress = compress
        if not self.compress:
            self.file = open(filename, 'wb')
        else:
            self.file = BytesIO()
        # Count the number of npy files that we have created:
        self._npy_counter = 0
        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        if protocol is None:
            protocol = (pickle.DEFAULT_PROTOCOL if PY3
                        else pickle.HIGHEST_PROTOCOL)

        Pickler.__init__(self, self.file,
                         protocol=protocol)
        # delayed import of numpy, to avoid tight coupling
        try:
            import numpy as np
        except ImportError:
            np = None
        self.np = np 
Example 12
Project: twitter-stock-recommendation   Author: alvarobartt   File: numpy_pickle.py    License: MIT License 6 votes vote down vote up
def __init__(self, fp, protocol=None):
        self.file_handle = fp
        self.buffered = isinstance(self.file_handle, BinaryZlibFile)

        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        if protocol is None:
            protocol = (pickle.DEFAULT_PROTOCOL if PY3_OR_LATER
                        else pickle.HIGHEST_PROTOCOL)

        Pickler.__init__(self, self.file_handle, protocol=protocol)
        # delayed import of numpy, to avoid tight coupling
        try:
            import numpy as np
        except ImportError:
            np = None
        self.np = np 
Example 13
Project: pywren-ibm-cloud   Author: pywren   File: cloudpickle.py    License: Apache License 2.0 5 votes vote down vote up
def __init__(self, file, protocol=None):
        if protocol is None:
            protocol = DEFAULT_PROTOCOL
        Pickler.__init__(self, file, protocol=protocol)
        # set of modules to unpickle
        self.modules = set()
        # map ids to dictionary. used to ensure that functions can share global env
        self.globals_ref = {} 
Example 14
Project: pywren-ibm-cloud   Author: pywren   File: cloudpickle.py    License: Apache License 2.0 5 votes vote down vote up
def dump(obj, file, protocol=None):
    """Serialize obj as bytes streamed into file

    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.

    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    CloudPickler(file, protocol=protocol).dump(obj) 
Example 15
Project: vnpy_crypto   Author: birforce   File: socket.py    License: MIT License 5 votes vote down vote up
def send_pyobj(self, obj, flags=0, protocol=DEFAULT_PROTOCOL, **kwargs):
        """Send a Python object as a message using pickle to serialize.

        Parameters
        ----------
        obj : Python object
            The Python object to send.
        flags : int
            Any valid flags for :func:`Socket.send`.
        protocol : int
            The pickle protocol number to use. The default is pickle.DEFAULT_PROTOCOL
            where defined, and pickle.HIGHEST_PROTOCOL elsewhere.
        """
        msg = pickle.dumps(obj, protocol)
        return self.send(msg, flags=flags, **kwargs) 
Example 16
Project: estimators   Author: fridiculous   File: hashing.py    License: MIT License 5 votes vote down vote up
def __init__(self, hash_name='md5'):
        self.stream = io.BytesIO()
        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        protocol = (pickle.DEFAULT_PROTOCOL if PY3_OR_LATER
                    else pickle.HIGHEST_PROTOCOL)
        Pickler.__init__(self, self.stream, protocol=protocol)
        # Initialise the hash obj
        self._hash = hashlib.new(hash_name) 
Example 17
Project: mlens   Author: flennerhag   File: hashing.py    License: MIT License 5 votes vote down vote up
def __init__(self, hash_name='md5'):
        self.stream = io.BytesIO()
        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        protocol = (pickle.DEFAULT_PROTOCOL if PY3_OR_LATER
                    else pickle.HIGHEST_PROTOCOL)
        Pickler.__init__(self, self.stream, protocol=protocol)
        # Initialise the hash obj
        self._hash = hashlib.new(hash_name) 
Example 18
Project: FATE   Author: FederatedAI   File: cloudpickle.py    License: Apache License 2.0 5 votes vote down vote up
def __init__(self, file, protocol=None):
        if protocol is None:
            protocol = DEFAULT_PROTOCOL
        Pickler.__init__(self, file, protocol=protocol)
        # set of modules to unpickle
        self.modules = set()
        # map ids to dictionary. used to ensure that functions can share global env
        self.globals_ref = {} 
Example 19
Project: FATE   Author: FederatedAI   File: cloudpickle.py    License: Apache License 2.0 5 votes vote down vote up
def dump(obj, file, protocol=None):
    """Serialize obj as bytes streamed into file
    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.
    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    CloudPickler(file, protocol=protocol).dump(obj) 
Example 20
Project: FATE   Author: FederatedAI   File: cloudpickle.py    License: Apache License 2.0 5 votes vote down vote up
def __init__(self, file, protocol=None):
        if protocol is None:
            protocol = DEFAULT_PROTOCOL
        Pickler.__init__(self, file, protocol=protocol)
        # set of modules to unpickle
        self.modules = set()
        # map ids to dictionary. used to ensure that functions can share global env
        self.globals_ref = {} 
Example 21
Project: FATE   Author: FederatedAI   File: cloudpickle.py    License: Apache License 2.0 5 votes vote down vote up
def dump(obj, file, protocol=None):
    """Serialize obj as bytes streamed into file
    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.
    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    CloudPickler(file, protocol=protocol).dump(obj) 
Example 22
Project: attention-lvcsr   Author: rizar   File: serialization.py    License: MIT License 5 votes vote down vote up
def dump_and_add_to_dump(object_, file_, parameters=None, to_add=None,
                         use_cpickle=False, protocol=DEFAULT_PROTOCOL,
                         **kwargs):
    r"""Calls both `dump` and `add_to_dump` to serialze several objects.

    This function is used to serialize several at the same time, using
    persistent ID. Its main advantage is that it can be used with
    `secure_dump`.

    Parameters
    ----------
    object_ : object
        The object to pickle. If None, only the parameters passed to the
        `parameters` argument will be saved.
    file_ : file
        The destination for saving.
    parameters : list, optional
        Shared variables whose internal numpy arrays should be saved
        separately in the `_parameters` field of the tar file.
    to_add : dict of objects
        A {'name': object} dictionnary of additional objects to save in
        the tar archive. Its keys will be used as name in the tar file.
    use_cpickle : bool
        Use cPickle instead of pickle. Setting it to true will disable the
        warning message if you try to pickle objects from the main module,
        so be sure that there is no warning before turning this flag
        on. Default: False.
    protocol : int, optional
        The pickling protocol to use. Unlike Python's built-in pickle, the
        default is set to `2` instead of 0 for Python 2. The Python 3
        default (level 3) is maintained.
    \*\*kwargs
        Keyword arguments to be passed to `pickle.Pickler`.

    """
    dump(object_, file_, parameters=parameters, use_cpickle=use_cpickle,
         protocol=protocol, **kwargs)
    if to_add is not None:
        for name, obj in six.iteritems(to_add):
            add_to_dump(obj, file_, name, parameters=parameters,
                        use_cpickle=use_cpickle, protocol=protocol, **kwargs) 
Example 23
Project: pySINDy   Author: luckystarufo   File: socket.py    License: MIT License 5 votes vote down vote up
def send_pyobj(self, obj, flags=0, protocol=DEFAULT_PROTOCOL, **kwargs):
        """Send a Python object as a message using pickle to serialize.

        Parameters
        ----------
        obj : Python object
            The Python object to send.
        flags : int
            Any valid flags for :func:`Socket.send`.
        protocol : int
            The pickle protocol number to use. The default is pickle.DEFAULT_PROTOCOL
            where defined, and pickle.HIGHEST_PROTOCOL elsewhere.
        """
        msg = pickle.dumps(obj, protocol)
        return self.send(msg, flags=flags, **kwargs) 
Example 24
Project: BentoML   Author: bentoml   File: cloudpickle.py    License: Apache License 2.0 5 votes vote down vote up
def __init__(self, file, protocol=None):
        if protocol is None:
            protocol = DEFAULT_PROTOCOL
        Pickler.__init__(self, file, protocol=protocol)
        # map ids to dictionary. used to ensure that functions can share global env
        self.globals_ref = {} 
Example 25
Project: BentoML   Author: bentoml   File: cloudpickle.py    License: Apache License 2.0 5 votes vote down vote up
def dump(obj, file, protocol=None):
    """Serialize obj as bytes streamed into file

    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.

    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    CloudPickler(file, protocol=protocol).dump(obj) 
Example 26
Project: django-estimators   Author: fridiculous   File: hashing.py    License: MIT License 5 votes vote down vote up
def __init__(self, hash_name='md5'):
        self.stream = io.BytesIO()
        # By default we want a pickle protocol that only changes with
        # the major python version and not the minor one
        protocol = (pickle.DEFAULT_PROTOCOL if PY3_OR_LATER
                    else pickle.HIGHEST_PROTOCOL)
        Pickler.__init__(self, self.stream, protocol=protocol)
        # Initialise the hash obj
        self._hash = hashlib.new(hash_name) 
Example 27
Project: ray   Author: ray-project   File: cloudpickle.py    License: Apache License 2.0 5 votes vote down vote up
def __init__(self, file, protocol=None):
        if protocol is None:
            protocol = DEFAULT_PROTOCOL
        Pickler.__init__(self, file, protocol=protocol)
        # map ids to dictionary. used to ensure that functions can share global env
        self.globals_ref = {} 
Example 28
Project: ray   Author: ray-project   File: cloudpickle.py    License: Apache License 2.0 5 votes vote down vote up
def dump(obj, file, protocol=None):
    """Serialize obj as bytes streamed into file

    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.

    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    CloudPickler(file, protocol=protocol).dump(obj) 
Example 29
Project: ray   Author: ray-project   File: cloudpickle_fast.py    License: Apache License 2.0 5 votes vote down vote up
def dump(obj, file, protocol=None, buffer_callback=None):
    """Serialize obj as bytes streamed into file

    protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to
    pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed
    between processes running the same Python version.

    Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure
    compatibility with older versions of Python.
    """
    CloudPickler(file, protocol=protocol, buffer_callback=buffer_callback).dump(obj) 
Example 30
Project: bugbug   Author: mozilla   File: test_scheduling.py    License: Mozilla Public License 2.0 5 votes vote down vote up
def get_past_failures(granularity, readonly):
    if granularity == "label":
        past_failures_db = os.path.join("data", PAST_FAILURES_LABEL_DB)
    elif granularity == "group":
        past_failures_db = os.path.join("data", PAST_FAILURES_GROUP_DB)
    elif granularity == "config_group":
        past_failures_db = os.path.join("data", PAST_FAILURES_CONFIG_GROUP_DB)
    else:
        raise Exception(f"{granularity} granularity unsupported")

    return shelve.Shelf(
        LMDBDict(past_failures_db[: -len(".tar.zst")], readonly=readonly),
        protocol=pickle.DEFAULT_PROTOCOL,
        writeback=not readonly,
    )