Python pickle.PicklingError() Examples

def _save_reader_to_cache(reader, cache_file_path):
    """Save a reader object to a pickle file.

    Args:
        reader: The Reader instance to be persisted.
        cache_file_path: A Path instance giving the path to the pickle file location.
    """
    cache_path = cache_file_path.parent
    os.makedirs(str(cache_path), exist_ok=True)
    try:
        with cache_file_path.open('wb') as cache_file:
            try:
                pickle.dump(reader, cache_file)
            except (AttributeError, pickle.PicklingError, TypeError) as pickling_error:
                log.warn("Could not pickle {} because {}".format(reader, pickling_error))
                pass
    except OSError as os_error:
        log.warn("Could not cache {} because {}".format(reader, os_error)) 

def tryPickleOnAllContents3(obj, ignore=None, path=None, verbose=False):
    """
    Definitely find pickle errors

    Notes
    -----
    In this form, this just finds one pickle error and then crashes. If you want
    to make it work like the other testPickle functions and handle errors, you could.
    But usually you just have to find one unpickleable SOB.
    """

    with tempfile.TemporaryFile() as output:
        try:
            MyPickler(output).dump(obj)
        except (pickle.PicklingError, TypeError):
            pass 

def lint(argv: List[str]) -> None:
    """
    Invoke Pylint with our preferred options
    """
    print('>>>> Running pylint')
    args = ['--rcfile=.pylintrc', # Load rcfile first.
            '--ignored-modules=alembic,MySQLdb,flask_sqlalchemy,distutils.dist', # override ignored-modules (codacy hack)
            '--load-plugins', 'pylint_quotes,pylint_monolith', # Plugins
            '-f', 'parseable', # Machine-readable output.
            '-j', str(configuration.get_int('pylint_threads')), # Use four cores for speed.
           ]
    args.extend(argv or find_files(file_extension='py'))
    # pylint: disable=import-outside-toplevel
    import pylint.lint
    try:
        linter = pylint.lint.Run(args, exit=False)
    except PicklingError:
        print('Error while running pylint with multiprocessing')
        configuration.write('pylint_threads', 1)
        lint(argv)
        return

    if linter.linter.msg_status:
        raise TestFailedException(linter.linter.msg_status) 

def put(self, o):
        """Encode object ``o`` and write it to the pipe.
        Block gevent-cooperatively until all data is written. The default
        encoder is ``pickle.dumps``.

        :arg o: a Python object that is encodable with the encoder of choice.

        Raises:
            - :exc:`GIPCError`
            - :exc:`GIPCClosed`
            - :exc:`pickle.PicklingError`

        """
        self._validate()
        with self._lock:
            bindata = self._encoder(o)
            self._write(struct.pack("!i", len(bindata)) + bindata) 

def __getstate__(self):
        d = dict(self.__dict__)

        def get_pickleable_dict(cacheDict):
            pickleableCache = dict()
            for k, v in cacheDict.items():
                try:
                    _pickle.dumps(v)
                    pickleableCache[k] = v
                except TypeError as e:
                    if isinstance(v, dict):
                        self.unpickleable.add(str(k[0]) + str(type(v)) + str(e) + str(list(v.keys())))
                    else:
                        self.unpickleable.add(str(k[0]) + str(type(v)) + str(e))  # + str(list(v.__dict__.keys())))
                except _pickle.PicklingError as e:
                    self.unpickleable.add(str(k[0]) + str(type(v)) + str(e))
            return pickleableCache

        d['cache'] = get_pickleable_dict(self.cache)
        d['outargs'] = get_pickleable_dict(self.outargs)
        return d 

def pickleRoundTrip(self, obj, name, dumper, loader, proto):
        save_m = sys.modules[name]
        try:
            sys.modules[name] = dumper
            temp = pickle.dumps(obj, proto)
            sys.modules[name] = loader
            result = pickle.loads(temp)
        except pickle.PicklingError as pe:
            # pyET must be second, because pyET may be (equal to) ET.
            human = dict([(ET, "cET"), (pyET, "pyET")])
            raise support.TestFailed("Failed to round-trip %r from %r to %r"
                                     % (obj,
                                        human.get(dumper, dumper),
                                        human.get(loader, loader))) from pe
        finally:
            sys.modules[name] = save_m
        return result 

def test_reduce_bad_iterator(self):
        # Issue4176: crash when 4th and 5th items of __reduce__()
        # are not iterators
        class C(object):
            def __reduce__(self):
                # 4th item is not an iterator
                return list, (), None, [], None
        class D(object):
            def __reduce__(self):
                # 5th item is not an iterator
                return dict, (), None, None, []

        # Protocol 0 in Python implementation is less strict and also accepts
        # iterables.
        for proto in protocols:
            try:
                self.dumps(C(), proto)
            except (AttributeError, pickle.PicklingError, cPickle.PicklingError):
                pass
            try:
                self.dumps(D(), proto)
            except (AttributeError, pickle.PicklingError, cPickle.PicklingError):
                pass 

def get_pickling_errors(self, obj, seen=None):
    if seen == None:
      seen = []
    try:
      state = obj.__getstate__()
    except AttributeError:
      return
    if state == None:
      return
    if isinstance(state, tuple):
      if not isinstance(state[0], dict):
        state = state[1]
      else:
        state = state[0].update(state[1])
    result = {}
    for i in state:
      try:
        pickle.dumps(state[i], protocol=2)
      except pickle.PicklingError:
        if not state[i] in seen:
          seen.append(state[i])
          result[i] = self.get_pickling_errors(state[i], seen)
    return result 

def get_pickling_errors(obj, seen=None):
    if seen == None:
        seen = []
    try:
        state = obj.__getstate__()
    except AttributeError:
        return
    if state == None:
        return
    if isinstance(state, tuple):
        if not isinstance(state[0], dict):
            state = state[1]
        else:
            state = state[0].crud_instances(state[1])
    result = {}
    for i in state:
        try:
            pickle.dumps(state[i], protocol=2)
        except pickle.PicklingError:
            if not state[i] in seen:
                seen.append(state[i])
                result[i] = get_pickling_errors(state[i], seen)
    return result 

def get_pickling_errors(obj, seen=None):
    if seen == None:
        seen = []
    try:
        state = obj.__getstate__()
    except AttributeError:
        return
    if state == None:
        return
    if isinstance(state, tuple):
        if not isinstance(state[0], dict):
            state = state[1]
        else:
            state = state[0].update(state[1])
    result = {}
    for i in state:
        try:
            pickle.dumps(state[i], protocol=2)
        except pickle.PicklingError:
            if not state[i] in seen:
                seen.append(state[i])
                result[i] = get_pickling_errors(state[i], seen)
    return result 

def save_global(self, obj, name=None, pack=struct.pack):
        # We have to override this method in order to deal with objects
        # defined interactively in IPython that are not injected in
        # __main__
        kwargs = dict(name=name, pack=pack)
        if sys.version_info >= (3, 4):
            del kwargs['pack']
        try:
            Pickler.save_global(self, obj, **kwargs)
        except pickle.PicklingError:
            Pickler.save_global(self, obj, **kwargs)
            module = getattr(obj, "__module__", None)
            if module == '__main__':
                my_name = name
                if my_name is None:
                    my_name = obj.__name__
                mod = sys.modules[module]
                if not hasattr(mod, my_name):
                    # IPython doesn't inject the variables define
                    # interactively in __main__
                    setattr(mod, my_name, obj) 

def pickle_function(func):
    mod_name = getattr(func, "__module__", None)
    qname = getattr(func, "__qualname__", None)
    self_ = getattr(func, "__self__", None)

    try:
        test = unpickle_function(mod_name, qname, self_)
    except pickle.UnpicklingError:
        test = None

    if test is not func:
        raise pickle.PicklingError(
            "Can't pickle {}: it's not the same object as {}".format(func, test)
        )

    return unpickle_function, (mod_name, qname, self_) 

def save_global(self, obj, name=None, pack=struct.pack):
        # We have to override this method in order to deal with objects
        # defined interactively in IPython that are not injected in
        # __main__
        kwargs = dict(name=name, pack=pack)
        if sys.version_info >= (3, 4):
            del kwargs['pack']
        try:
            Pickler.save_global(self, obj, **kwargs)
        except pickle.PicklingError:
            Pickler.save_global(self, obj, **kwargs)
            module = getattr(obj, "__module__", None)
            if module == '__main__':
                my_name = name
                if my_name is None:
                    my_name = obj.__name__
                mod = sys.modules[module]
                if not hasattr(mod, my_name):
                    # IPython doesn't inject the variables define
                    # interactively in __main__
                    setattr(mod, my_name, obj) 

def sendFuture(self, future):
        """Send a Future to be executed remotely."""
        future = copy.copy(future)
        future.greenlet = None
        future.children = {}

        try:
            if shared.getConst(hash(future.callable), timeout=0):
                # Enforce name reference passing if already shared
                future.callable = SharedElementEncapsulation(hash(future.callable))
            self.socket.send_multipart([
                TASK,
                pickle.dumps(future.id, pickle.HIGHEST_PROTOCOL),
                pickle.dumps(future, pickle.HIGHEST_PROTOCOL),
            ])
        except (pickle.PicklingError, TypeError) as e:
            # If element not picklable, pickle its name
            # TODO: use its fully qualified name
            scoop.logger.warn("Pickling Error: {0}".format(e))
            future.callable = hash(future.callable)
            self.socket.send_multipart([
                TASK,
                pickle.dumps(future.id, pickle.HIGHEST_PROTOCOL),
                pickle.dumps(future, pickle.HIGHEST_PROTOCOL),
            ]) 

def sendFuture(self, future):
        """Send a Future to be executed remotely."""
        try:
            if shared.getConst(hash(future.callable),
                               timeout=0):
                # Enforce name reference passing if already shared
                future.callable = SharedElementEncapsulation(hash(future.callable))
            self.socket.send_multipart([b"TASK",
                                        pickle.dumps(future,
                                                     pickle.HIGHEST_PROTOCOL)])
        except pickle.PicklingError as e:
            # If element not picklable, pickle its name
            # TODO: use its fully qualified name
            scoop.logger.warn("Pickling Error: {0}".format(e))
            previousCallable = future.callable
            future.callable = hash(future.callable)
            self.socket.send_multipart([b"TASK",
                                        pickle.dumps(future,
                                                     pickle.HIGHEST_PROTOCOL)])
            future.callable = previousCallable 

def pickleRoundTrip(self, obj, name, dumper, loader, proto):
        save_m = sys.modules[name]
        try:
            sys.modules[name] = dumper
            temp = pickle.dumps(obj, proto)
            sys.modules[name] = loader
            result = pickle.loads(temp)
        except pickle.PicklingError as pe:
            # pyET must be second, because pyET may be (equal to) ET.
            human = dict([(ET, "cET"), (pyET, "pyET")])
            raise support.TestFailed("Failed to round-trip %r from %r to %r"
                                     % (obj,
                                        human.get(dumper, dumper),
                                        human.get(loader, loader))) from pe
        finally:
            sys.modules[name] = save_m
        return result 

def _read(etree):
        states = [CRFInfo.State._read(et) for et in
                  etree.findall('states/state')]
        weight_groups = [CRFInfo.WeightGroup._read(et) for et in
                         etree.findall('weightGroups/weightGroup')]
        fd = etree.find('featureDetector')
        feature_detector = fd.get('name')
        if fd.find('pickle') is not None:
            try: feature_detector = pickle.loads(fd.find('pickle').text)
            except pickle.PicklingError, e: pass # unable to unpickle it.

        return CRFInfo(states,
                       float(etree.find('gaussianVariance').text),
                       etree.find('defaultLabel').text,
                       int(etree.find('maxIterations').text),
                       etree.find('transductionType').text,
                       weight_groups,
                       bool(etree.find('addStartState').text),
                       bool(etree.find('addEndState').text),
                       etree.find('modelFile').text,
                       feature_detector) 

def test_pickle(self):
        a = ET.Element('a')
        it = a.iter()
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            with self.assertRaises((TypeError, pickle.PicklingError)):
                pickle.dumps(it, proto) 

def dump(self, obj):
        self.inject_addons()
        try:
            return Pickler.dump(self, obj)
        except RuntimeError as e:
            if 'recursion' in e.args[0]:
                msg = """Could not pickle object as excessively deep recursion required."""
                raise pickle.PicklingError(msg)
            else:
                raise 

def __reduce__(self):
        # In general, only the root logger will not be accessible via its name.
        # However, the root logger's class has its own __reduce__ method.
        if getLogger(self.name) is not self:
            import pickle
            raise pickle.PicklingError('logger cannot be pickled')
        return getLogger, (self.name,) 

def test_pickle(self):
        d = {1: 10, "a": "ABC"}
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            self.assertRaises((TypeError, pickle.PicklingError),
                pickle.dumps, d.keys(), proto)
            self.assertRaises((TypeError, pickle.PicklingError),
                pickle.dumps, d.values(), proto)
            self.assertRaises((TypeError, pickle.PicklingError),
                pickle.dumps, d.items(), proto) 

def test_secure_dump():
    foo = object()
    bar = lambda: None  # flake8: noqa
    with NamedTemporaryFile(delete=False, dir=config.temp_dir) as f:
        secure_dump(foo, f.name)
    assert_raises(PicklingError, secure_dump, bar, f.name)
    with open(f.name, 'rb') as f:
        assert type(load(f)) is object 

def test_bad_init(self):
        # Test issue3664 (pickle can segfault from a badly initialized Pickler).
        # Override initialization without calling __init__() of the superclass.
        class BadPickler(pickle.Pickler):
            def __init__(self): pass

        class BadUnpickler(pickle.Unpickler):
            def __init__(self): pass

        self.assertRaises(pickle.PicklingError, BadPickler().dump, 0)
        self.assertRaises(pickle.UnpicklingError, BadUnpickler().load) 

def test_exploding_pickle(self):
        BadPickle = Enum(
                'BadPickle', 'dill sweet bread-n-butter', module=__name__)
        globals()['BadPickle'] = BadPickle
        # now break BadPickle to test exception raising
        enum._make_class_unpicklable(BadPickle)
        test_pickle_exception(self.assertRaises, TypeError, BadPickle.dill)
        test_pickle_exception(self.assertRaises, PicklingError, BadPickle) 

def test_pickle(self):
        def f():
            yield 1
        g = f()
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            with self.assertRaises((TypeError, pickle.PicklingError)):
                pickle.dumps(g, proto) 

def toxml(self):
        info = self.__dict__.copy()
        info['states'] = '\n'.join(state.toxml() for state in self.states)
        info['w_groups'] = '\n'.join(wg.toxml() for wg in self.weight_groups)
        info['feature_detector_name'] = (info['feature_detector_name']
                                         .replace('&', '&')
                                         .replace('<', '&lt;'))
        try:
            fd = pickle.dumps(self.feature_detector)
            fd = fd.replace('&', '&amp;').replace('<', '&lt;')
            fd = fd.replace('\n', '&#10;') # put pickle data all on 1 line.
            info['feature_detector'] = '<pickle>%s</pickle>' % fd
        except pickle.PicklingError:
            info['feature_detector'] = ''
        return self._XML_TEMPLATE % info 

def save_file(self, obj):
        """Save a file"""
        try:
            import StringIO as pystringIO  # we can't use cStringIO as it lacks the name attribute
        except ImportError:
            import io as pystringIO

        if not hasattr(obj, 'name') or not hasattr(obj, 'mode'):
            raise pickle.PicklingError("Cannot pickle files that do not map to an actual file")
        if obj is sys.stdout:
            return self.save_reduce(getattr, (sys, 'stdout'), obj=obj)
        if obj is sys.stderr:
            return self.save_reduce(getattr, (sys, 'stderr'), obj=obj)
        if obj is sys.stdin:
            raise pickle.PicklingError("Cannot pickle standard input")
        if obj.closed:
            raise pickle.PicklingError("Cannot pickle closed files")
        if hasattr(obj, 'isatty') and obj.isatty():
            raise pickle.PicklingError("Cannot pickle files that map to tty objects")
        if 'r' not in obj.mode and '+' not in obj.mode:
            raise pickle.PicklingError("Cannot pickle files that are not opened for reading: %s" % obj.mode)

        name = obj.name

        retval = pystringIO.StringIO()

        try:
            # Read the whole file
            curloc = obj.tell()
            obj.seek(0)
            contents = obj.read()
            obj.seek(curloc)
        except IOError:
            raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name)
        retval.write(contents)
        retval.seek(curloc)

        retval.name = name
        self.save(retval)
        self.memoize(obj) 

def save_unsupported(self, obj):
        raise pickle.PicklingError("Cannot pickle objects of type %s" % type(obj)) 

def save_file(self, obj):
        """Save a file"""
        try:
            import StringIO as pystringIO #we can't use cStringIO as it lacks the name attribute
        except ImportError:
            import io as pystringIO

        if not hasattr(obj, 'name') or  not hasattr(obj, 'mode'):
            raise pickle.PicklingError("Cannot pickle files that do not map to an actual file")
        if obj is sys.stdout:
            return self.save_reduce(getattr, (sys,'stdout'), obj=obj)
        if obj is sys.stderr:
            return self.save_reduce(getattr, (sys,'stderr'), obj=obj)
        if obj is sys.stdin:
            raise pickle.PicklingError("Cannot pickle standard input")
        if obj.closed:
            raise pickle.PicklingError("Cannot pickle closed files")
        if hasattr(obj, 'isatty') and obj.isatty():
            raise pickle.PicklingError("Cannot pickle files that map to tty objects")
        if 'r' not in obj.mode and '+' not in obj.mode:
            raise pickle.PicklingError("Cannot pickle files that are not opened for reading: %s" % obj.mode)

        name = obj.name

        retval = pystringIO.StringIO()

        try:
            # Read the whole file
            curloc = obj.tell()
            obj.seek(0)
            contents = obj.read()
            obj.seek(curloc)
        except IOError:
            raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name)
        retval.write(contents)
        retval.seek(curloc)

        retval.name = name
        self.save(retval)
        self.memoize(obj) 

def dumps(self, obj):
        try:
            return cloudpickle.dumps(obj, 2)
        except pickle.PickleError:
            raise
        except Exception as e:
            emsg = _exception_message(e)
            if "'i' format requires" in emsg:
                msg = "Object too large to serialize: %s" % emsg
            else:
                msg = "Could not serialize object: %s: %s" % (e.__class__.__name__, emsg)
            cloudpickle.print_exec(sys.stderr)
            raise pickle.PicklingError(msg)