Python pickle.UnpicklingError() Examples

def __init__(self, getter=DEFAULT_GETTER, *args, **kwargs):
        super(TokenDictionaryFeature, self).__init__(getter=getter, *args, **kwargs)
        self._is_boolean = True
        
        if self._path is not None:
            try:
                self._value = pickle.load(open(self._path))
            except (pickle.UnpicklingError, ImportError, EOFError, IndexError, TypeError):
                self._value = compile_token(self._path, "utf-8")
            self._entries = None
        elif self._entries is not None:
            self._value = set()
            for entry in self._entries:
                entry = entry.strip()
                if entry:
                    self._value.add(entry)
        
        assert self._value is not None 

def __init__(self, *args, **kwargs):
        super(MultiwordDictionaryFeature, self).__init__(*args, **kwargs)
        self._is_sequence = True
        self._entry       = kwargs["entry"]
        self._appendice   = kwargs.get("appendice", "")
        
        if self._path is not None:
            try:
                self._value = pickle.load(open(self._path))
            except (pickle.UnpicklingError, ImportError, EOFError):
                self._value = compile_multiword(self._path, "utf-8")
            self._entries = None
        elif self._entries:
            self._value = Trie()
            for entry in self._entries:
                entry = entry.strip()
                if entry:
                    self._value.add(entry.split())
        else:
            self._value = Trie() 

def Do_SConsignDir(name):
    try:
        fp = open(name, 'rb')
    except (IOError, OSError) as e:
        sys.stderr.write("sconsign: %s\n" % e)
        return
    try:
        sconsign = SCons.SConsign.Dir(fp)
    except KeyboardInterrupt:
        raise
    except pickle.UnpicklingError:
        err = "sconsign: ignoring invalid .sconsign file `%s'\n" % (name)
        sys.stderr.write(err)
        return
    except Exception as e:
        err = "sconsign: ignoring invalid .sconsign file `%s': %s\n" % (name, e)
        sys.stderr.write(err)
        return
    printentries(sconsign.entries, args[0])


############################################################################## 

def fromFile(cls, file):
        try:
            cacheName = file + ".pickle"
            cacheKey = binStat(file) + BOB_INPUT_HASH
            with open(cacheName, "rb") as f:
                persistedCacheKey = f.read(len(cacheKey))
                if cacheKey == persistedCacheKey:
                    return pickle.load(f)
        except (EOFError, OSError, pickle.UnpicklingError) as e:
            pass

        audit = cls()
        try:
            with gzip.open(file, 'rb') as gzf:
                audit.load(gzf, file)
            with open(cacheName, "wb") as f:
                f.write(cacheKey)
                pickle.dump(audit, f, -1)
        except OSError as e:
            raise ParseError("Error loading audit: " + str(e))
        return audit 

def load(self):
        """Load the file from disk.

        Returns:
            bool: True if successfully loaded, False if the file
                doesn't exist.

        Raises:
            UnknownFormat: When the file exists but couldn't be loaded.
        """

        if not self._loaded and os.path.exists(self.file_path):
            with open(self.file_path, 'rb') as f:
                for loader in (pickle.load, json.load):
                    try:
                        f.seek(0)
                        self._store = loader(f)
                        self._loaded = True
                        break
                    except pickle.UnpicklingError:
                        pass
            # If the file exists and wasn't able to be loaded, raise an error.
            if not self._loaded:
                raise UnknownFormat('Failed to load file')
        return self._loaded 

def _get_authenticated_session(self, cookies_file):
        session = Session()
        try:
            cfile = open(cookies_file, 'rb')
        except FileNotFoundError:
            message = 'Could not open cookies file, either file does not exist or no read access.'
            raise InvalidCookies(message)
        try:
            session.cookies.update(pickle.load(cfile))
            self._logger.debug('Successfully loaded pickled cookie!')
            warnings.warn('Pickled cookie format is going to be deprecated in a future version, '
                          'please start using a text base cookie file!')
        except (pickle.UnpicklingError, KeyError, AttributeError, EOFError, ValueError):
            self._logger.debug('Trying to load text based cookies.')
            session = self._load_text_cookies(session, cfile)
        cfile.close()
        return session 

def open_pickle(path: str):
    """Open a pickle and return loaded pickle object.
    :type path: str
    :param : path: File path to pickle file to be opened.
    :rtype : object
    """
    try:
        with open(path, 'rb') as opened_pickle:
            try:
                return pickle.load(opened_pickle)
            except Exception as pickle_error:
                logger.error(pickle_error)
                raise
    except FileNotFoundError as fnf_error:
        logger.error(fnf_error)
        raise
    except IOError as io_err:
        logger.error(io_err)
        raise
    except EOFError as eof_error:
        logger.error(eof_error)
        raise
    except pickle.UnpicklingError as unp_error:
        logger.error(unp_error)
        raise 

def do_trace(proj, test_name, input_data, **kwargs):
    """
    trace, magic, crash_mode, crash_addr = load_cached_trace(proj, "test_blurble")
    """
    fname = os.path.join(bin_location, 'tests_data', 'runner_traces', '%s_%s_%s.p' % (test_name, os.path.basename(proj.filename), proj.arch.name))

    if os.path.isfile(fname):
        try:
            with open(fname, 'rb') as f:
                r = pickle.load(f)
                if type(r) is tuple and len(r) == 2 and r[1] == TRACE_VERSION:
                    return r[0]
        except (pickle.UnpicklingError, UnicodeDecodeError):
            print("Can't unpickle trace - rerunning")

    if tracer is None:
        raise Exception("Tracer is not installed and cached data is not present - cannot run test")

    runner = tracer.QEMURunner(project=proj, input=input_data, **kwargs)
    r = (runner.trace, runner.magic, runner.crash_mode, runner.crash_addr)
    with open(fname, 'wb') as f:
        pickle.dump((r, TRACE_VERSION), f, -1)
    return r 

def __init__(self,
                 gtf_file,
                 fasta_file,
                 intron5prime_len=100,
                 intron3prime_len=100,
                 transform=None,
                 **kwargs):

        try:
            with open(gtf_file, 'rb') as f:
                self.exons = pickle.load(f)
        except (FileNotFoundError, pickle.UnpicklingError, ModuleNotFoundError):
            self.exons = generate_exons(gtf_file=gtf_file,
                                        overhang=(intron5prime_len,
                                                  intron3prime_len),
                                        **kwargs)
        import six
        if isinstance(fasta_file, six.string_types):
            fasta = Fasta(fasta_file, as_raw=False)
        self.fasta = fasta
        self.transform = transform 

def unpickle_function(mod_name, qname, self_):
    import importlib

    try:
        module = importlib.import_module(mod_name)
        qname = qname.split(".")
        func = module
        for q in qname:
            func = getattr(func, q)

        if self_ is not None:
            func = types.MethodType(func, self_)

        return func
    except (ImportError, AttributeError) as e:
        from pickle import UnpicklingError

        raise UnpicklingError from e 

def pickle_function(func):
    mod_name = getattr(func, "__module__", None)
    qname = getattr(func, "__qualname__", None)
    self_ = getattr(func, "__self__", None)

    try:
        test = unpickle_function(mod_name, qname, self_)
    except pickle.UnpicklingError:
        test = None

    if test is not func:
        raise pickle.PicklingError(
            "Can't pickle {}: it's not the same object as {}".format(func, test)
        )

    return unpickle_function, (mod_name, qname, self_) 

def test_badly_quoted_string(self):
        # Issue #17710
        badpickles = [b"S'\n.",
                      b'S"\n.',
                      b'S\' \n.',
                      b'S" \n.',
                      b'S\'"\n.',
                      b'S"\'\n.',
                      b"S' ' \n.",
                      b'S" " \n.',
                      b"S ''\n.",
                      b'S ""\n.',
                      b'S \n.',
                      b'S\n.',
                      b'S.']
        for p in badpickles:
            self.check_unpickling_error(pickle.UnpicklingError, p) 

def get_settings(self):
        # Try opening and loading the settings from file.
        filename = os.path.join(self.__path, self.FILENAME)
        try:
            with open(filename, 'rb') as file:
                settings = pickle.load(file)
            # Test the pickle and check each setting inside it.
            assert isinstance(settings, dict)
            key_list = list(self.DEFAULT)
            for key in settings:
                assert isinstance(key, str)
                assert key in self.DEFAULT
                key_list.remove(key)
            # Add new settings as needed (when new ones are created).
            for key in key_list:
                settings[key] = self.DEFAULT[key]
            # Return old settings, or on error, the default settings.
            return False, settings
        except (IOError, pickle.UnpicklingError, AssertionError):
            return True, self.DEFAULT 

def unpickle_pipe(self, fifo):
        frontend_utils.echo_info("Gathering data...")
        try:
            items = pickle.load(fifo)
            if items:
                if isinstance(items, Exception):
                    raise items
                return items
        except EOFError:
            return
        except pickle.UnpicklingError as e:
            frontend_utils.echo_error(f"Error retrieving data from process: {e}")
            raise
        except Exception as e:
            frontend_utils.echo_error(
                f"{type(e).__name__} occurred during analysis: {e}"
            )
            raise 

def test_direct_rest_calls(session, change_dir):
    """Ensure the direct_REST_calls.py example executes successfully."""
    from pickle import UnpicklingError

    # Mock up Session() to return the Betamax-recorded session
    def Session(*args, **kwargs):
        return session

    change_dir('examples')
    with open('direct_REST_calls.py') as f:
        # Remove import of Session to ensure mock function will be used
        # instead.
        code = f.read().replace('from sasctl import get, get_link, request_link, Session',
                                'from sasctl import get, get_link, request_link')
        try:
            six.exec_(code)
        except (UnpicklingError, KeyError) as e:
            if "'\xef'" in str(e):
                # Betamax recording adds additional bytes to the content which
                # breaks unpickling.  Ignore when this happens as correct
                # handling of binary contents should be validated in integration
                # tests
                pass 

def test_get_file_content(self, dummy_file):

        with open(dummy_file, 'r') as f:
            target = f.read()

        # Should return binary pickle of text file contents
        content = files.get_file_content(self.filename)

        # Betamax recording seems to add 4 extra bytes to the beginning?
        # Doesn't occur during real requests.
        try:
            result = pickle.loads(content)
        except (KeyError, pickle.UnpicklingError):
            result = pickle.loads(content[4:])

        assert target == result 

def _GetValueAndType(self, key):
    """Fetch value from memcache and detect its type.

    Args:
      key: String

    Returns:
      (value, type), value is a Python object or None if the key was not set in
      the cache, type is a string describing the type of the value.
    """
    try:
      value = memcache.get(key)
    except (pickle.UnpicklingError, AttributeError, EOFError, ImportError,
            IndexError), e:


      msg = 'Failed to retrieve value from cache: %s' % e
      return msg, 'error' 

def get(self, timeout=None):
        """Receive, decode and return data from the pipe. Block
        gevent-cooperatively until data is available or timeout expires. The
        default decoder is ``pickle.loads``.

        :arg timeout: ``None`` (default) or a ``gevent.Timeout``
            instance. The timeout must be started to take effect and is
            canceled when the first byte of a new message arrives (i.e.
            providing a timeout does not guarantee that the method completes
            within the timeout interval).

        :returns: a Python object.

        Raises:
            - :exc:`gevent.Timeout` (if provided)
            - :exc:`GIPCError`
            - :exc:`GIPCClosed`
            - :exc:`pickle.UnpicklingError`

        Recommended usage for silent timeout control::

            with gevent.Timeout(TIME_SECONDS, False) as t:
                reader.get(timeout=t)

        .. warning::

            The timeout control is currently not available on Windows,
            because Windows can't apply select() to pipe handles.
            An ``OSError`` is expected to be raised in case you set a
            timeout.
        """
        self._validate()
        with self._lock:
            if timeout:
                # Wait for ready-to-read event.
                h = gevent.get_hub()
                h.wait(h.loop.io(self._fd, 1))
                timeout.cancel()
            msize, = struct.unpack("!i", self._recv_in_buffer(4).getvalue())
            bindata = self._recv_in_buffer(msize).getvalue()
        return self._decoder(bindata) 

def find_class(self, module, name):
        # Only allow safe classes from builtins.
        if module == "builtins" and name in safe_builtins:
            return getattr(builtins, name)
        # Forbid everything else.
        raise pickle.UnpicklingError("global '%s.%s' is forbidden" %
                                     (module, name)) 

def load(file_or_path):
    """
    Load a previously pickled model, using `m.pickle('path/to/file.pickle)'`

    :param file_name: path/to/file.pickle
    """
    from pickle import UnpicklingError
    _python3 = True
    try: 
        import cPickle as pickle
        _python3 = False
    except ImportError: #python3
        import pickle
   
    try:
        if _python3:
            strcl = str
            p3kw = dict(encoding='latin1')
            return _unpickle(file_or_path, pickle, strcl, p3kw)
        else:
            strcl = basestring
            p3kw = {}
            return _unpickle(file_or_path, pickle, strcl, p3kw)
    
    except UnpicklingError: # pragma: no coverage
        import pickle
        return _unpickle(file_or_path, pickle, strcl, p3kw) 

def __generatePackages(self, nameFormatter, env, cacheKey, sandboxEnabled):
        # use separate caches with and without sandbox
        if sandboxEnabled:
            cacheName = ".bob-packages-sb.pickle"
        else:
            cacheName = ".bob-packages.pickle"

        # try to load the persisted packages
        try:
            with open(cacheName, "rb") as f:
                persistedCacheKey = f.read(len(cacheKey))
                if cacheKey == persistedCacheKey:
                    tmp = PackageUnpickler(f, self.getRecipe, self.__plugins,
                                           nameFormatter).load()
                    return tmp.refDeref([], {}, None, nameFormatter)
        except (EOFError, OSError, pickle.UnpicklingError):
            pass

        # not cached -> calculate packages
        states = { n:s() for (n,s) in self.__states.items() }
        result = self.__rootRecipe.prepare(env, sandboxEnabled, states)[0]

        # save package tree for next invocation
        try:
            newCacheName = cacheName + ".new"
            with open(newCacheName, "wb") as f:
                f.write(cacheKey)
                PackagePickler(f, nameFormatter).dump(result)
            os.replace(newCacheName, cacheName)
        except OSError as e:
            print("Error saving internal state:", str(e), file=sys.stderr)

        return result.refDeref([], {}, None, nameFormatter) 

def persistent_load(self, pid):
        (tag, key) = pid
        if tag == "pathfmt":
            return self.__pathFormatter
        elif tag == "recipe":
            return self.__recipeGetter(key)
        else:
            raise pickle.UnpicklingError("unsupported object") 

def test_load_negative(self):
        if cPickle.__name__ == "cPickle":   # pickle vs. cPickle report different exceptions, even on Cpy
            filename = os.tempnam()
            for temp in ['\x02', "No"]:
                self.write_to_file(filename, content=temp)
                f = open(filename)
                self.assertRaises(cPickle.UnpicklingError, cPickle.load, f)
                f.close() 

def test_corrupt_shelf(self):
        ''' Reusing the values of a previous shelf '''

        emptyfile = os.path.join(self.shelf_dir, "emptyfile")
        invalidfile = os.path.join(self.shelf_dir, "invalid_file")
        with open(emptyfile, 'w+b'), open(invalidfile, 'w+b') as inf:
            inf.write(b'ohno')

        files = {emptyfile: ['empty file', (EOFError, IndexError)],
                 invalidfile: ['invalid file', (pickle.UnpicklingError, IndexError)]}

        for fn in files:
            with open(fn, 'rb') as f:
                msg, error = files[fn]
                a = ShelfDummyPlugin(info={
                    'name': 'test_corrupt_shelf_{}'.format(msg),
                    'description': 'Dummy plugin for tests',
                    'version': 'test',
                    'shelf_file': f.name
                })
                assert os.path.isfile(a.shelf_file)
                print('Shelf file: %s' % a.shelf_file)
                with self.assertRaises(error):
                    a.sh['a'] = 'fromA'
                    a.save()
                del a._sh
                assert os.path.isfile(a.shelf_file)
                a.force_shelf = True
                a.sh['a'] = 'fromA'
                a.save()
                b = pickle.load(f)
                assert b['a'] == 'fromA' 

def sh(self):
        if not hasattr(self, '_sh') or self._sh is None:
            self._sh = {}
            if os.path.isfile(self.shelf_file):
                try:
                    with self.open(self.shelf_file, 'rb') as p:
                        self._sh = pickle.load(p)
                except (IndexError, EOFError, pickle.UnpicklingError):
                    self.log.warning('Corrupted shelf file: {}'.format(
                        self.shelf_file))
                    if not self.get('force_shelf', False):
                        raise
        return self._sh 

def load(self):
        with open(self.path, 'rb') as f:
            try:
                loaded_state = pickle.load(f)
            except (pickle.UnpicklingError, IndexError):
                raise ValueError("Corrupt state file: {}".format(self.path))

        return loaded_state 

def load(self):
        try:
            loaded_state = pickle.loads(self.redis.get(self.REDIS_STATE_KEY))
            return loaded_state
        except pickle.UnpicklingError:
            raise ValueError("Corrupt state file in redis") 

def deserialize_data(value):
    try:
        if g.PY_IS_VER2:
            # On python 2 pickle.loads wants str
            from base64 import standard_b64decode
            return pickle.loads(standard_b64decode(value))
        # On python 3 pickle.loads wants byte
        return pickle.loads(value)
    except (pickle.UnpicklingError, TypeError, EOFError):
        # TypeError/EOFError happen when standard_b64decode fails
        # This should happen only if manually mixing the database data
        common.error('It was not possible to deserialize the cache data, try purge cache from expert settings menu')
        raise CacheMiss() 

def file_infos(names=None):
    """
    iterates over storage files metadata.
    note: we put the storage name into the metadata as ID

    :param names: None means "all items"
                  otherwise give a list of storage item names
    """
    storage = current_app.storage
    if names is None:
        names = list(storage)
    for name in names:
        try:
            with storage.open(name) as item:
                meta = dict(item.meta)
                if not meta:
                    # we got empty metadata, this happens for 0-byte .meta files.
                    # ignore it for now.
                    continue
                if delete_if_lifetime_over(item, name):
                    continue
                meta[ID] = name
                yield meta
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
        except pickle.UnpicklingError:
            # corrupted meta file, just ignore it for now
            pass 

def test_open_pickle_fail_corrupt(self):
        """Test failure to open corrupted pickle."""
        bad_file = 'cltk/tests/test_nlp/bad_pickle.pickle'
        with self.assertRaises(UnpicklingError):
            open_pickle(bad_file)