Python multiprocessing.pool.MaybeEncodingError() Examples

The following are 13 code examples of multiprocessing.pool.MaybeEncodingError(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module multiprocessing.pool , or try the search function .
Example #1
Source File: _test_multiprocessing.py    From Fluid-Designer with GNU General Public License v3.0 6 votes vote down vote up 
def test_unpickleable_result(self):
        from multiprocessing.pool import MaybeEncodingError
        p = multiprocessing.Pool(2)

        # Make sure we don't lose pool processes because of encoding errors.
        for iteration in range(20):

            scratchpad = [None]
            def errback(exc):
                scratchpad[0] = exc

            res = p.apply_async(unpickleable_result, error_callback=errback)
            self.assertRaises(MaybeEncodingError, res.get)
            wrapped = scratchpad[0]
            self.assertTrue(wrapped)
            self.assertIsInstance(scratchpad[0], MaybeEncodingError)
            self.assertIsNotNone(wrapped.exc)
            self.assertIsNotNone(wrapped.value)

        p.close()
        p.join()
Example #2
Source File: _test_multiprocessing.py    From ironpython3 with Apache License 2.0 6 votes vote down vote up 
def test_unpickleable_result(self):
        from multiprocessing.pool import MaybeEncodingError
        p = multiprocessing.Pool(2)

        # Make sure we don't lose pool processes because of encoding errors.
        for iteration in range(20):

            scratchpad = [None]
            def errback(exc):
                scratchpad[0] = exc

            res = p.apply_async(unpickleable_result, error_callback=errback)
            self.assertRaises(MaybeEncodingError, res.get)
            wrapped = scratchpad[0]
            self.assertTrue(wrapped)
            self.assertIsInstance(scratchpad[0], MaybeEncodingError)
            self.assertIsNotNone(wrapped.exc)
            self.assertIsNotNone(wrapped.value)

        p.close()
        p.join()
Example #3
Source File: update_politician_pictures.py    From politicos with GNU Affero General Public License v3.0 6 votes vote down vote up 
def main():
    client = Elasticsearch()

    create_db(cursor)
    clear_db(conn, cursor)

    try:
        logging.info(f'Getting photo URL...')
        pool = Pool(NUM_WORKERS, init_worker)
        data = pool.imap_unordered(_http_get, get_pictures(client))
    except KeyboardInterrupt:
        pool.terminate()
        pool.join()
        raise SystemExit(1)
    except MaybeEncodingError:
        pass

    insert_sqlite_data(data)
    update_es_data(cursor, client)

    conn.close()
Example #4
Source File: _test_multiprocessing.py    From Project-New-Reign---Nemesis-Main with GNU General Public License v3.0 6 votes vote down vote up 
def test_unpickleable_result(self):
        from multiprocessing.pool import MaybeEncodingError
        p = multiprocessing.Pool(2)

        # Make sure we don't lose pool processes because of encoding errors.
        for iteration in range(20):

            scratchpad = [None]
            def errback(exc):
                scratchpad[0] = exc

            res = p.apply_async(unpickleable_result, error_callback=errback)
            self.assertRaises(MaybeEncodingError, res.get)
            wrapped = scratchpad[0]
            self.assertTrue(wrapped)
            self.assertIsInstance(scratchpad[0], MaybeEncodingError)
            self.assertIsNotNone(wrapped.exc)
            self.assertIsNotNone(wrapped.value)

        p.close()
        p.join()
Example #5
Source File: test_multiprocessing.py    From ironpython2 with Apache License 2.0 5 votes vote down vote up 
def test_unpickleable_result(self):
        from multiprocessing.pool import MaybeEncodingError
        p = multiprocessing.Pool(2)

        # Make sure we don't lose pool processes because of encoding errors.
        for iteration in range(20):
            res = p.apply_async(unpickleable_result)
            self.assertRaises(MaybeEncodingError, res.get)

        p.close()
        p.join()
Example #6
Source File: test_multiprocessing.py    From BinderFilter with MIT License 5 votes vote down vote up 
def test_unpickleable_result(self):
        from multiprocessing.pool import MaybeEncodingError
        p = multiprocessing.Pool(2)

        # Make sure we don't lose pool processes because of encoding errors.
        for iteration in range(20):
            res = p.apply_async(unpickleable_result)
            self.assertRaises(MaybeEncodingError, res.get)

        p.close()
        p.join()
Example #7
Source File: test_multiprocessing.py    From oss-ftp with MIT License 5 votes vote down vote up 
def test_unpickleable_result(self):
        from multiprocessing.pool import MaybeEncodingError
        p = multiprocessing.Pool(2)

        # Make sure we don't lose pool processes because of encoding errors.
        for iteration in range(20):
            res = p.apply_async(unpickleable_result)
            self.assertRaises(MaybeEncodingError, res.get)

        p.close()
        p.join()
Example #8
Source File: process.py    From green with MIT License 5 votes vote down vote up 
def __init__(self, exc, value):
            self.exc = repr(exc)
            self.value = repr(value)
            super(MaybeEncodingError, self).__init__(self.exc, self.value)
Example #9
Source File: process.py    From green with MIT License 5 votes vote down vote up 
def __repr__(self):
            return "<MaybeEncodingError: %s>" % str(self)


# Python 2 and 3 raise a different error when they exit
Example #10
Source File: test_multiprocessing.py    From gcblue with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up 
def test_unpickleable_result(self):
        from multiprocessing.pool import MaybeEncodingError
        p = multiprocessing.Pool(2)

        # Make sure we don't lose pool processes because of encoding errors.
        for iteration in range(20):
            res = p.apply_async(unpickleable_result)
            self.assertRaises(MaybeEncodingError, res.get)

        p.close()
        p.join()
Example #11
Source File: test_multiprocessing.py    From CTFCrackTools-V2 with GNU General Public License v3.0 5 votes vote down vote up 
def test_unpickleable_result(self):
        from multiprocessing.pool import MaybeEncodingError
        p = multiprocessing.Pool(2)

        # Make sure we don't lose pool processes because of encoding errors.
        for iteration in range(20):
            res = p.apply_async(unpickleable_result)
            self.assertRaises(MaybeEncodingError, res.get)

        p.close()
        p.join()
Example #12
Source File: test_multiprocessing.py    From CTFCrackTools with GNU General Public License v3.0 5 votes vote down vote up 
def test_unpickleable_result(self):
        from multiprocessing.pool import MaybeEncodingError
        p = multiprocessing.Pool(2)

        # Make sure we don't lose pool processes because of encoding errors.
        for iteration in range(20):
            res = p.apply_async(unpickleable_result)
            self.assertRaises(MaybeEncodingError, res.get)

        p.close()
        p.join()
Example #13
Source File: process.py    From green with MIT License 4 votes vote down vote up 
def worker(
    inqueue,
    outqueue,
    initializer=None,
    initargs=(),
    maxtasks=None,
    wrap_exception=False,
    finalizer=None,
    finalargs=(),
):  # pragma: no cover
    assert maxtasks is None or (type(maxtasks) == int and maxtasks > 0)
    put = outqueue.put
    get = inqueue.get
    if hasattr(inqueue, "_writer"):
        inqueue._writer.close()
        outqueue._reader.close()

    if initializer is not None:
        try:
            initializer(*initargs)
        except InitializerOrFinalizerError as e:
            print(str(e))

    completed = 0
    while maxtasks is None or (maxtasks and completed < maxtasks):
        try:
            task = get()
        except (EOFError, PortableOSError):
            util.debug("worker got EOFError or OSError -- exiting")
            break

        if task is None:
            util.debug("worker got sentinel -- exiting")
            break

        job, i, func, args, kwds = task
        try:
            result = (True, func(*args, **kwds))
        except Exception as e:
            if wrap_exception:
                e = ExceptionWithTraceback(e, e.__traceback__)
            result = (False, e)
        try:
            put((job, i, result))
        except Exception as e:
            wrapped = MaybeEncodingError(e, result[1])
            util.debug("Possible encoding error while sending result: %s" % (wrapped))
            put((job, i, (False, wrapped)))
        completed += 1

    if finalizer:
        try:
            finalizer(*finalargs)
        except InitializerOrFinalizerError as e:
            print(str(e))

    util.debug("worker exiting after %d tasks" % completed)


# Unmodified (see above)