Python multiprocessing.get_start_method() Examples

def __init__(self, pool_names, max_restarts=0, options=None):
		self.names = pool_names
		self.queue = multiprocessing.Queue()
		self.pool = dict()
		self.max_restarts = max_restarts
		self.options = options or dict()

		self.dog_path = os.curdir
		self.dog_handler = LiveReload(self)
		# self.dog_observer = Observer()
		# self.dog_observer.schedule(self.dog_handler, self.dog_path, recursive=True)

		if multiprocessing.get_start_method() != 'fork':  # pragma: no cover
			root_logger = logging.getLogger()
			self.log_listener = QueueListener(self.queue, *root_logger.handlers)

		# TODO: Find out how to get the watchdog + livereload working on a later moment.
		# self.dog_observer.start()

		self._restarts = dict() 

def main(args):
    with open(args.config) as f:
        if version.parse(yaml.version >= "5.1"):
            config = yaml.load(f, Loader=yaml.FullLoader)
        else:
            config = yaml.load(f)

    for k, v in config.items():
        setattr(args, k, v)

    # exp path
    if not hasattr(args, 'exp_path'):
        args.exp_path = os.path.dirname(args.config)

    # dist init
    if mp.get_start_method(allow_none=True) != 'spawn':
        mp.set_start_method('spawn', force=True)
    dist_init(args.launcher, backend='nccl')

    # train
    trainer = Trainer(args)
    trainer.run() 

def start_proc_mask_signal(proc):
    """
    Start process(es) with SIGINT ignored.

    Args:
        proc: (mp.Process or list)

    Note:
        The signal mask is only applied when called from main thread.
    """
    if not isinstance(proc, list):
        proc = [proc]

    with mask_sigint():
        for p in proc:
            if isinstance(p, mp.Process):
                if sys.version_info < (3, 4) or mp.get_start_method() == 'fork':
                    log_once("""
Starting a process with 'fork' method is efficient but not safe and may cause deadlock or crash.
Use 'forkserver' or 'spawn' method instead if you run into such issues.
See https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods on how to set them.
""".replace("\n", ""),
'warn')  # noqa
            p.start() 

def init_processes(addr, port, gpu_num, backend):
    from mpi4py import MPI
    comm = MPI.COMM_WORLD
    size = comm.Get_size()
    rank = comm.Get_rank()
    print(rank, size)
    if mp.get_start_method(allow_none=True) != 'spawn':
        mp.set_start_method('spawn')
    torch.cuda.set_device(rank % gpu_num)
    os.environ['MASTER_ADDR'] = addr
    os.environ['MASTER_PORT'] = port
    os.environ['WORLD_SIZE'] = str(size)
    os.environ['RANK'] = str(rank)
    dist.init_process_group(backend)
    print('initialize {} successfully (rank {})'.format(backend, rank))
    return rank, size 

def test_preload_resources(self):
        if multiprocessing.get_start_method() != 'forkserver':
            self.skipTest("test only relevant for 'forkserver' method")
        name = os.path.join(os.path.dirname(__file__), 'mp_preload.py')
        rc, out, err = test.support.script_helper.assert_python_ok(name)
        out = out.decode()
        err = err.decode()
        if out.rstrip() != 'ok' or err != '':
            print(out)
            print(err)
            self.fail("failed spawning forkserver or grandchild")


#
# Check that killing process does not leak named semaphores
# 

def test_interactive_contex_no_main(self):
        # Ensure that loky context is working properly
        code = '\n'.join([
            'from loky.backend import get_context',
            'ctx = get_context()',
            'assert ctx.get_start_method() == "loky"',
            'p = ctx.Process(target=id, args=(1,))',
            'p.start()',
            'p.join()',
            'msg = "loky context failed to load without safeguard"',
            'assert p.exitcode == 0, msg',
            'print("ok")'
        ])
        try:
            fid, filename = mkstemp(suffix="_joblib.py")
            os.close(fid)
            with open(filename, mode='wb') as f:
                f.write(code.encode('ascii'))
            check_subprocess_call([sys.executable, filename],
                                  stdout_regex=r'ok', timeout=10)
        finally:
            os.unlink(filename) 

def start_proc_mask_signal(proc):
    """
    Start process(es) with SIGINT ignored.

    Args:
        proc: (mp.Process or list)

    Note:
        The signal mask is only applied when called from main thread.
    """
    if not isinstance(proc, list):
        proc = [proc]

    with mask_sigint():
        for p in proc:
            if isinstance(p, mp.Process):
                if sys.version_info < (3, 4) or mp.get_start_method() == 'fork':
                    log_once(
"Starting a process with 'fork' method is not safe and may consume unnecessary extra CPU memory."
" Use 'forkserver' or 'spawn' method (available after Py3.4) instead if you run into any issues. "
"See https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods on how to set them.",
'warn')  # noqa
            p.start() 

def start_proc_mask_signal(proc):
    """
    Start process(es) with SIGINT ignored.

    Args:
        proc: (mp.Process or list)

    Note:
        The signal mask is only applied when called from main thread.
    """
    if not isinstance(proc, list):
        proc = [proc]

    with mask_sigint():
        for p in proc:
            if isinstance(p, mp.Process):
                if sys.version_info < (3, 4) or mp.get_start_method() == 'fork':
                    log_once("""
Starting a process with 'fork' method is efficient but not safe and may cause deadlock or crash.
Use 'forkserver' or 'spawn' method instead if you run into such issues.
See https://docs.python.org/3/library/multiprocessing.html#contexts-and-start-methods on how to set them.
""".replace("\n", ""),
'warn')  # noqa
            p.start() 

def test_closefd(self):
        if not HAS_REDUCTION:
            raise unittest.SkipTest('requires fd pickling')

        reader, writer = multiprocessing.Pipe()
        fd = self.get_high_socket_fd()
        try:
            p = multiprocessing.Process(target=self._test_closefds,
                                        args=(writer, fd))
            p.start()
            writer.close()
            e = reader.recv()
            p.join(timeout=5)
        finally:
            self.close(fd)
            writer.close()
            reader.close()

        if multiprocessing.get_start_method() == 'fork':
            self.assertIs(e, None)
        else:
            WSAENOTSOCK = 10038
            self.assertIsInstance(e, OSError)
            self.assertTrue(e.errno == errno.EBADF or
                            e.winerror == WSAENOTSOCK, e)

#
# Issue #17097: EINTR should be ignored by recv(), send(), accept() etc
# 

def default_test_processes():
    """
    Default number of test processes when using the --parallel option.
    """
    # The current implementation of the parallel test runner requires
    # multiprocessing to start subprocesses with fork().
    # On Python 3.4+: if multiprocessing.get_start_method() != 'fork':
    if not hasattr(os, 'fork'):
        return 1
    try:
        return int(os.environ['DJANGO_TEST_PROCESSES'])
    except KeyError:
        return multiprocessing.cpu_count() 

def test_loky_get_context(self):
        # check the behavior of get_context
        ctx_default = get_context()
        assert ctx_default.get_start_method() == "loky"

        ctx_loky = get_context("loky")
        assert ctx_loky.get_start_method() == "loky"

        ctx_loky_init_main = get_context("loky_init_main")
        assert ctx_loky_init_main.get_start_method() == "loky_init_main"

        with pytest.raises(ValueError):
            get_context("not_available") 

def dist_init(port):
    if mp.get_start_method(allow_none=True) != 'spawn':
        mp.set_start_method('spawn')
    proc_id = int(os.environ['SLURM_PROCID'])
    ntasks = int(os.environ['SLURM_NTASKS'])
    node_list = os.environ['SLURM_NODELIST']
    num_gpus = torch.cuda.device_count()
    torch.cuda.set_device(proc_id % num_gpus)

    if '[' in node_list:
        beg = node_list.find('[')
        pos1 = node_list.find('-', beg)
        if pos1 < 0:
            pos1 = 1000
        pos2 = node_list.find(',', beg)
        if pos2 < 0:
            pos2 = 1000
        node_list = node_list[:min(pos1, pos2)].replace('[', '')
    addr = node_list[8:].replace('-', '.')
    print(addr)

    os.environ['MASTER_PORT'] = port
    os.environ['MASTER_ADDR'] = addr
    os.environ['WORLD_SIZE'] = str(ntasks)
    os.environ['RANK'] = str(proc_id)
    dist.init_process_group(backend='nccl')

    rank = dist.get_rank()
    world_size = dist.get_world_size()
    return rank, world_size 

def _test_default_subcontext(queue):
    if sys.version_info >= (3, 3):
        start_method = mp.get_start_method()
    else:
        from loky.backend.context import _DEFAULT_START_METHOD
        start_method = _DEFAULT_START_METHOD

    queue.put(start_method) 

def test_default_subcontext(method):
    code = """if True:
        import sys

        from loky.backend.context import get_context, set_start_method
        from tests.test_loky_backend import _test_default_subcontext

        set_start_method('{method}')
        ctx = get_context()
        assert ctx.get_start_method() == '{method}'

        queue = ctx.SimpleQueue()
        p = ctx.Process(target=_test_default_subcontext, args=(queue,))
        p.start()
        p.join()
        start_method = queue.get()
        assert start_method == '{method}', start_method

        try:
            set_start_method('loky')
        except RuntimeError:
            pass
        else:
            raise AssertionError("Did not raise RuntimeError when resetting"
                                 "start_method without force")

        set_start_method(None, force=True)
        ctx = get_context()
        assert ctx.get_start_method() == 'loky'
    """.format(method=method)

    cmd = [sys.executable, "-c", code]
    check_subprocess_call(cmd, timeout=10)

    ctx_default = get_context()
    assert ctx_default.get_start_method() == "loky" 

def np_mp_array(shape, dtype):
    """Allocate a numpy array on OS shared memory."""
    if mp.get_start_method() == "spawn":
        return np_mp_array_spawn(shape, dtype)
    size = int(np.prod(shape))
    nbytes = size * np.dtype(dtype).itemsize
    mp_array = mp.RawArray(ctypes.c_char, nbytes)
    return np.frombuffer(mp_array, dtype=dtype, count=size).reshape(shape) 

def test_noforkbomb(self):
        sm = multiprocessing.get_start_method()
        name = os.path.join(os.path.dirname(__file__), 'mp_fork_bomb.py')
        if sm != 'fork':
            rc, out, err = test.support.script_helper.assert_python_failure(name, sm)
            self.assertEqual(out, b'')
            self.assertIn(b'RuntimeError', err)
        else:
            rc, out, err = test.support.script_helper.assert_python_ok(name, sm)
            self.assertEqual(out.rstrip(), b'123')
            self.assertEqual(err, b'')

#
# Issue #17555: ForkAwareThreadLock
# 

def test_closefd(self):
        if not HAS_REDUCTION:
            raise unittest.SkipTest('requires fd pickling')

        reader, writer = multiprocessing.Pipe()
        fd = self.get_high_socket_fd()
        try:
            p = multiprocessing.Process(target=self._test_closefds,
                                        args=(writer, fd))
            p.start()
            writer.close()
            e = reader.recv()
            p.join(timeout=5)
        finally:
            self.close(fd)
            writer.close()
            reader.close()

        if multiprocessing.get_start_method() == 'fork':
            self.assertIs(e, None)
        else:
            WSAENOTSOCK = 10038
            self.assertIsInstance(e, OSError)
            self.assertTrue(e.errno == errno.EBADF or
                            e.winerror == WSAENOTSOCK, e)

#
# Issue #17097: EINTR should be ignored by recv(), send(), accept() etc
# 

def _check_context(cls, conn):
        conn.send(multiprocessing.get_start_method()) 

def check_context(self, ctx):
        r, w = ctx.Pipe(duplex=False)
        p = ctx.Process(target=self._check_context, args=(w,))
        p.start()
        w.close()
        child_method = r.recv()
        r.close()
        p.join()
        self.assertEqual(child_method, ctx.get_start_method()) 

def test_context(self):
        for method in ('fork', 'spawn', 'forkserver'):
            try:
                ctx = multiprocessing.get_context(method)
            except ValueError:
                continue
            self.assertEqual(ctx.get_start_method(), method)
            self.assertIs(ctx.get_context(), ctx)
            self.assertRaises(ValueError, ctx.set_start_method, 'spawn')
            self.assertRaises(ValueError, ctx.set_start_method, None)
            self.check_context(ctx) 

def _get_multiprocessing_start_method(self):
        """
        Determine method of creating new processes by checking if the
        mp_start_method is set in configs, else, it uses the OS default.
        """
        if conf.has_option('core', 'mp_start_method'):
            return conf.get('core', 'mp_start_method')

        return multiprocessing.get_start_method() 

def default_test_processes():
    """Default number of test processes when using the --parallel option."""
    # The current implementation of the parallel test runner requires
    # multiprocessing to start subprocesses with fork().
    if multiprocessing.get_start_method() != 'fork':
        return 1
    try:
        return int(os.environ['DJANGO_TEST_PROCESSES'])
    except KeyError:
        return multiprocessing.cpu_count() 

def dist_init(launcher, backend='nccl', **kwargs):
    if mp.get_start_method(allow_none=True) is None:
        mp.set_start_method('spawn')
    if launcher == 'pytorch':
        _init_dist_pytorch(backend, **kwargs)
    elif launcher == 'mpi':
        _init_dist_mpi(backend, **kwargs)
    elif launcher == 'slurm':
        _init_dist_slurm(backend, **kwargs)
    else:
        raise ValueError('Invalid launcher type: {}'.format(launcher)) 

def setUpClass(cls):
        import multiprocessing as mp
        try:
            mp.set_start_method("spawn")
        except RuntimeError:
            pass
        assert mp.get_start_method() == "spawn"

        write_settings()
        cls.app = QApplication([cls.__name__]) 

def test_noforkbomb(self):
        sm = multiprocessing.get_start_method()
        name = os.path.join(os.path.dirname(__file__), 'mp_fork_bomb.py')
        if sm != 'fork':
            rc, out, err = test.script_helper.assert_python_failure(name, sm)
            self.assertEqual(out, b'')
            self.assertIn(b'RuntimeError', err)
        else:
            rc, out, err = test.script_helper.assert_python_ok(name, sm)
            self.assertEqual(out.rstrip(), b'123')
            self.assertEqual(err, b'')

#
# Issue #17555: ForkAwareThreadLock
# 

def _check_context(cls, conn):
        conn.send(multiprocessing.get_start_method()) 

def check_context(self, ctx):
        r, w = ctx.Pipe(duplex=False)
        p = ctx.Process(target=self._check_context, args=(w,))
        p.start()
        w.close()
        child_method = r.recv()
        r.close()
        p.join()
        self.assertEqual(child_method, ctx.get_start_method()) 

def test_context(self):
        for method in ('fork', 'spawn', 'forkserver'):
            try:
                ctx = multiprocessing.get_context(method)
            except ValueError:
                continue
            self.assertEqual(ctx.get_start_method(), method)
            self.assertIs(ctx.get_context(), ctx)
            self.assertRaises(ValueError, ctx.set_start_method, 'spawn')
            self.assertRaises(ValueError, ctx.set_start_method, None)
            self.check_context(ctx) 

def setUpClass(cls):
        if get_start_method(True) != "spawn":
            set_start_method("spawn")
        cls.tearDownClass() 

def test_read_data_multiprocessing(self):
        if get_start_method(True) != "spawn":
            set_start_method("spawn")
        workers = 1
        output_queue = MPQueue()
        processes = [Process(target=run_read, args=[output_queue]) for i in range(workers)]
        [process.start() for process in processes]
        [process.join() for process in processes]
        for i in range(workers):
            data = output_queue.get(timeout=0.1)
            assert_that(data["frame"][0][0][0], is_(equal_to(97)))
            assert_that(data["frame"][0][0][1], is_(equal_to(98)))
            assert_that(data["frame"][0][0][2], is_(equal_to(99)))
            assert_that(data["test1"], is_(equal_to(1)))
            assert_that(data["test2"], is_(equal_to(2)))