Python multiprocessing.reduction() Examples

The following are 4 code examples of multiprocessing.reduction(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module multiprocessing , or try the search function .
Example #1
Source File: gipc.py    From gipc with MIT License 6 votes vote down vote up 
def _winapi_childhandle_after_createprocess_child(self):
        """Called on Windows in the child process after the CreateProcess()
        system call. This is required for making the handle usable in the child.
        """

        if WINAPI_HANDLE_TRANSFER_STEAL:
            # In this case the handle has not been inherited by the child
            # process during CreateProcess(). Steal it from the parent.
            new_winapihandle = multiprocessing.reduction.steal_handle(
                self._parent_pid, self._parent_winapihandle)
            del self._parent_winapihandle
            del self._parent_pid
            # Restore C file descriptor with (read/write)only flag.
            self._fd = msvcrt.open_osfhandle(new_winapihandle, self._fd_flag)
            return

        # In this case the handle has been inherited by the child process during
        # the CreateProcess() system call. Get C file descriptor from Windows
        # file handle.
        self._fd = msvcrt.open_osfhandle(
            self._inheritable_winapihandle, self._fd_flag)

        del self._inheritable_winapihandle
Example #2
Source File: dataloader.py    From dynamic-training-with-apache-mxnet-on-aws with Apache License 2.0 5 votes vote down vote up 
def rebuild_ndarray(pid, fd, shape, dtype):
        """Rebuild ndarray from pickled shared memory"""
        # pylint: disable=no-value-for-parameter
        if sys.version_info[0] == 2:
            fd = multiprocessing.reduction.rebuild_handle(fd)
        else:
            fd = fd.detach()
        return nd.NDArray(nd.ndarray._new_from_shared_mem(pid, fd, shape, dtype))
Example #3
Source File: dataloader.py    From dynamic-training-with-apache-mxnet-on-aws with Apache License 2.0 5 votes vote down vote up 
def reduce_ndarray(data):
        """Reduce ndarray to shared memory handle"""
        # keep a local ref before duplicating fd
        data = data.as_in_context(context.Context('cpu_shared', 0))
        pid, fd, shape, dtype = data._to_shared_mem()
        if sys.version_info[0] == 2:
            fd = multiprocessing.reduction.reduce_handle(fd)
        else:
            fd = multiprocessing.reduction.DupFd(fd)
        return rebuild_ndarray, (pid, fd, shape, dtype)
Example #4
Source File: parallelized_loader.py    From gluon-ts with Apache License 2.0 5 votes vote down vote up 
def reduce_ndarray(data):
        """Reduce ndarray to shared memory handle"""
        # keep a local ref before duplicating fd
        data = data.as_in_context(context.Context("cpu_shared", 0))
        pid, fd, shape, dtype = data._to_shared_mem()
        fd = multiprocessing.reduction.DupFd(fd)
        return rebuild_ndarray, (pid, fd, shape, dtype)