Python multiprocessing.dummy.Process() Examples

def __init__(self, constructor, strategy='thread'):
    """Step environment in a separate process for lock free parallelism.

    The environment will be created in the external process by calling the
    specified callable. This can be an environment class, or a function
    creating the environment and potentially wrapping it. The returned
    environment should not access global variables.

    Args:
      constructor: Callable that creates and returns an OpenAI gym environment.

    Attributes:
      observation_space: The cached observation space of the environment.
      action_space: The cached action space of the environment.
    """
    if strategy == 'thread':
      import multiprocessing.dummy as mp
    elif strategy == 'process':
      import multiprocessing as mp
    else:
      raise NotImplementedError(strategy)
    self._conn, conn = mp.Pipe()
    self._process = mp.Process(target=self._worker, args=(constructor, conn))
    atexit.register(self.close)
    self._process.start()
    self._observ_space = None
    self._action_space = None 

def _get_memory(pid):
        process = psutil.Process(pid)
        try:
            mem = float(process.get_memory_info()[0]) / (1024 ** 2)
        except psutil.AccessDenied:
            mem = -1
        return mem 

def _get_memory(pid):
        process = psutil.Process(pid)
        try:
            mem = float(process.get_memory_info()[0]) / (1024 ** 2)
        except psutil.AccessDenied:
            mem = -1
        return mem 

def __init__(self, constructor, strategy='thread'):
    if strategy == 'thread':
      import multiprocessing.dummy as mp
    elif strategy == 'process':
      import multiprocessing as mp
    else:
      raise NotImplementedError(strategy)
    self._strategy = strategy
    self._conn, conn = mp.Pipe()
    self._process = mp.Process(target=self._worker, args=(constructor, conn))
    atexit.register(self.close)
    self._process.start()
    self._observ_space = None
    self._action_space = None 

def start(self):
        """ Start the thread """
        self._process = Process(target=self.run)
        self._process.daemon = True
        self._running = True
        self._process.start() 

def calc_density_cpu(xs,ys,weights,kernel_type,cpu_core,cutoffd=0,sigma=0):
    xs=xs-xs.min()
    ys=ys-ys.min()
        
    def calc_density_np(gidxys,result_q,xs,ys,weights,kernel_type,cutoffd=0,sigma=0):
        while True:
            try:
                i=gidxys.get_nowait()
                distpow2=(xs-xs[i])**2+(ys-ys[i])**2
                if kernel_type=='GAUSS':
                    result_q.put( (i,((distpow2<((3*sigma)**2))*np.exp(-distpow2/(sigma**2))*weights).sum()))
                else:
                    result_q.put( (i,((distpow2<(cutoffd**2))*weights).sum()))                    
            except queue.Empty:
                break;
        
    n=xs.shape[0]
    gidxys=queue.Queue()
    result_q=queue.Queue()
    for i in range(n):
        gidxys.put(i)
    
    arcpy.SetProgressor("step", "Calculate Densities on CPU...",0, n, 1)
    
    ts=[]
    for i in range(cpu_core):
        t=Process(target=calc_density_np,args=(gidxys,result_q,xs,ys,weights,kernel_type,cutoffd,sigma))
        t.start()
        ts.append(t)
    for t in ts:
        while t.is_alive():
            arcpy.SetProgressorPosition(n-gidxys.qsize())
            time.sleep(0.05)
        
    result_a=[]
    while result_q.empty()==False:
        result_a.append(result_q.get())
    result_a.sort()
    result_d=[]
    for v in result_a:
        result_d.append(v[1])
    return np.array(result_d) 

def calc_nrst_dist_cpu(gids,xs,ys,densities,cpu_core):
    n=xs.shape[0]
    
    def calc_nrst_dist_np(gidxys,result_q,gids,xs,ys,densities):
        while True:
            try:
                i=gidxys.get_nowait()
                distpow2=(xs-xs[i])**2+(ys-ys[i])**2
                distpow2[densities<=densities[i]]=1e100
                pg=distpow2.argsort()[0]
                if distpow2[pg]>1e99:
                    result_q.put((i,1e10,-1))
                else:
                    result_q.put((i,math.sqrt(distpow2[pg]),gids[pg]))
            except queue.Empty:
                break;
                
    n=xs.shape[0]
    gidxys=queue.Queue()
    result_q=queue.Queue()
    for i in range(n):
        gidxys.put(i)
    
    arcpy.SetProgressor("step", "Find Point with Higher Density on CPU...",0, n, 1)
    
    ts=[]
    for i in range(cpu_core):
        t=Process(target=calc_nrst_dist_np,args=(gidxys,result_q,gids,xs,ys,densities))
        t.start()
        ts.append(t)
    for t in ts:
        while t.is_alive():
            arcpy.SetProgressorPosition(n-gidxys.qsize())
            time.sleep(0.05)
        
    result_a=[]
    while result_q.empty()==False:
        result_a.append(result_q.get())
    result_a.sort()
    result_nd=[]
    result_pg=[]
    for v in result_a:
        result_nd.append(v[1])
        result_pg.append(v[2])
    return (np.array(result_nd),np.array(result_pg)) 

def _get_memory(pid, timestamps=False, include_children=False):

    # .. only for current process and only on unix..
    if pid == -1:
        pid = os.getpid()

    # .. cross-platform but but requires psutil ..
    if has_psutil:
        process = psutil.Process(pid)
        try:
            mem_info = getattr(process, 'memory_info', process.get_memory_info)
            mem = mem_info()[0] / _TWO_20
            if include_children:
                for p in process.get_children(recursive=True):
                    mem_info = getattr(p, 'memory_info', p.get_memory_info)
                    mem += mem_info()[0] / _TWO_20
            if timestamps:
                return (mem, time.time())
            else:
                return mem
        except psutil.AccessDenied:
            pass
            # continue and try to get this from ps

    # .. scary stuff ..
    if os.name == 'posix':
        if include_children:
            raise NotImplementedError('The psutil module is required when to'
                                      ' monitor memory usage of children'
                                      ' processes')
        warnings.warn("psutil module not found. memory_profiler will be slow")
        # ..
        # .. memory usage in MiB ..
        # .. this should work on both Mac and Linux ..
        # .. subprocess.check_output appeared in 2.7, using Popen ..
        # .. for backwards compatibility ..
        out = subprocess.Popen(['ps', 'v', '-p', str(pid)],
                               stdout=subprocess.PIPE
                               ).communicate()[0].split(b'\n')
        try:
            vsz_index = out[0].split().index(b'RSS')
            mem = float(out[1].split()[vsz_index]) / 1024
            if timestamps:
                return(mem, time.time())
            else:
                return mem
        except:
            if timestamps:
                return (-1, time.time())
            else:
                    return -1
    else:
        raise NotImplementedError('The psutil module is required for non-unix '
                                  'platforms')