Python multiprocessing.Value() Examples

def __init__(self, cfg):
        self.__is_use_can_port = cfg['use_can']

        # Data for store
        self.tx_counter_camera_unit = multiprocessing.Value(ctypes.c_int,0)
        self.tx_servo_on_flag = multiprocessing.Value(ctypes.c_bool,False)
        self.tx_target_angle = multiprocessing.Value(ctypes.c_float,0.0)
        self.can_error_count_tx = multiprocessing.Value(ctypes.c_int,0)

        # Initialize process
        self.__m = multiprocessing.Process(target=self.__update, \
                                           args=(cfg['can_name'], \
                                                 cfg['can_bustype'], \
                                                 cfg['can_bitrate'], \
                                                 cfg['can_dbc_path'], \
                                                 cfg['can_tx_interval']))
        # Start process
        self.__m.start()

        return 

def test_init():
    manager = Manager()
    return_dict = manager.dict()

    # make server init before worker
    server_init = Value('i', False)
    serv_p = Process(target=server_func, args=(2, 'test_graph1', server_init))
    serv_p.start()
    while server_init.value == 0:
      time.sleep(1)
    work_p1 = Process(target=check_init_func, args=(0, 'test_graph1', return_dict))
    work_p2 = Process(target=check_init_func, args=(1, 'test_graph1', return_dict))
    work_p1.start()
    work_p2.start()
    serv_p.join()
    work_p1.join()
    work_p2.join()
    for worker_id in return_dict.keys():
        assert return_dict[worker_id] == 0, "worker %d fails" % worker_id 

def test_history_recording_multidim_model(self):
        """Test that history in memory matches with that recorded for test multi-dimensional model."""
        self.param, self.like = multidmodel()
        model = Model(self.like, self.param)
        dream = Dream(model=model, model_name='test_history_recording')
        history_arr = mp.Array('d', [0]*4*dream.total_var_dimension*3)
        n = mp.Value('i', 0)
        nchains = mp.Value('i', 3)
        pydream.Dream_shared_vars.history = history_arr
        pydream.Dream_shared_vars.count = n
        pydream.Dream_shared_vars.nchains = nchains
        test_history = np.array([[[1, 2, 3, 4], [3, 4, 5, 6], [5, 6, 7, 8]], [[7, 8, 9, 10], [9, 12, 18, 20], [11, 14, 18, 8]], [[13, 14, 18, 4], [15, 17, 11, 8], [17, 28, 50, 4]], [[19, 21, 1, 18], [21, 19, 19, 11], [23, 4, 3, 2]]])
        for chainpoint in test_history:
            for point in chainpoint:
                dream.record_history(nseedchains=0, ndimensions=dream.total_var_dimension, q_new=point, len_history=len(history_arr))
        history_arr_np = np.frombuffer(pydream.Dream_shared_vars.history.get_obj())
        history_arr_np_reshaped = history_arr_np.reshape(np.shape(test_history))
        self.assertIs(np.array_equal(history_arr_np_reshaped, test_history), True)
        remove('test_history_recording_DREAM_chain_history.npy')
        remove('test_history_recording_DREAM_chain_adapted_crossoverprob.npy')
        remove('test_history_recording_DREAM_chain_adapted_gammalevelprob.npy') 

def _sync(pack):
    """Simulate a package travelling through the cluster."""
    from django_q.cluster import worker, monitor

    task_queue = Queue()
    result_queue = Queue()
    task = SignedPackage.loads(pack)
    task_queue.put(task)
    task_queue.put("STOP")
    worker(task_queue, result_queue, Value("f", -1))
    result_queue.put("STOP")
    monitor(result_queue)
    task_queue.close()
    task_queue.join_thread()
    result_queue.close()
    result_queue.join_thread()
    return task["id"] 

def test_history_recording_simple_model(self):
        """Test that history in memory matches with that recorded for test one-dimensional model."""
        self.param, self.like = onedmodel()
        model = Model(self.like, self.param)
        step = Dream(model=model, model_name='test_history_recording')
        history_arr = mp.Array('d', [0]*4*step.total_var_dimension)
        n = mp.Value('i', 0)
        nchains = mp.Value('i', 3)
        pydream.Dream_shared_vars.history = history_arr
        pydream.Dream_shared_vars.count = n
        pydream.Dream_shared_vars.nchains = nchains
        test_history = np.array([[1], [3], [5], [7]])
        for chainpoint in test_history:
            for point in chainpoint:
                step.record_history(nseedchains=0, ndimensions=step.total_var_dimension, q_new=point, len_history=len(history_arr))
        history_arr_np = np.frombuffer(pydream.Dream_shared_vars.history.get_obj())
        history_arr_np_reshaped = history_arr_np.reshape(np.shape(test_history))
        self.assertIs(np.array_equal(history_arr_np_reshaped, test_history), True)
        remove('test_history_recording_DREAM_chain_history.npy')
        remove('test_history_recording_DREAM_chain_adapted_crossoverprob.npy')
        remove('test_history_recording_DREAM_chain_adapted_gammalevelprob.npy') 

def test_chain_sampling_multidim_model(self):
        """Test that sampling from DREAM history for multi-dimensional model when the history is known matches with expected possible samples."""
        self.params, self.like = multidmodel()
        model = Model(likelihood=self.like, sampled_parameters=self.params)
        dream = Dream(model=model)
        history_arr = mp.Array('d', [0]*2*dream.total_var_dimension)
        n = mp.Value('i', 0)
        pydream.Dream_shared_vars.history = history_arr
        pydream.Dream_shared_vars.count = n
        chains_added_to_history = []
        for i in range(2):
            start = i*dream.total_var_dimension
            end = start+dream.total_var_dimension
            chain = dream.draw_from_prior(model.sampled_parameters)
            pydream.Dream_shared_vars.history[start:end] = chain
            chains_added_to_history.append(chain)       
        sampled_chains = dream.sample_from_history(nseedchains=2, DEpairs=1, ndimensions=dream.total_var_dimension)
        sampled_chains = np.array(sampled_chains)
        chains_added_to_history = np.array(chains_added_to_history)
        self.assertIs(np.array_equal(chains_added_to_history[chains_added_to_history[:,0].argsort()], sampled_chains[sampled_chains[:,0].argsort()]), True) 

def setUp(self):
        super(TestCNIHealthServer, self).setUp()
        healthy = multiprocessing.Value(c_bool, True)
        self.srv = health.CNIHealthServer(healthy)
        self.srv.application.testing = True
        self.test_client = self.srv.application.test_client() 

def __init__(self, action_dim, observation_dim, **kwargs):
        # shared variable that all processes will see
        self.crash_flag = Value('i', 0)
        self.reset_call_flag = Value('i', 0)

        # Communicator Parameters
        communicator_setups = {'generic1': {'Communicator': MockCommunicator,
                                            'kwargs': {}},
                               'generic2': {'Communicator': MockCommunicator,
                                            'kwargs': {}}
                              }

        self._uniform_array_ = np.frombuffer(Array('d', 3).get_obj(), dtype=np.float64)

        super().__init__(communicator_setups=communicator_setups,
                         action_dim=action_dim,
                         observation_dim=observation_dim,
                         **kwargs) 

def __init__(self, cfg):
        self.__is_use_can_port = cfg['use_can']

        # Data for store
        self.rx_counter_servo_unit = multiprocessing.Value(ctypes.c_int,0)
        self.rx_time_us_diff = multiprocessing.Value(ctypes.c_int,0)
        self.rx_button_y = multiprocessing.Value(ctypes.c_bool,False)
        self.rx_button_g = multiprocessing.Value(ctypes.c_bool,False)
        self.rx_button_r = multiprocessing.Value(ctypes.c_bool,False)
        self.rx_actual_angle = multiprocessing.Value(ctypes.c_float,0.0)
        self.can_error_count_rx = multiprocessing.Value(ctypes.c_int,0)

        # Initialize process
        self.__m = multiprocessing.Process(target=self.__process_can, \
                                           args=(cfg['can_name'], \
                                                 cfg['can_bustype'], \
                                                 cfg['can_bitrate'], \
                                                 cfg['can_dbc_path'], \
                                                 cfg['can_rx_interval']))
        # Start process
        self.__m.start()

        return 

def test_run_async(self):
        counter = mp.Value('l', 0)

        def run_func(process_idx):
            for _ in range(1000):
                with counter.get_lock():
                    counter.value += 1
        async_.run_async(4, run_func)
        self.assertEqual(counter.value, 4000) 

def __init__(self, cfg):
        # Bridge Tx data from subscribe process to publish process
        self.__can_error_count_tx = multiprocessing.Value(ctypes.c_int,0)

        # Initialize process
        self.__m_pub = multiprocessing.Process(target=self.__process_pub, \
                                               args=(cfg,
                                                     cfg['zmq_localhost'], \
                                                     cfg['zmq_port_pubsub_devicerx'], \
                                                     cfg['zmq_topic_devicerx'], \
                                                     cfg['zmq_interval_devicerx']))
        self.__m_sub = multiprocessing.Process(target=self.__process_sub, \
                                               args=(cfg,
                                                     cfg['zmq_localhost'], \
                                                     cfg['zmq_port_pubsub_devicetx'], \
                                                     cfg['zmq_topic_devicetx'], \
                                                     cfg['zmq_interval_devicetx']))
        # Start process
        self.__m_pub.start()
        self.__m_sub.start()

        return 

def test_sync_barrier():
    manager = Manager()
    return_dict = manager.dict()

    # make server init before worker
    server_init = Value('i', 0)
    serv_p = Process(target=server_func, args=(2, 'test_graph4', server_init))
    serv_p.start()
    while server_init.value == 0:
      time.sleep(1)
    work_p1 = Process(target=check_sync_barrier, args=(0, 'test_graph4', return_dict))
    work_p2 = Process(target=check_sync_barrier, args=(1, 'test_graph4', return_dict))
    work_p1.start()
    work_p2.start()
    serv_p.join()
    work_p1.join()
    work_p2.join()
    for worker_id in return_dict.keys():
        assert return_dict[worker_id] == 0, "worker %d fails" % worker_id 

def test_compute():
    manager = Manager()
    return_dict = manager.dict()

    # make server init before worker
    server_init = Value('i', 0)
    serv_p = Process(target=server_func, args=(2, 'test_graph3', server_init))
    serv_p.start()
    while server_init.value == 0:
      time.sleep(1)
    work_p1 = Process(target=check_compute_func, args=(0, 'test_graph3', return_dict))
    work_p2 = Process(target=check_compute_func, args=(1, 'test_graph3', return_dict))
    work_p1.start()
    work_p2.start()
    serv_p.join()
    work_p1.join()
    work_p2.join()
    for worker_id in return_dict.keys():
        assert return_dict[worker_id] == 0, "worker %d fails" % worker_id 

def test_proposal_generation_nosnooker_CR66(self):
        """Test proposal generation without a snooker update with a single or multiple proposed points and a crossover value of 2/3 gives 2/3 of all dimensions changed on average as expected."""
        self.param, self.like = multidmodel()
        model = Model(self.like, self.param)
        step = Dream(model=model)
        history_arr = mp.Array('d', list(range(120)))
        n = mp.Value('i', 0)
        pydream.Dream_shared_vars.history = history_arr
        pydream.Dream_shared_vars.count = n
        step.nseedchains = 20
        q0 = np.array([2, 3, 4, 5])
        dims_kept = 0
        for iteration in range(100000):
            proposed_pt = step.generate_proposal_points(n_proposed_pts=1, q0=q0, CR=.66, DEpairs=1, gamma_level=1, snooker=False)
            if iteration == 1:
                self.assertEqual(len(proposed_pt), 1)
            dims_change_vec = np.squeeze(q0 == proposed_pt)
            for dim in dims_change_vec:
                if dim:
                    dims_kept += 1
        frac_kept = dims_kept/(step.total_var_dimension*100000.0)
        self.assertAlmostEqual(frac_kept, 1-.66, places=1)
        dims_kept = 0
        for iteration in range(10000): 
            proposed_pts = step.generate_proposal_points(n_proposed_pts=5, q0=q0, CR=.66, DEpairs=1, gamma_level=1, snooker=False)
            if iteration == 1:
                self.assertEqual(len(proposed_pts), 5)
            for pt in proposed_pts:
                dims_change_vec = (q0 == pt)
                for dim in dims_change_vec:
                    if dim:
                        dims_kept += 1
        frac_kept = dims_kept/(step.total_var_dimension*10000.0*5)
        self.assertAlmostEqual(frac_kept, 1-.66, places=1) 

def spawn_worker(self):
        self.spawn_process(
            worker, self.task_queue, self.result_queue, Value("f", -1), self.timeout
        ) 

def __init__(self, n=0):
        self.count = multiprocessing.Value("i", n) 

def test_reap_process_group(self):
        """
        Spin up a process that can't be killed by SIGTERM and make sure
        it gets killed anyway.
        """
        parent_setup_done = multiprocessing.Semaphore(0)
        parent_pid = multiprocessing.Value('i', 0)
        child_pid = multiprocessing.Value('i', 0)
        args = [parent_pid, child_pid, parent_setup_done]
        parent = multiprocessing.Process(target=TestReapProcessGroup._parent_of_ignores_sigterm, args=args)
        try:
            parent.start()
            self.assertTrue(parent_setup_done.acquire(timeout=5.0))
            self.assertTrue(psutil.pid_exists(parent_pid.value))
            self.assertTrue(psutil.pid_exists(child_pid.value))

            process_utils.reap_process_group(parent_pid.value, logging.getLogger(), timeout=1)

            self.assertFalse(psutil.pid_exists(parent_pid.value))
            self.assertFalse(psutil.pid_exists(child_pid.value))
        finally:
            try:
                os.kill(parent_pid.value, signal.SIGKILL)  # terminate doesnt work here
                os.kill(child_pid.value, signal.SIGKILL)  # terminate doesnt work here
            except OSError:
                pass 

def __init__(self):
        self._dt = 0.008

        # shared variable that all processes will see
        self.crash_flag = Value('i', 0)

        sensor_args = {'array_len': 1, 'array_type': 'd', 'np_array_type': 'd'}
        actuator_args = {'array_len': 1, 'array_type': 'd', 'np_array_type': 'd'}
        super().__init__(use_sensor=True, use_actuator=True, sensor_args=sensor_args, actuator_args=actuator_args) 

def testLock(self):
        def spawn_write_process(buffer):
            buffer.write([1, 2])

        def spawn_read_process(buffer, hold_lock):
            def mock_read(*args, **kwargs):
                while hold_lock.value == 1:
                    time.sleep(0.01)
            buffer._read = mock_read
            buffer.read_update()

        buffer = SharedBuffer(array_len=2, array_type='d', np_array_type='d', buffer_len=3)
        hold_lock = Value('i', 1)
        # spawn a read process that will just hold onto the read to simulate large read
        read_process = Process(target=spawn_read_process, args=(buffer, hold_lock))
        read_process.start()
        time.sleep(0.1)
        # spawn a write process that should get stuck unable to get lock to write
        write_process = Process(target=spawn_write_process, args=(buffer,))
        write_process.start()
        time.sleep(0.5)

        self.assertTrue(write_process.is_alive())
        hold_lock.value = 0
        time.sleep(0.5)
        self.assertFalse(write_process.is_alive()) 

def add_scalar(self, tag, scalar_value, walltime=None):
        """Add scalar data to summary.

        Args:
            tag (string): Data identifier
            scalar_value (float): Value to save
            walltime (float): Optional override default walltime (time.time()) of event

        """
        with self.lock:
            if tag in self.scalar_tag_to_step:
                self.scalar_tag_to_step[tag] += 1
            else:
                self.scalar_tag_to_step[tag] = 0

            self.smw.add_scalar(tag, scalar_value, self.scalar_tag_to_step[tag], walltime)

    # def add_histogram(self, tag, values, bins='tensorflow', walltime=None, max_bins=None):
    #     """Add histogram to summary.

    #     Args:
    #         tag (string): Data identifier
    #         values (torch.Tensor, numpy.array): Values to build histogram
    #         bins (string): One of {'tensorflow','auto', 'fd', ...}.
    #           This determines how the bins are made. You can find
    #           other options in: https://docs.scipy.org/doc/numpy/reference/generated/numpy.histogram.html
    #         walltime (float): Optional override default walltime (time.time()) of event

    #     """
    #     with self.new_tag_mutex.get_lock():
    #         if tag in self.histogram_tag_to_step:
    #             self.histogram_tag_to_step[tag] += 1
    #         else:
    #             self.histogram_tag_to_step[tag] = 0
    #         self.smw.add_histogram(tag,
    #                                values,
    #                                self.histogram_tag_to_step[tag],
    #                                bins=bins,
    #                                walltime=walltime,
    #                                max_bins=max_bins) 

def __init__(self, cfg):
        self.__is_use_gps_port = cfg['use_gps']

        # Data for store
        self.latitude = multiprocessing.Value(ctypes.c_float,cfg['latitude_init'])
        self.longitude = multiprocessing.Value(ctypes.c_float,cfg['longitude_init'])

        # Initialize process
        self.__m = multiprocessing.Process(target=self.__process_gps, \
                                           args=(cfg['gps_interval'],))
        # Start process
        self.__m.start()
        return 

def test_proposal_generation_nosnooker_CR33(self):
        """Test proposal generation without a snooker update with a single or multiple proposed points and a crossover value of .33 gives 1/3 of all dimensions changed on average as expected."""
        self.param, self.like = multidmodel()
        model = Model(self.like, self.param)
        step = Dream(model=model)
        history_arr = mp.Array('d', list(range(120)))
        n = mp.Value('i', 0)
        pydream.Dream_shared_vars.history = history_arr
        pydream.Dream_shared_vars.count = n
        step.nseedchains = 20
        q0 = np.array([2, 3, 4, 5])
        dims_kept = 0
        for iteration in range(100000):
            proposed_pt = step.generate_proposal_points(n_proposed_pts=1, q0=q0, CR=.33, DEpairs=1, gamma_level=1, snooker=False)
            if iteration == 1:
                self.assertEqual(len(proposed_pt), 1)
            dims_change_vec = np.squeeze(q0 == proposed_pt)
            for dim in dims_change_vec:
                if dim:
                    dims_kept += 1
        frac_kept = dims_kept/(step.total_var_dimension*100000.0)
        self.assertAlmostEqual(frac_kept, 1-.33, places=1)
        dims_kept = 0
        for iteration in range(10000): 
            proposed_pts = step.generate_proposal_points(n_proposed_pts=5, q0=q0, CR=.33, DEpairs=1, gamma_level=1, snooker=False)
            if iteration == 1:
                self.assertEqual(len(proposed_pts), 5)
            for pt in proposed_pts:
                dims_change_vec = (q0 == pt)
                for dim in dims_change_vec:
                    if dim:
                        dims_kept += 1
        frac_kept = dims_kept/(step.total_var_dimension*10000.0*5)
        self.assertAlmostEqual(frac_kept, 1-.33, places=1) 

def test_proposal_generation_nosnooker_CR1(self):
        """Test proposal generation without a snooker update with a single or multiple proposed points and a crossover value of 1 gives all dimensions changed on average as expected."""
        self.param, self.like = multidmodel()
        model = Model(self.like, self.param)
        step = Dream(model=model)
        history_arr = mp.Array('d', list(range(120)))
        n = mp.Value('i', 0)
        pydream.Dream_shared_vars.history = history_arr
        pydream.Dream_shared_vars.count = n
        step.nseedchains = 20
        q0 = np.array([2, 3, 4, 5])
        dims_kept = 0
        for iteration in range(10000):
            proposed_pt = step.generate_proposal_points(n_proposed_pts=1, q0=q0, CR=1, DEpairs=1, gamma_level=1, snooker=False)
            if iteration == 1:
                self.assertEqual(len(proposed_pt), 1)
            dims_change_vec = np.squeeze(q0 == proposed_pt)
            for dim in dims_change_vec:
                if dim:
                    dims_kept += 1
        frac_kept = dims_kept/(step.total_var_dimension*10000.0)
        self.assertAlmostEqual(frac_kept, 0, places=1)
        dims_kept = 0
        for iteration in range(1000):
            proposed_pts = step.generate_proposal_points(n_proposed_pts=5, q0=q0, CR=1, DEpairs=1, gamma_level=1, snooker=False)
            if iteration == 1:
                self.assertEqual(len(proposed_pts), 5)
            for pt in proposed_pts:
                dims_change_vec = (q0 == pt)
                for dim in dims_change_vec:
                    if dim:
                        dims_kept += 1
        frac_kept = dims_kept/(step.total_var_dimension*1000.0*5)
        self.assertAlmostEqual(frac_kept, 0, places=1) 

def test_chain_sampling_simple_model(self):
        """Test that sampling from DREAM history for one dimensional model when the history is known matches with expected possible samples."""
        self.param, self.like = onedmodel()
        model = Model(likelihood=self.like, sampled_parameters=self.param)
        dream = Dream(model=model)
        history_arr = mp.Array('d', [0]*2*dream.total_var_dimension)
        n = mp.Value('i', 0)
        pydream.Dream_shared_vars.history = history_arr
        pydream.Dream_shared_vars.count = n
        chains_added_to_history = []
        for i in range(2):
            start = i*dream.total_var_dimension
            end = start+dream.total_var_dimension
            chain = dream.draw_from_prior(dream.variables)
            pydream.Dream_shared_vars.history[start:end] = chain
            chains_added_to_history.append(chain)
        sampled_chains = dream.sample_from_history(nseedchains=2, DEpairs=1, ndimensions=dream.total_var_dimension)
        sampled_chains = np.array(sampled_chains)
        chains_added_to_history = np.array(chains_added_to_history)
        self.assertIs(np.array_equal(chains_added_to_history[chains_added_to_history[:,0].argsort()], sampled_chains[sampled_chains[:,0].argsort()]), True) 

def stop_stream(self):
        logger.debug("CameraStream: Stop stream " + self.name)
        if not self.imageurl:
            #Only stop something if this is not an imageurl, for imageurl nothing has to be stopped
            #Stopworker shared value will stop the while loop in the worker, so the worker will run to end. No need to explicitely terminate the worker
            self.stopworker.value= True
            logger.debug("CameraStream: MAIN Value of stopworker for " + self.name + " is " + str(self.stopworker.value))

            #Wait for the worker to be terminated before continuing https://github.com/SvenVD/rpisurv/issues/84
            logger.debug("CameraStream: Executing join for stream " + self.name)
            self.worker.join() 

def start_stream(self, coordinates, pygamescreen, cached):
        self.coordinates=coordinates
        self.pygamescreen=pygamescreen
        logger.debug("CameraStream: Start stream " + self.name)


        if not self.imageurl:
            #Start worker process and dbus connnection only if it isn't running already
            if self.worker and self.worker.is_alive():
                logger.debug("CameraStream: Worker from " + self.name + " is still alive not starting new worker")
            else:
                self.stopworker = multiprocessing.Value('b', False)
                self.worker = multiprocessing.Process(target=worker.worker, args=(self.name,self.url,self.omxplayer_extra_options,self.coordinates,self.stopworker,self.aidx))
                self.worker.daemon = True
                self.worker.start()
                if platform.system() == "Linux":
                    #dbus connection can only be setup once the stream is correctly started
                    self._setup_dbus_connection()

            #Update position
            self.set_videopos(self.coordinates)

        if not cached:
            logger.debug("CameraStream: This stream " + self.name + " is not running in cache")
            self.show_status()

            if self.imageurl:
                self.refresh_image_from_url()


        else:
            logger.debug("CameraStream: This stream " + self.name + " is running in cache") 

def __init__(self):
        self.val = Value('i', 0) 

def __init__(self):
        self.val = Value('i', 0) 

def __init__(self, num_processes, max_queue_size, fn):
        """

        Parameters
        ----------
        num_processes: int
            Number of processes to spawn
        max_queue_size: int
            Maximum samples in the queue before processes wait
        fn: function
            function that generates samples, executed on separate processes.
        """
        self.queue = mp.Queue(maxsize=int(max_queue_size))
        self.alive = mp.Value(c_bool, False, lock=False)
        self.num_proc = num_processes
        self.proc = list()
        self.fn = fn 

def __init__(self, manager, base_value=0):
        self._lock, self._value = manager.Lock(), manager.Value('i', base_value)