Python multiprocessing.RLock() Examples

def _edit_lock(self):
        # type: () -> ()
        if self.__edit_lock:
            return self.__edit_lock
        if not PROC_MASTER_ID_ENV_VAR.get() or len(PROC_MASTER_ID_ENV_VAR.get().split(':')) < 2:
            self.__edit_lock = RLock()
        elif PROC_MASTER_ID_ENV_VAR.get().split(':')[1] == str(self.id):
            # remove previous file lock instance, just in case.
            filename = os.path.join(gettempdir(), 'trains_{}.lock'.format(self.id))
            # noinspection PyBroadException
            try:
                os.unlink(filename)
            except Exception:
                pass
            # create a new file based lock
            self.__edit_lock = FileRLock(filename=filename)
        else:
            self.__edit_lock = RLock()
        return self.__edit_lock 

def initialize_unittest_logging():
    # ACE is multi-process multi-threaded
    # so we use this special logging mechanism to keep a central repository of the log events generated
    # that the original process can access

    global test_log_manager
    global test_log_sync
    global test_log_messages
    global memory_log_handler

    test_log_manager = Manager()
    atexit.register(_atexit_callback)
    test_log_sync = RLock()
    test_log_messages = test_log_manager.list()

    log_format = logging.Formatter(datefmt='%(asctime)s')

    memory_log_handler = MemoryLogHandler()
    memory_log_handler.setLevel(logging.DEBUG)
    memory_log_handler.setFormatter(log_format)
    logging.getLogger().addHandler(memory_log_handler) 

def __init__(self, auth_handler, update_interval, accept_lang,
                 country_code, excluded_cars, save_car_details,
                 pin, save_path):

        self.__lock = RLock()
        self.accept_lang = accept_lang
        self.country_code = country_code
        self.auth_handler = auth_handler
        self.cars = []
        self.update_interval = update_interval
        self.excluded_cars = excluded_cars
        self.is_valid_session = False
        self.last_update_time = 0
        self.save_car_details = save_car_details
        self.save_path = save_path
        self.pin = pin
        self.region = "-an" if country_code == "US" else ""

        self.session = requests.session()
        # self.session.proxies.update(HTTP_PROXY)

        _LOGGER.debug("Controller init complete. Start _get_cars")
        self._get_cars() 

def __init__(self, milliseconds, bucket_numbers, _time=None):
        self.time = _time or ActualTime()  # Create a instance of time here
        self.milliseconds = milliseconds
        self.buckets = BucketCircular(bucket_numbers)
        self.bucket_numbers = bucket_numbers
        self.cumulative = CumulativeSum()
        self._new_bucket_lock = RLock()

        if self.milliseconds % self.bucket_numbers != 0:
            raise Exception('The milliseconds must divide equally into '
                            'bucket_numbers. For example 1000/10 is ok, '
                            '1000/11 is not.') 

def Array(typecode_or_type, size_or_initializer, **kwds):
    '''
    Return a synchronization wrapper for a RawArray
    '''
    lock = kwds.pop('lock', None)
    if kwds:
        raise ValueError('unrecognized keyword argument(s): %s' % kwds.keys())
    obj = RawArray(typecode_or_type, size_or_initializer)
    if lock is False:
        return obj
    if lock in (True, None):
        lock = RLock()
    if not hasattr(lock, 'acquire'):
        raise AttributeError("'%r' has no method 'acquire'" % lock)
    return synchronized(obj, lock) 

def __init__(self, obj, lock=None):
        self._obj = obj
        self._lock = lock or RLock()
        self.acquire = self._lock.acquire
        self.release = self._lock.release 

def _edit_lock(self, value):
        # type: (RLock) -> ()
        self.__edit_lock = value 

def __init__(self, task):
        self._task = task
        # notice the double link, this important since the Artifact
        # dictionary needs to signal the Artifacts base on changes
        self._artifacts_container = self._ProxyDictWrite(self)
        self._last_artifacts_upload = {}
        self._unregister_request = set()
        self._thread = None
        self._flush_event = Event()
        self._exit_flag = False
        self._summary = ''
        self._temp_folder = []
        self._task_artifact_list = []
        self._task_edit_lock = RLock()
        self._storage_prefix = None 

def __init__(
            self, filename, mode='a', timeout=DEFAULT_TIMEOUT,
            check_interval=DEFAULT_CHECK_INTERVAL, fail_when_locked=False,
            flags=LOCK_METHOD):
        super(RLock, self).__init__(filename, mode, timeout, check_interval,
                                    fail_when_locked, flags)
        self._acquire_count = 0
        self._lock = ProcessRLock()
        self._pid = os.getpid() 

def test_rlock(self):
        lock = self.RLock()
        self.assertEqual(lock.acquire(), True)
        self.assertEqual(lock.acquire(), True)
        self.assertEqual(lock.acquire(), True)
        self.assertEqual(lock.release(), None)
        self.assertEqual(lock.release(), None)
        self.assertEqual(lock.release(), None)
        self.assertRaises((AssertionError, RuntimeError), lock.release) 

def setUp(self):
        self.step = PipelineStepDownloadAccessions(
            name="rapsearch2_out",
            input_files=[
                [
                    "rapsearch2.m8",
                    "rapsearch2.deduped.m8",
                    "rapsearch2.hitsummary.tab",
                    "rapsearch2_counts.json"
                ]
            ],
            output_files=["assembly/nr.refseq.fasta"],
            output_dir_local=tempfile.TemporaryDirectory().name,
            output_dir_s3="s3://dummy_bucket",
            ref_dir_local=tempfile.TemporaryDirectory().name,
            additional_files={
                "lineage_db": "s3://idseq-database/taxonomy/2018-12-01/taxid-lineages.sqlite3",
                "loc_db": "s3://idseq-database/alignment_data/2018-12-01/nr_loc.sqlite3"
            },
            additional_attributes={
                "db":  "s3://idseq-database/alignment_data/2018-12-01/nr",
                "db_type": "nr"
            },
            step_status_lock=RLock(),
            step_status_local="./dummy_status.json"
        ) 

def __init__(self, lock_name, lock=multiprocessing.RLock(), debug=True):
        self._lock = lock
        self._lock_name = lock_name
        self.debug = debug 

def refreshed_credentials(credentials_mutex=multiprocessing.RLock(), credentials_cache={}):  # pylint: disable=dangerous-default-value
    with credentials_mutex:
        if credentials_cache.get("expiration_time", 0) < time.time() + 5 * 60:
            try:
                credentials_cache["vars"] = _get_credentials()
            except:
                log.write("ERROR:  Failed to refresh credentials with boto, even after retries.  Subcommands will have to do it themselves.")
                log.write(traceback.format_exc())
                return {}
            credentials_cache["expiration_time"] = time.time() + 15 * 60  # this is the default for most accounts
    return credentials_cache["vars"] 

def make_space(done={}, mutex=TraceLock("make_space", multiprocessing.RLock())):  # pylint: disable=dangerous-default-value
    with mutex:
        if not done:
            try:
                really_make_space()
            except:
                log.write("Error making space.  Please attend to this before instance storage fills up.")
                log.write(traceback.format_exc())
            done['time'] = time.time() 

def Value(typecode_or_type, *args, **kwds):
    '''
    Return a synchronization wrapper for a Value
    '''
    lock = kwds.pop('lock', None)
    if kwds:
        raise ValueError('unrecognized keyword argument(s): %s' % kwds.keys())
    obj = RawValue(typecode_or_type, *args)
    if lock is False:
        return obj
    if lock in (True, None):
        lock = RLock()
    if not hasattr(lock, 'acquire'):
        raise AttributeError("'%r' has no method 'acquire'" % lock)
    return synchronized(obj, lock) 

def __init__(self, _time, milliseconds, bucket_numbers,
                 bucket_data_length, enabled):
        self.time = _time
        self.milliseconds = milliseconds
        self.buckets = BucketCircular(bucket_numbers)
        self.bucket_numbers = bucket_numbers
        self.bucket_data_length = bucket_data_length
        self.enabled = enabled
        self.snapshot = PercentileSnapshot(0)
        self._new_bucket_lock = RLock() 

def __init__(self, data_length):
        self.data_length = data_length
        self.list = Array('i', self.data_length, lock=RLock())
        # TODO: Change this to use a generator
        self.index = itertools.count()
        self.number = 0 

def test_threading():
    """Test multiprocess/thread-realted features"""
    from multiprocessing import RLock
    try:
        mp_lock = RLock()
    except OSError:
        pass
    else:
        tqdm.set_lock(mp_lock)
    # TODO: test interleaved output #445 

def Value(typecode_or_type, *args, **kwds):
    '''
    Return a synchronization wrapper for a Value
    '''
    lock = kwds.pop('lock', None)
    if kwds:
        raise ValueError('unrecognized keyword argument(s): %s' % kwds.keys())
    obj = RawValue(typecode_or_type, *args)
    if lock is False:
        return obj
    if lock in (True, None):
        lock = RLock()
    if not hasattr(lock, 'acquire'):
        raise AttributeError("'%r' has no method 'acquire'" % lock)
    return synchronized(obj, lock) 

def Array(typecode_or_type, size_or_initializer, **kwds):
    '''
    Return a synchronization wrapper for a RawArray
    '''
    lock = kwds.pop('lock', None)
    if kwds:
        raise ValueError('unrecognized keyword argument(s): %s' % kwds.keys())
    obj = RawArray(typecode_or_type, size_or_initializer)
    if lock is False:
        return obj
    if lock in (True, None):
        lock = RLock()
    if not hasattr(lock, 'acquire'):
        raise AttributeError("'%r' has no method 'acquire'" % lock)
    return synchronized(obj, lock) 

def __init__(self, obj, lock=None):
        self._obj = obj
        self._lock = lock or RLock()
        self.acquire = self._lock.acquire
        self.release = self._lock.release 

def Value(typecode_or_type, *args, **kwds):
    '''
    Return a synchronization wrapper for a Value
    '''
    lock = kwds.pop('lock', None)
    if kwds:
        raise ValueError('unrecognized keyword argument(s): %s' % kwds.keys())
    obj = RawValue(typecode_or_type, *args)
    if lock is False:
        return obj
    if lock in (True, None):
        lock = RLock()
    if not hasattr(lock, 'acquire'):
        raise AttributeError("'%r' has no method 'acquire'" % lock)
    return synchronized(obj, lock) 

def Array(typecode_or_type, size_or_initializer, **kwds):
    '''
    Return a synchronization wrapper for a RawArray
    '''
    lock = kwds.pop('lock', None)
    if kwds:
        raise ValueError('unrecognized keyword argument(s): %s' % kwds.keys())
    obj = RawArray(typecode_or_type, size_or_initializer)
    if lock is False:
        return obj
    if lock in (True, None):
        lock = RLock()
    if not hasattr(lock, 'acquire'):
        raise AttributeError("'%r' has no method 'acquire'" % lock)
    return synchronized(obj, lock) 

def __init__(self, obj, lock=None):
        self._obj = obj
        self._lock = lock or RLock()
        self.acquire = self._lock.acquire
        self.release = self._lock.release 

def __init__(self,
                 index_file: str,
                 identifier: int,
                 params_file: str,
                 lock: mp.RLock,
                 conf: dict,
                 maxobjects: int,
                 logging_queue=None,
                 log_level="DEBUG",
                 sql_level="DEBUG",
                 matrix_name=None, qmult=3, tmult=1, **kwargs):

        super().__init__()
        self.matrix_name = matrix_name
        self.qmult, self.tmult = qmult, tmult
        self.identifier = identifier
        self.log_level, self.sql_level = log_level, sql_level
        self.lock = lock
        self.maxobjects = maxobjects
        self.conf = conf
        self.params_file, self.index_file = params_file, index_file
        if logging_queue is None:
            self.logger = create_null_logger("preparer-{}".format(self.identifier))  # create_null_logger
            self.logging_queue = None
        else:
            self.logging_queue = logging_queue 

def __init__(self, directory, **kwargs):
        """
        :param directory: root dir for storage
        """
        super(FileSystemResourceAPI, self).__init__(**kwargs)
        self.directory = directory
        self.base_path = os.path.join(self.directory, self.name)
        self._join_path = partial(os.path.join, self.base_path)
        self._lock = RLock() 

def Value(typecode_or_type, *args, **kwds):
    '''
    Return a synchronization wrapper for a Value
    '''
    lock = kwds.pop('lock', None)
    if kwds:
        raise ValueError('unrecognized keyword argument(s): %s' % kwds.keys())
    obj = RawValue(typecode_or_type, *args)
    if lock is False:
        return obj
    if lock in (True, None):
        lock = RLock()
    if not hasattr(lock, 'acquire'):
        raise AttributeError("'%r' has no method 'acquire'" % lock)
    return synchronized(obj, lock) 

def Array(typecode_or_type, size_or_initializer, **kwds):
    '''
    Return a synchronization wrapper for a RawArray
    '''
    lock = kwds.pop('lock', None)
    if kwds:
        raise ValueError('unrecognized keyword argument(s): %s' % kwds.keys())
    obj = RawArray(typecode_or_type, size_or_initializer)
    if lock is False:
        return obj
    if lock in (True, None):
        lock = RLock()
    if not hasattr(lock, 'acquire'):
        raise AttributeError("'%r' has no method 'acquire'" % lock)
    return synchronized(obj, lock) 

def __init__(self, obj, lock=None):
        self._obj = obj
        self._lock = lock or RLock()
        self.acquire = self._lock.acquire
        self.release = self._lock.release 

def _start_io_tracker():
    import multiprocessing

    global _track_io
    global _io_tracker_manager
    global _io_tracker_sync
    global _write_count
    global _read_count

    _io_tracker_manager = multiprocessing.Manager()
    _io_tracker_sync = multiprocessing.RLock()
    _write_count = _io_tracker_manager.Value('I', 0, lock=False)
    _read_count = _io_tracker_manager.Value('I', 0, lock=False)
    _track_io = True