Python threading.local() Examples

def __init__(self, *args, **kwargs):
        class LocalContexts(threading.local):
            def __init__(self):
                super(LocalContexts, self).__init__()
                self._contexts = []

            def append(self, item):
                self._contexts.append(item)

            def pop(self):
                return self._contexts.pop()

        super(ThreadSafeStackContext, self).__init__(*args, **kwargs)

        if hasattr(self, 'contexts'):
            # only patch if context exists
            self.contexts = LocalContexts() 

def _put(self, lpath, rpath, recursive=False, **kwargs):
        """copy local files to remote
        """
        from .implementations.local import make_path_posix, LocalFileSystem

        rpath = self._strip_protocol(rpath)
        if isinstance(lpath, str):
            lpath = make_path_posix(lpath)
        fs = LocalFileSystem()
        lpaths = fs.expand_path(lpath, recursive=recursive)
        rpaths = other_paths(lpaths, rpath)

        await asyncio.gather(
            *[
                self._put_file(lpath, rpath, **kwargs)
                for lpath, rpath in zip(lpaths, rpaths)
            ]
        ) 

def __init__(self, threads=4, *args, **kwargs):
        self.local = threading.local()

        super(ThreadBaseScheduler, self).__init__(*args, **kwargs)

        if isinstance(self.taskdb, SQLiteMixin):
            self.threads = 1
        else:
            self.threads = threads

        self._taskdb = self.taskdb
        self._projectdb = self.projectdb
        self._resultdb = self.resultdb

        self.thread_objs = []
        self.thread_queues = []
        self._start_threads()
        assert len(self.thread_queues) > 0 

def set_trace(self, frame=None):
        """Remember starting frame.

        This is used with pytest, which does not use pdb.set_trace().
        """
        if getattr(local, "_pdbpp_completing", False):
            # Handle set_trace being called during completion, e.g. with
            # fancycompleter's attr_matches.
            return
        if self.disabled:
            return

        if frame is None:
            frame = sys._getframe().f_back
        self._via_set_trace_frame = frame
        self._stopped_for_set_trace = False

        self.start_filename = frame.f_code.co_filename
        self.start_lineno = frame.f_lineno

        return super(Pdb, self).set_trace(frame) 

def localtime(value, timezone=None):
    """
    Converts an aware datetime.datetime to local time.

    Local time is defined by the current time zone, unless another time zone
    is specified.
    """
    if timezone is None:
        timezone = get_current_timezone()
    # If `value` is naive, astimezone() will raise a ValueError,
    # so we don't need to perform a redundant check.
    value = value.astimezone(timezone)
    if hasattr(timezone, 'normalize'):
        # This method is available for pytz time zones.
        value = timezone.normalize(value)
    return value 

def template_localtime(value, use_tz=None):
    """
    Checks if value is a datetime and converts it to local time if necessary.

    If use_tz is provided and is not None, that will force the value to
    be converted (or not), overriding the value of settings.USE_TZ.

    This function is designed for use by the template engine.
    """
    should_convert = (isinstance(value, datetime)
        and (settings.USE_TZ if use_tz is None else use_tz)
        and not is_naive(value)
        and getattr(value, 'convert_to_local_time', True))
    return localtime(value) if should_convert else value


# Utilities 

def open_rpc_interface(self):
		try:
			self.local.rpc_interface.close()
		except Exception:   # pylint: disable=W0703
			pass

		for x in range(100):
			try:
				self.local.rpc_interface = common.get_rpyc.RemoteJobInterface("RawMirror")
				return

			except (TypeError, KeyError, socket.timeout, ConnectionRefusedError):
				for line in traceback.format_exc().split("\n"):
					self.log.error(line)
				if x > 90:
					raise RuntimeError("Could not establish connection to RPC remote!")

		raise RuntimeError("Could not establish connection to RPC remote!") 

def __exit__(self, type, value, traceback):
        try:
            self.exit(type, value, traceback)
        finally:
            final_contexts = _state.contexts
            _state.contexts = self.old_contexts

            # Generator coroutines and with-statements with non-local
            # effects interact badly.  Check here for signs of
            # the stack getting out of sync.
            # Note that this check comes after restoring _state.context
            # so that if it fails things are left in a (relatively)
            # consistent state.
            if final_contexts is not self.new_contexts:
                raise StackContextInconsistentError(
                    'stack_context inconsistency (may be caused by yield '
                    'within a "with StackContext" block)')

            # Break up a reference to itself to allow for faster GC on CPython.
            self.new_contexts = None 

def __exit__(self, type, value, traceback):
        try:
            self.exit(type, value, traceback)
        finally:
            final_contexts = _state.contexts
            _state.contexts = self.old_contexts

            # Generator coroutines and with-statements with non-local
            # effects interact badly.  Check here for signs of
            # the stack getting out of sync.
            # Note that this check comes after restoring _state.context
            # so that if it fails things are left in a (relatively)
            # consistent state.
            if final_contexts is not self.new_contexts:
                raise StackContextInconsistentError(
                    'stack_context inconsistency (may be caused by yield '
                    'within a "with StackContext" block)')

            # Break up a reference to itself to allow for faster GC on CPython.
            self.new_contexts = None 

def _init(self):
        endpoint, kwargs = self._endpoint, self._kws
        if endpoint is not None:
            if 'http' in endpoint:
                # connect to web
                from .web.session import Session as WebSession

                self._sess = WebSession(endpoint, **kwargs)
            else:
                # connect to local cluster

                self._sess = ClusterSession(endpoint, **kwargs)
        else:
            try:
                endpoint = os.environ['MARS_SCHEDULER_ADDRESS']
                session_id = os.environ.get('MARS_SESSION_ID', None)
                self._sess = ClusterSession(endpoint, session_id=session_id)
            except KeyError:
                self._sess = LocalSession(**kwargs) 

def emit(self, record: logging.LogRecord):
        """Print the log record formatted as JSON to stdout."""
        created = datetime.datetime.fromtimestamp(record.created, timezone)
        obj = {
            "level": record.levelname.lower(),
            "msg": record.msg % record.args,
            "source": "%s:%d" % (record.filename, record.lineno),
            "time": format_datetime(created),
            "thread": reduce_thread_id(record.thread),
        }
        if record.exc_info is not None:
            obj["error"] = traceback.format_exception(*record.exc_info)[1:]
        try:
            obj["context"] = self.local.context
        except AttributeError:
            pass
        json.dump(obj, sys.stdout, sort_keys=True)
        sys.stdout.write("\n")
        sys.stdout.flush() 

def disable_job(self, rid, joburl):
		self.log.warning("Disabling job for url: '%s'", joburl)
		cursor = self.local.db_interface.cursor()
		cursor.execute("""UPDATE raw_web_pages SET state = %s WHERE raw_web_pages.id = %s AND raw_web_pages.url = %s;""", ('disabled', rid, joburl))
		self.local.db_interface.commit() 

def queue_drainer_proc(self):

		common.process.name_process("raw job filler worker")
		self.open_rpc_interface()

		try:
			signal.signal(signal.SIGINT, signal.SIG_IGN)
		except ValueError:
			self.log.warning("Cannot configure job fetcher task to ignore SIGINT. May be an issue.")

		self.log.info("Job queue fetcher starting.")

		msg_loop = 0
		while self.run_flag.value == 1:
			# print("Drainer looping@")
			self.process_responses()

			msg_loop += 1
			time.sleep(0.2)
			if msg_loop > 25:

				with self.count_lock:
					self.log.info("Job queue filler process. Current job queue size: %s (out: %s, in: %s). Runstate: %s", self.active_jobs, self.jobs_out, self.jobs_in, self.run_flag.value==1)
				msg_loop = 0
				with self.limiter_lock:
					self.ratelimiter.job_reduce()


		self.log.info("Job queue fetcher saw exit flag. Halting.")
		self.local.rpc_interface.close()

		# Consume the remaining items in the output queue so it shuts down cleanly.
		try:
			while 1:
				self.normal_out_queue.get_nowait()
		except queue.Empty:
			pass

		with self.count_lock:
			self.log.info("Job queue filler process. Current job queue size: %s. ", self.active_jobs)
		self.log.info("Job queue fetcher halted.") 

def __del__(self):
        if self.ptr and lgeos:
            lgeos.finishGEOS_r(self.ptr)


# Defining a thread-local object and creating an instance
# to hold a reference to GEOSContextHandle for this thread. 

def get_local_address(self):
        """
        Get local address

        :return: local address
        """
        raise NotImplementedError 

def running_mode(self):
        """
        Get the running mode, could be local, local_cluster or distributed.
        """
        raise NotImplementedError 

def __init__(self, enabled=True, trace=False):
    self._times = defaultdict(Stat)
    self._local = threading.local()
    self.enabled = enabled
    self.trace = trace 

def __init__(self, username, password):
        self.username = username
        self.password = password
        # Keep state in per-thread local storage
        self._thread_local = threading.local() 

def do_interact(self, arg):
        """
        interact

        Start an interative interpreter whose global namespace
        contains all the names found in the current scope.
        """
        ns = self.curframe.f_globals.copy()
        ns.update(self.curframe_locals)
        code.interact("*interactive*", local=ns) 

def get_flow(self):
    """A thread local Flow object.

    Returns:
      A credentials.Flow object, or None if the flow hasn't been set in this
      thread yet, which happens in _create_flow() since Flows are created
      lazily.
    """
    return getattr(self._tls, 'flow', None) 

def get_credentials(self):
    """A thread local Credentials object.

    Returns:
      A client.Credentials object, or None if credentials hasn't been set in
      this thread yet, which may happen when calling has_credentials inside
      oauth_aware.
    """
    return getattr(self._tls, 'credentials', None) 

def _get(self, rpath, lpath, recursive=False, **kwargs):
        """Copy file(s) to local.
        """
        from fsspec.implementations.local import make_path_posix

        rpath = self._strip_protocol(rpath)
        lpath = make_path_posix(lpath)
        rpaths = await self._expand_path(rpath, recursive=recursive)
        lpaths = other_paths(rpaths, lpath)
        await asyncio.gather(
            *[
                self._get_file(rpath, lpath, **kwargs)
                for lpath, rpath in zip(lpaths, rpaths)
            ]
        ) 

def __init__(self, username, password):
        self.username = username
        self.password = password
        # Keep state in per-thread local storage
        self._thread_local = threading.local() 

def __init__(self):
        if threading.current_thread().__class__.__name__ == '_MainThread':
            self._mapstack = _GlobalMapStack({
                k: v.sanitize(v.bottom_value) for (k, v) in _OPTIONS.items()
            })
        else:
            self._mapstack = _GlobalMapStack()
        threading.local.__init__(self) 

def __init__(self, username, password):
        self.username = username
        self.password = password
        # Keep state in per-thread local storage
        self._thread_local = threading.local() 

def __init__(self, username, password):
        self.username = username
        self.password = password
        # Keep state in per-thread local storage
        self._thread_local = threading.local() 

def InterruptibleQueueJoin(queue,
                           thread_local,
                           thread_pool,
                           queue_join_thread_factory=QueueJoinThread,
                           check_workers=True):
  """Repeatedly joins the given ReQueue or Queue.Queue with short timeout.

  Between each timeout on the join, worker threads are checked.

  Args:
    queue: A Queue.Queue or ReQueue instance.
    thread_local: A threading.local instance which indicates interrupts.
    thread_pool: An AdaptiveThreadPool instance.
    queue_join_thread_factory: Used for dependency injection.
    check_workers: Whether to interrupt the join on worker death.

  Returns:
    True unless the queue join is interrupted by SIGINT or worker death.
  """
  thread = queue_join_thread_factory(queue)
  thread.start()
  while True:
    thread.join(timeout=.5)
    if not thread.isAlive():
      return True
    if thread_local.shut_down:
      logger.debug('Queue join interrupted')
      return False
    if check_workers:
      for worker_thread in thread_pool.Threads():
        if not worker_thread.isAlive():
          return False 

def _thread_init(cls):
    """Ensure thread local is initialized."""
    if not hasattr(cls._local, '_in_order_futures'):
      cls._local._in_order_futures = set()
      cls._local._activated = False


################################################################################ 

def _thread_init(cls):
    """Ensure thread local is initialized."""
    if not hasattr(cls._local, '_after_all_futures'):
      cls._local._after_all_futures = [] 

def get_chunk_results(self, chunk_keys: List[str]) -> List:
        # As the context is actually holding the data,
        # so for the local context, we just fetch data from itself
        return [self[chunk_key] for chunk_key in chunk_keys]