Python Queue.Queue() Examples

def run(self):

        while True:

            try:
                func, server, data = self.queue.get(timeout=1)
            except Queue.Empty:
                self.monitor.workers_threads.remove(self)

                break

            try:
                LOG.info("-->[ q:monitor/%s ]", data['MonitorMethod'])
                func(server, data)
            except Exception as error:
                LOG.exception(error)

            self.queue.task_done()

            if window('emby_should_stop.bool'):
                break 

def __init__(self, batch_size, input_length, nthreads=6, web_viz=False):
        super(RLDataIter, self).__init__()
        self.batch_size = batch_size
        self.input_length = input_length
        self.env = [self.make_env() for _ in range(batch_size)]
        self.act_dim = self.env[0].action_space.n

        self.state_ = None

        self.reset()

        self.provide_data = [mx.io.DataDesc('data', self.state_.shape, np.uint8)]

        self.web_viz = web_viz
        if web_viz:
            self.queue = queue.Queue()
            self.thread = Thread(target=make_web, args=(self.queue,))
            self.thread.daemon = True
            self.thread.start()

        self.nthreads = nthreads
        if nthreads > 1:
            self.pool = multiprocessing.pool.ThreadPool(6) 

def recvPacket(self, timeout=None):
        """
        This function polls the recvQueue for the next available HCI
        packet and returns it. The function checks whether it is called
        from the sendThread or any other thread and respectively chooses
        either the sendThreadrecvQueue or the recvQueue.

        The recvQueue is filled by the recvThread. If the queue fills up
        the recvThread empties the queue (unprocessed packets are lost).
        The recvPacket function is meant to receive raw HCI packets in
        a blocking manner. Consider using the registerHciCallback()
        functionality as an alternative which works asynchronously.
        """

        if not self.check_running():
            return None

        try:
            return self.recvQueue.get(timeout=timeout)
        except Queue.Empty:
            return None 

def sendHciCommand(self, opcode, data, timeout=2):
        """
        Send an arbitrary HCI packet by pushing a send-task into the
        sendQueue. This function blocks until the response is received
        or the timeout expires. The return value is the Payload of the
        HCI Command Complete Event which was received in response to
        the command or None if no response was received within the timeout.
        """

        queue = Queue.Queue(1)
        try:
            self.sendQueue.put((opcode, data, queue), timeout=timeout)
            return queue.get(timeout=timeout)
        except Queue.Empty:
            log.warn("sendHciCommand: waiting for response timed out!")
            return None
        except Queue.Full:
            log.warn("sendHciCommand: send queue is full!")
            return None 

def bThread(iplist):

    threadl = []
    queue = Queue.Queue()
    for host in iplist:
        queue.put(host)

    for x in xrange(0, int(sys.argv[2])):
        threadl.append(tThread(queue))

    for t in threadl:
        t.start()
    for t in threadl:
        t.join()

#create thread 

def bThread(iplist):

    threadl = []
    queue = Queue.Queue()
    for host in iplist:
        queue.put(host)

    for x in xrange(0, int(sys.argv[1])):
        threadl.append(tThread(queue))

    for t in threadl:
        t.start()
    for t in threadl:
        t.join()

#create thread 

def threaded_generator(generator, num_cached=10):
    # this code is written by jan Schluter
    # copied from https://github.com/benanne/Lasagne/issues/12
    import Queue
    queue = Queue.Queue(maxsize=num_cached)
    sentinel = object()  # guaranteed unique reference

    # define producer (putting items into queue)
    def producer():
        for item in generator:
            queue.put(item)
        queue.put(sentinel)

    # start producer (in a background thread)
    import threading
    thread = threading.Thread(target=producer)
    thread.daemon = True
    thread.start()

    # run as consumer (read items from queue, in current thread)
    item = queue.get()
    while item is not sentinel:
        yield item
        queue.task_done()
        item = queue.get() 

def generate_in_background(generator, num_cached=10):
    """
    Runs a generator in a background thread, caching up to `num_cached` items.
    """
    import Queue
    queue = Queue.Queue(maxsize=num_cached)
    sentinel = object()  # guaranteed unique reference

    # define producer (putting items into queue)
    def producer():
        for item in generator:
            queue.put(item)
        queue.put(sentinel)

    # start producer (in a background thread)
    import threading
    thread = threading.Thread(target=producer)
    thread.daemon = True
    thread.start()

    # run as consumer (read items from queue, in current thread)
    item = queue.get()
    while item is not sentinel:
        yield item
        item = queue.get() 

def get_sizes(jsn):
   thrd = threading.current_thread()
   logging.info('starting thread '+str(thrd.ident)+' ...')
   try:
      while True:
         if queue.empty() == True: 
            break
         itm = queue.get()
         logging.info(str(thrd.ident)+' :' +str(itm))
         val = get_remote_size(itm)
         if val != None: jsn[itm]['objsize'] = val
         queue.task_done()
   except Queue.Empty: 
      pass
   logging.info('thread '+str(thrd.ident)+' done...') 
#get_sizes

#--------------------------------------------------------------------------------- 

def extractInfo(self):
        try:
            while not self.exit:
                try:
                    frame = self.frame_queue.get(block=True, timeout=1)
                except queue.Empty:
                    print("Queue empty")
                    continue
                try:
                    # Publish new image
                    msg = self.bridge.cv2_to_imgmsg(frame, 'rgb8')
                    if not self.exit:
                        self.image_publisher.publish(msg)
                except CvBridgeError as e:
                    print("Error Converting cv image: {}".format(e.message))
                self.frame_num += 1
        except Exception as e:
            print("Exception after loop: {}".format(e))
            raise 

def __init__(self, max_workers=None, thread_name_prefix=''):
        """Initializes a new ThreadPoolExecutor instance.

        Args:
            max_workers: The maximum number of threads that can be used to
                execute the given calls.
            thread_name_prefix: An optional name prefix to give our threads.
        """
        if max_workers is None:
            # Use this number because ThreadPoolExecutor is often
            # used to overlap I/O instead of CPU work.
            max_workers = (cpu_count() or 1) * 5
        if max_workers <= 0:
            raise ValueError("max_workers must be greater than 0")

        self._max_workers = max_workers
        self._work_queue = queue.Queue()
        self._idle_semaphore = threading.Semaphore(0)
        self._threads = set()
        self._shutdown = False
        self._shutdown_lock = threading.Lock()
        self._thread_name_prefix = (thread_name_prefix or
                                    ("ThreadPoolExecutor-%d" % self._counter())) 

def __init__(self, stream, event):
        self.strm = stream
        self.que = Queue()
	self.event = event

        def populateQueue(stream, queue, event):
            while not event.is_set():
                line = stream.readline()
                if line:
                    queue.put(line)
                else:
		     break

        self.thr = Thread(target = populateQueue,
                args = (self.strm, self.que, self.event))
        self.thr.daemon = True
        self.thr.start() #start collecting lines from the stream 

def bfs_print(root):
    print "*** bfs print ***"
    q = Queue()
    s_node=root
    q.put(s_node)
    seen_list = []

    while(not q.empty()):
        node=q.get() # removes
        print node.n_node_id
        if(node not in seen_list):
            seen_list.append(node)
        for child in node.children:
            if(child not in seen_list):
                q.put(child)
    print "---------------" 

def add_square_at(root,nodeid):
    q = Queue()
    s_node=root
    q.put(s_node)
    seen_list = []

    while(not q.empty()):
        node=q.get() # removes
        if nodeid==node.n_node_id:
            node.add_square()
        else:
            if(node not in seen_list):
                seen_list.append(node)
            for child in node.children:
                if(child not in seen_list):
                    q.put(child) 

def _check_ip_all(self):
        rows = self.session.query(Proxy).all()
        self.thread_pool = ThreadPool(thread_count=10 if not len(rows)/20 else len(rows)/20)
        for row in rows:
            self.thread_pool.add_func(self._check, ip=row.ip, port=row.port, save_to_queue=True)
        self.thread_pool.close()
        self.thread_pool.join()
        while True:
            if self.thread_pool.exit is True and self.result_queue.empty():
                break
            else:
                try:
                    res = self.result_queue.get_nowait()
                    ip = res[0]
                    port = res[1]
                    delay = res[3]
                    alive = res[2]
                    logger.info("IP {0} Connect {1}, time: {2:.2f}s".format(ip, "success", delay)) if alive \
                        else logger.error("IP {0} Connect failed.".format(ip))
                    self._update_db(ip, port, delay, alive)
                except Queue.Empty:
                    time.sleep(2) 

def _check_and_execute(self):
        wakeup_queue = self._wakeup_queue
        while 1:
            (next_expired_time, expired_timers) = self._get_expired_timers()
            for timer in expired_timers:
                try:
                    # Note, please make timer callback effective/short
                    timer()
                except Exception:
                    logging.error(traceback.format_exc())

            self._reset_timers(expired_timers)

            sleep_time = _calc_sleep_time(next_expired_time)
            try:
                wakeup = wakeup_queue.get(timeout=sleep_time)
                if wakeup is TEARDOWN_SENTINEL:
                    break
            except Queue.Empty:
                pass
        logging.info('TimerQueue stopped.') 

def _check_and_execute(self):
        wakeup_queue = self._wakeup_queue
        while 1:
            (next_expired_time, expired_timers) = self._get_expired_timers()
            for timer in expired_timers:
                try:
                    # Note, please make timer callback effective/short
                    timer()
                except Exception:
                    logging.error(traceback.format_exc())

            self._reset_timers(expired_timers)

            sleep_time = _calc_sleep_time(next_expired_time)
            try:
                wakeup = wakeup_queue.get(timeout=sleep_time)
                if wakeup is TEARDOWN_SENTINEL:
                    break
            except Queue.Empty:
                pass
        logging.info('TimerQueue stopped.') 

def __init__(self, max_workers=None, thread_name_prefix=''):
        """Initializes a new ThreadPoolExecutor instance.

        Args:
            max_workers: The maximum number of threads that can be used to
                execute the given calls.
            thread_name_prefix: An optional name prefix to give our threads.
        """
        if max_workers is None:
            # Use this number because ThreadPoolExecutor is often
            # used to overlap I/O instead of CPU work.
            max_workers = (cpu_count() or 1) * 5
        if max_workers <= 0:
            raise ValueError("max_workers must be greater than 0")

        self._max_workers = max_workers
        self._work_queue = queue.Queue()
        self._threads = set()
        self._shutdown = False
        self._shutdown_lock = threading.Lock()
        self._thread_name_prefix = (thread_name_prefix or
                                    ("ThreadPoolExecutor-%d" % self._counter())) 

def run(self):

        while True:

            try:
                item = self.queue.get(timeout=3)
            except Queue.Empty:
                break

            time = self.music_time if item[0] == 'Audio' else self.video_time

            if time and (not self.player.isPlayingVideo() or xbmc.getCondVisibility('VideoPlayer.Content(livetv)')):
                dialog("notification", heading="%s %s" % (_(33049), item[0]), message=item[1],
                       icon="{emby}", time=time, sound=False)

            self.queue.task_done()

            if window('emby_should_stop.bool'):
                break

        LOG.info("--<[ q:notify/%s ]", id(self))
        self.is_done = True 

def __init__(self, thread_count=cpu_count()*2):
        self.__thread_count = thread_count
        self.__function_list = Queue.Queue()
        self.__thread_list = []
        self.__alive_thread_counts = 0
        self.__working_thread_list = []
        self.__dead_threads = []
        self.finished = False 

def __exit__(self, extype, exvalue, traceback):
        # if we had a MySQL related error we try to rollback the cursor.
        if extype is mysql.MySQLError:
            self.cursor.rollback()

        self.cursor.close()
        self.conn.commit()

        # Put it back on the queue
        try:
            self._cache.put_nowait(self.conn)
        except Queue.Full:
            self.conn.close() 

def clear_cache(cls):
        cls._cache = Queue.Queue(maxsize=5) 

def __init__(self, cursor_type=mysql.cursors.Cursor, **options):
        super(Cursor, self).__init__()

        try:
            conn = self._cache.get_nowait()
        except Queue.Empty:
            conn = mysql.connect(**options)
        else:
            # Ping the connection before using it from the cache.
            conn.ping(True)

        self.conn = conn
        self.conn.autocommit(False)
        self.cursor_type = cursor_type 

def _producer_multi_processes(queue_task,
                              queue_product,
                              threads_per_process,
                              worker_function):
    """
    接收与多进程任务并分发给子线程

    :type queue_task: multiprocessing.JoinableQueue
    :type queue_product: multiprocessing.JoinableQueue
    :type threads_per_process: int
    :type worker_function: Callable[[Any], Any]
    """
    _queue_task = queue.Queue(maxsize=threads_per_process)
    _queue_product = queue.Queue()

    pool = [threading.Thread(target=_producer_multi_threads, args=(_queue_task, _queue_product, worker_function))
            for _ in range(threads_per_process)]
    for t in pool:
        t.daemon = True
        t.start()

    th = threading.Thread(target=_subprocesses_queue_transfer, args=(queue_task, _queue_task))
    th.daemon = True
    th.start()

    th = threading.Thread(target=_subprocesses_queue_transfer, args=(_queue_product, queue_product))
    th.daemon = True
    th.start()

    # 等待所有子线程结束
    for t in pool:
        t.join()
        logger.debug("subthread {} of {} stopped".format(t.name, multiprocessing.current_process().name))
    logger.debug("subprocess {} completed".format(multiprocessing.current_process().name)) 

def __init__(self, thread_count=cpu_count()):

        # 初始化相关变量
        super(ThreadPool, self).__init__()
        self.thread_count = thread_count if thread_count else 1

        # 任务队列和正在运行的线程列表
        self.working_list = list()
        self.func_list = Queue.Queue()

        # 结果队列
        self.result_queue = Queue.Queue()

        # 线程数量统计
        self.dead_thread_number = 0
        self.all_thread_number = 0

        # join和close标志
        self.joined = False
        self.closed = False

        # 退出标志
        self.exit = False
        self.already_exit = False

        # 开启主线程
        loop_thread = threading.Thread(target=self.__loop, name="ThreadPool.loop")
        # 如果开启了这个，主线程退出的时候loop就会终止
        loop_thread.daemon = False
        loop_thread.start()

        self.working_thread_number = 0

        # 线程池debug标志
        self.DEBUG = False 

def __init__(self):
        super(WebClicker, self).__init__()

        # 工作队列
        self.work_queue = Queue.Queue()
        # 退出标志
        self._exit = False 

def _loop(self):
        while True:
            # 如果检测到退出标志，则退出
            if self._exit:
                break

            target_url = None
            try:
                target_url = self.work_queue.get(block=True, timeout=1)
            except Queue.Empty:
                time.sleep(5) 

def __init__(self, config):
        self.config = config
        self.logger = util.log.getLogger('OneSwitchController')
        self.logger.info('os_ctrlr: creating an instance of OneSwitchController')

        self.fm_queue = Queue()
        self.last_command_type = {} 

def check_async(paths, options, rootdir=None):
    """Check given paths asynchronously.

    :return list: list of errors

    """
    LOGGER.info('Async code checking is enabled.')
    path_queue = Queue.Queue()
    result_queue = Queue.Queue()

    for num in range(CPU_COUNT):
        worker = Worker(path_queue, result_queue)
        worker.setDaemon(True)
        LOGGER.info('Start worker #%s', (num + 1))
        worker.start()

    for path in paths:
        path_queue.put((path, dict(options=options, rootdir=rootdir)))

    path_queue.join()

    errors = []
    while True:
        try:
            errors += result_queue.get(False)
        except Queue.Empty:
            break

    return errors


# pylama:ignore=W0212,D210,F0001 

def __init__(self):
        self.spiders = []
        self.results = Queue.Queue()