Python threading.enumerate() Examples

The following are code examples for showing how to use threading.enumerate(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: ras-frontstage   Author: ONSdigital   File: gunicorn.py    MIT License 7 votes vote down vote up
def worker_int(worker):
    worker.log.info("worker received INT or QUIT signal")

    ## get traceback info
    import threading, sys, traceback
    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
    code = []
    for threadId, stack in sys._current_frames().items():
        code.append("\n# Thread: %s(%d)" % (id2name.get(threadId,""),
            threadId))
        for filename, lineno, name, line in traceback.extract_stack(stack):
            code.append('File: "%s", line %d, in %s' % (filename,
                lineno, name))
            if line:
                code.append("  %s" % (line.strip()))
    worker.log.debug("\n".join(code)) 
Example 2
Project: backend.ai-agent   Author: lablup   File: __init__.py    GNU Lesser General Public License v3.0 7 votes vote down vote up
def interrupt(self):
        if self.inproc_runner is None:
            log.error('No user code is running!')
            return
        # A dirty hack to raise an exception inside a running thread.
        target_tid = self.inproc_runner.ident
        if target_tid not in {t.ident for t in threading.enumerate()}:
            log.error('Interrupt failed due to missing thread.')
            return
        affected_count = ctypes.pythonapi.PyThreadState_SetAsyncExc(
            ctypes.c_long(target_tid),
            ctypes.py_object(KeyboardInterrupt))
        if affected_count == 0:
            log.error('Interrupt failed due to invalid thread identity.')
        elif affected_count > 1:
            ctypes.pythonapi.PyThreadState_SetAsyncExc(
                ctypes.c_long(target_tid),
                ctypes.c_long(0))
            log.error('Interrupt broke the interpreter state -- '
                      'recommended to reset the session.') 
Example 3
Project: pyblish-win   Author: pyblish   File: test_threading.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_enumerate_after_join(self):
        # Try hard to trigger #1703448: a thread is still returned in
        # threading.enumerate() after it has been join()ed.
        enum = threading.enumerate
        old_interval = sys.getcheckinterval()
        try:
            for i in xrange(1, 100):
                # Try a couple times at each thread-switching interval
                # to get more interleavings.
                sys.setcheckinterval(i // 5)
                t = threading.Thread(target=lambda: None)
                t.start()
                t.join()
                l = enum()
                self.assertNotIn(t, l,
                    "#1703448 triggered after %d trials: %s" % (i, l))
        finally:
            sys.setcheckinterval(old_interval) 
Example 4
Project: ANN   Author: waynezv   File: spider.py    MIT License 6 votes vote down vote up
def run(self):
        while True:
            try:
                if self.task.current_picture_num >= config.max_picture_num and \
                   config.max_picture_num != -1:
                    self.task.to_crawl = []
                    self.task.to_filt = []
                elif self.task.collect_lock and \
                   len(threading.enumerate()) < config.max_thread_num and\
                   len(self.task.to_filt) > 0:
                    config.collect_sleep_time = float(config.collect_sleep_time/config.speed_ratio)
                    url = self.task.to_filt.pop()

                    new_thread = threading.Thread(target=self.start_filt, args=(url,))
                    new_thread.start()
                else:
                    config.collect_sleep_time = config.init_sleep_time
            except:
                traceback.print_exc()
            finally:
                time.sleep(config.collect_sleep_time) 
Example 5
Project: ngo-addons-backport   Author: camptocamp   File: server.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def dumpstacks(sig, frame):
    """ Signal handler: dump a stack trace for each existing thread."""
    # code from http://stackoverflow.com/questions/132058/getting-stack-trace-from-a-running-python-application#answer-2569696
    # modified for python 2.5 compatibility
    threads_info = dict([(th.ident, {'name': th.name,
                                    'uid': getattr(th,'uid','n/a')})
                                for th in threading.enumerate()])
    code = []
    for threadId, stack in sys._current_frames().items():
        thread_info = threads_info.get(threadId)
        code.append("\n# Thread: %s (id:%s) (uid:%s)" % \
                    (thread_info and thread_info['name'] or 'n/a',
                     threadId,
                     thread_info and thread_info['uid'] or 'n/a'))
        for filename, lineno, name, line in traceback.extract_stack(stack):
            code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
            if line:
                code.append("  %s" % (line.strip()))
    _logger.info("\n".join(code)) 
Example 6
Project: openhatch   Author: campbe13   File: __init__.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def teardown():
    # Don't want SUBDEBUG log messages at finalization.
    try:
        from multiprocessing.util import get_logger
    except ImportError:
        pass
    else:
        get_logger().setLevel(logging.WARNING)

    # Make sure test database is removed.
    import os
    if os.path.exists("test.db"):
        try:
            os.remove("test.db")
        except WindowsError:
            pass

    # Make sure there are no remaining threads at shutdown.
    import threading
    remaining_threads = [thread for thread in threading.enumerate()
                            if thread.getName() != "MainThread"]
    if remaining_threads:
        sys.stderr.write(
            "\n\n**WARNING**: Remaining threads at teardown: %r...\n" % (
                remaining_threads)) 
Example 7
Project: flight-lab   Author: google   File: main.py    Apache License 2.0 6 votes vote down vote up
def _on_command(self, command):
    if command == controller_pb2.SystemCommand.EXIT:
      self.exit()
      return

    if command == controller_pb2.SystemCommand.DEBUG:
      self.logger.debug('{0} threads are alive.'.format(
          threading.active_count()))
      for thread in threading.enumerate():
        self.logger.debug('Thread (name="{0}")'.format(thread.name))
      return

    for component in self._components:
      pattern.run_as_thread(
          name='{0}.on_command({1})'.format(component.name, command),
          target=component.on_command,
          kwargs={'command': command}) 
Example 8
Project: python_course   Author: ktbyers   File: exercise2_with_threads.py    Apache License 2.0 6 votes vote down vote up
def main():
    password = getpass()
    start_time = datetime.now()

    hostnames = [
        'arista1.twb-tech.com',
        'arista2.twb-tech.com',
        'arista3.twb-tech.com',
        'arista4.twb-tech.com',
    ]

    print()
    print(">>>>>")
    for host in hostnames:
        net_device = create_device_dict(host, password)
        my_thread = threading.Thread(target=scp_file, args=(net_device,))
        my_thread.start()

    main_thread = threading.currentThread()
    for some_thread in threading.enumerate():
        if some_thread != main_thread:
            some_thread.join()
    print(">>>>>")

    print("\nElapsed time: " + str(datetime.now() - start_time)) 
Example 9
Project: python_course   Author: ktbyers   File: ex6_threads_show_ver.py    Apache License 2.0 6 votes vote down vote up
def main():
    '''
    Use threads and Netmiko to connect to each of the devices in the database. Execute
    'show version' on each device. Record the amount of time required to do this.
    '''
    start_time = datetime.now()
    devices = NetworkDevice.objects.all()

    for a_device in devices:
        my_thread = threading.Thread(target=show_version, args=(a_device,))
        my_thread.start()

    main_thread = threading.currentThread()
    for some_thread in threading.enumerate():
        if some_thread != main_thread:
            print(some_thread)
            some_thread.join()

    print("\nElapsed time: " + str(datetime.now() - start_time)) 
Example 10
Project: NiujiaoDebugger   Author: MrSrc   File: test_threading.py    GNU General Public License v3.0 6 votes vote down vote up
def test_enumerate_after_join(self):
        # Try hard to trigger #1703448: a thread is still returned in
        # threading.enumerate() after it has been join()ed.
        enum = threading.enumerate
        old_interval = sys.getswitchinterval()
        try:
            for i in range(1, 100):
                sys.setswitchinterval(i * 0.0002)
                t = threading.Thread(target=lambda: None)
                t.start()
                t.join()
                l = enum()
                self.assertNotIn(t, l,
                    "#1703448 triggered after %d trials: %s" % (i, l))
        finally:
            sys.setswitchinterval(old_interval) 
Example 11
Project: NiujiaoDebugger   Author: MrSrc   File: lock_tests.py    GNU General Public License v3.0 6 votes vote down vote up
def test_thread_leak(self):
        # The lock shouldn't leak a Thread instance when used from a foreign
        # (non-threading) thread.
        lock = self.locktype()
        def f():
            lock.acquire()
            lock.release()
        n = len(threading.enumerate())
        # We run many threads in the hope that existing threads ids won't
        # be recycled.
        Bunch(f, 15).wait_for_finished()
        if len(threading.enumerate()) != n:
            # There is a small window during which a Thread instance's
            # target function has finished running, but the Thread is still
            # alive and registered.  Avoid spurious failures by waiting a
            # bit more (seen on a buildbot).
            time.sleep(0.4)
            self.assertEqual(n, len(threading.enumerate())) 
Example 12
Project: NiujiaoDebugger   Author: MrSrc   File: __init__.py    GNU General Public License v3.0 6 votes vote down vote up
def run_unittest(*classes):
    """Run tests from unittest.TestCase-derived classes."""
    valid_types = (unittest.TestSuite, unittest.TestCase)
    suite = unittest.TestSuite()
    for cls in classes:
        if isinstance(cls, str):
            if cls in sys.modules:
                suite.addTest(unittest.findTestCases(sys.modules[cls]))
            else:
                raise ValueError("str arguments must be keys in sys.modules")
        elif isinstance(cls, valid_types):
            suite.addTest(cls)
        else:
            suite.addTest(unittest.makeSuite(cls))
    _filter_suite(suite, match_test)
    _run_suite(suite)

#=======================================================================
# Check for the presence of docstrings.

# Rather than trying to enumerate all the cases where docstrings may be
# disabled, we just check for that directly 
Example 13
Project: datasette   Author: simonw   File: app.py    Apache License 2.0 6 votes vote down vote up
def threads(self):
        threads = list(threading.enumerate())
        d = {
            "num_threads": len(threads),
            "threads": [
                {"name": t.name, "ident": t.ident, "daemon": t.daemon} for t in threads
            ],
        }
        # Only available in Python 3.7+
        if hasattr(asyncio, "all_tasks"):
            tasks = asyncio.all_tasks()
            d.update(
                {
                    "num_tasks": len(tasks),
                    "tasks": [_cleaner_task_str(t) for t in tasks],
                }
            )
        return d 
Example 14
Project: virtual_warehouse_api   Author: includeamin   File: gunicorn_config.py    MIT License 6 votes vote down vote up
def worker_int(worker):
    worker.log.info("worker received INT or QUIT signal")

    ## get traceback info
    import threading, sys, traceback
    id2name = {th.ident: th.name for th in threading.enumerate()}
    code = []
    for threadId, stack in sys._current_frames().items():
        code.append("\n# Thread: %s(%d)" % (id2name.get(threadId,""),
            threadId))
        for filename, lineno, name, line in traceback.extract_stack(stack):
            code.append('File: "%s", line %d, in %s' % (filename,
                lineno, name))
            if line:
                code.append("  %s" % (line.strip()))
    worker.log.debug("\n".join(code)) 
Example 15
Project: Fragile   Author: dmgolembiowski   File: __init__.py    Apache License 2.0 6 votes vote down vote up
def print_debugger(function):
    @functools.wraps(function)
    def result(*args, **kwargs):
        debugName = function.__name__ + "("
        if args:
            for i, arg in enumerate(args):
                if i != 0:
                    debugName += ", "
                debugName += repr(arg)
        if kwargs:
            for i, (arg, val) in enumerate(kwargs.items()):
                if i != 0 or len(args) > 0:
                    debugName += ", "
                debugName += str(arg) + "=" + str(val)
        debugName += ")"
        print(f"C {debugName}")
        res = function(*args, **kwargs)
        if res:
            print(f"R {debugName} = {res}")
        return res
    return result
#----------------------------------------------------------------------------- 
Example 16
Project: Alexa-Ecovacs   Author: EtienneMD   File: xmlstream.py    MIT License 6 votes vote down vote up
def _wait_for_threads(self):
        with self.__thread_cond:
            if self.__thread_count != 0:
                log.debug("Waiting for %s threads to exit." %
                        self.__thread_count)
                name = threading.current_thread().name
                if name in self.__thread:
                    self._end_thread(name, early=True)
                self.__thread_cond.wait(4)
                if self.__thread_count != 0:
                    log.error("Hanged threads: %s" % threading.enumerate())
                    log.error("This may be due to calling disconnect() " + \
                              "from a non-threaded event handler. Be " + \
                              "sure that event handlers that call " + \
                              "disconnect() are registered using: " + \
                              "add_event_handler(..., threaded=True)") 
Example 17
Project: pyDcop   Author: Orange-OpenSource   File: orchestrator.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def on_timeout():
    logger.debug("cli timeout ")
    # Timeout should have been handled by the orchestrator, if the cli timeout
    # has been reached, something is probably wrong : dump threads.
    for th in threading.enumerate():
        print(th)
        traceback.print_stack(sys._current_frames()[th.ident])
        print()

    if orchestrator is None:
        logger.debug("cli timeout with no orchestrator ?")
        return
    global timeout_stopped
    timeout_stopped = True

    orchestrator.stop_agents(20)
    orchestrator.stop()
    _results("TIMEOUT")
    sys.exit(0) 
Example 18
Project: pyDcop   Author: Orange-OpenSource   File: solve.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def on_timeout():
    logger.debug("cli timeout ")
    # Timeout should have been handled by the orchestrator, if the cli timeout
    # has been reached, something is probably wrong : dump threads.
    for th in threading.enumerate():
        print(th)
        traceback.print_stack(sys._current_frames()[th.ident])
        print()

    if orchestrator is None:
        logger.debug("cli timeout with no orchestrator ?")
        return
    global timeout_stopped
    timeout_stopped = True
    # Stopping agents can be rather long, we need a big timeout !
    logger.debug("stop agent on cli timeout ")
    orchestrator.stop_agents(20)
    logger.debug("stop orchestrator on cli timeout ")
    orchestrator.stop()
    _results("TIMEOUT")
    # sys.exit(0)
    os._exit(2) 
Example 19
Project: pyDcop   Author: Orange-OpenSource   File: run.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def on_timeout():
    if orchestrator is None:
        return
    # Timeout should have been handled by the orchestrator, if the cli timeout
    # has been reached, something is probably wrong : dump threads.
    for th in threading.enumerate():
        print(th)
        traceback.print_stack(sys._current_frames()[th.ident])
        print()
    if orchestrator is None:
        logger.debug("cli timeout with no orchestrator ?")
        return
    global timeout_stopped
    timeout_stopped = True

    # Stopping agents can be rather long, we need a big timeout !
    orchestrator.stop_agents(20)
    orchestrator.stop()
    _results("TIMEOUT")
    sys.exit(0) 
Example 20
Project: pyDcop   Author: Orange-OpenSource   File: distribute.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def on_timeout():
    global result, output_file
    global start_t
    duration = time.time() - start_t

    print("TIMEOUT when distributing")
    logger.info("cli timeout when distributing")
    for th in threading.enumerate():
        print(th)
        traceback.print_stack(sys._current_frames()[th.ident])

    result["status"] =  "TIMEOUT"
    result["inputs"]["duration"] = duration

    if output_file is not None:
        with open(output_file, encoding="utf-8", mode="w") as fo:
            fo.write(yaml.dump(result))
    print(yaml.dump(result))

    #os._exit(0)
    sys.exit(0) 
Example 21
Project: stem   Author: torproject   File: output.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def thread_stacktraces():
  """
  Provides a dump of the stacktrace information for all active threads.

  :returns: **dict** that maps thread names to their stacktrace
  """

  stacktraces = {}

  for thread in threading.enumerate():
    frame = sys._current_frames().get(thread.ident, None)

    if frame:
      stacktraces[thread.name] = ''.join(traceback.format_stack(frame))
    else:
      stacktraces[thread.name] = 'No traceback available'

  return stacktraces 
Example 22
Project: pymacaron   Author: pymacaron   File: gunicorn.py    BSD 2-Clause "Simplified" License 6 votes vote down vote up
def worker_int(worker):
    worker.log.info("worker received INT or QUIT signal")

    # get traceback info
    import threading
    import sys
    import traceback
    id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
    code = []
    for threadId, stack in sys._current_frames().items():
        code.append("\n# Thread: %s(%d)" % (id2name.get(threadId, ""), threadId))
        for filename, lineno, name, line in traceback.extract_stack(stack):
            code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
            if line:
                code.append("  %s" % (line.strip()))
    worker.log.debug("\n".join(code)) 
Example 23
Project: PyChroner   Author: NephyProject   File: manager.py    MIT License 6 votes vote down vote up
def watchThreads(self) -> None:
        while True:
            try:
                workingThreads: List[Thread] = threading.enumerate()
                with open(f"{self.core.config.directory.api}/{API.Thread.value}", "w") as f:
                    json.dump([x.name for x in workingThreads], f, sort_keys=True, indent=4)

                for i, thread, func in enumerate(self.threads):
                    if not thread.is_alive() or thread not in workingThreads:
                        if thread.name in [x.meta.name for x in self.core.PM.plugins.values()]:
                            self.threads[i] = self.startThread(func, name=thread.name)
                        else:
                            self.threads[i] = self.startThread(func, name=thread.name, args=[self.core])

            except:
                pass
            finally:
                time.sleep(10) 
Example 24
Project: ironpython2   Author: IronLanguages   File: test_threading.py    Apache License 2.0 6 votes vote down vote up
def test_enumerate_after_join(self):
        # Try hard to trigger #1703448: a thread is still returned in
        # threading.enumerate() after it has been join()ed.
        enum = threading.enumerate
        old_interval = sys.getcheckinterval()
        try:
            for i in xrange(1, 100):
                # Try a couple times at each thread-switching interval
                # to get more interleavings.
                sys.setcheckinterval(i // 5)
                t = threading.Thread(target=lambda: None)
                t.start()
                t.join()
                l = enum()
                self.assertNotIn(t, l,
                    "#1703448 triggered after %d trials: %s" % (i, l))
        finally:
            sys.setcheckinterval(old_interval) 
Example 25
Project: pdbclone   Author: corpusops   File: pdb.py    GNU General Public License v2.0 6 votes vote down vote up
def _print_lines(self, lines, start, breaks=(), frame=None):
        """Print a range of lines."""
        if frame:
            current_lineno = frame.f_lineno
            exc_lineno = self.tb_lineno.get(frame, -1)
        else:
            current_lineno = exc_lineno = -1
        for lineno, line in enumerate(lines, start):
            s = str(lineno).rjust(3)
            if len(s) < 4:
                s += ' '
            if lineno in breaks:
                s += 'B'
            else:
                s += ' '
            if lineno == current_lineno:
                s += '->'
            elif lineno == exc_lineno:
                s += '>>'
            self.message(s + '\t' + line.rstrip()) 
Example 26
Project: pynet-ons-jul17   Author: ktbyers   File: threads_show_ver.py    Apache License 2.0 6 votes vote down vote up
def main():
    '''
    Use threads and Netmiko to connect to each of the devices. Execute
    'show version' on each device. Record the amount of time required to do this.
    '''
    start_time = datetime.now()

    for a_device in devices:
        my_thread = threading.Thread(target=show_version, args=(a_device,))
        my_thread.start()

    main_thread = threading.currentThread()
    for some_thread in threading.enumerate():
        if some_thread != main_thread:
            print(some_thread)
            some_thread.join()

    print("\nElapsed time: " + str(datetime.now() - start_time)) 
Example 27
Project: banruo   Author: yingshang   File: __init__.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def teardown():
    # Don't want SUBDEBUG log messages at finalization.
    try:
        from multiprocessing.util import get_logger
    except ImportError:
        pass
    else:
        get_logger().setLevel(logging.WARNING)

    # Make sure test database is removed.
    import os
    if os.path.exists('test.db'):
        try:
            os.remove('test.db')
        except WindowsError:
            pass

    # Make sure there are no remaining threads at shutdown.
    import threading
    remaining_threads = [thread for thread in threading.enumerate()
                         if thread.getName() != 'MainThread']
    if remaining_threads:
        sys.stderr.write(
            '\n\n**WARNING**: Remaining threads at teardown: %r...\n' % (
                remaining_threads)) 
Example 28
Project: cdn-cache   Author: gbrunacci   File: test_bus.py    Apache License 2.0 6 votes vote down vote up
def test_builtin_channels(self):
        b = wspbus.Bus()

        self.responses, expected = [], []

        for channel in b.listeners:
            for index, priority in enumerate([100, 50, 0, 51]):
                b.subscribe(channel,
                            self.get_listener(channel, index), priority)

        for channel in b.listeners:
            b.publish(channel)
            expected.extend([msg % (i, channel, None) for i in (2, 1, 3, 0)])
            b.publish(channel, arg=79347)
            expected.extend([msg % (i, channel, 79347) for i in (2, 1, 3, 0)])

        self.assertEqual(self.responses, expected) 
Example 29
Project: cdn-cache   Author: gbrunacci   File: test_bus.py    Apache License 2.0 6 votes vote down vote up
def test_custom_channels(self):
        b = wspbus.Bus()

        self.responses, expected = [], []

        custom_listeners = ('hugh', 'louis', 'dewey')
        for channel in custom_listeners:
            for index, priority in enumerate([None, 10, 60, 40]):
                b.subscribe(channel,
                            self.get_listener(channel, index), priority)

        for channel in custom_listeners:
            b.publish(channel, 'ah so')
            expected.extend([msg % (i, channel, 'ah so')
                            for i in (1, 3, 0, 2)])
            b.publish(channel)
            expected.extend([msg % (i, channel, None) for i in (1, 3, 0, 2)])

        self.assertEqual(self.responses, expected) 
Example 30
Project: pyblish-win   Author: pyblish   File: lock_tests.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_thread_leak(self):
        # The lock shouldn't leak a Thread instance when used from a foreign
        # (non-threading) thread.
        lock = self.locktype()
        def f():
            lock.acquire()
            lock.release()
        n = len(threading.enumerate())
        # We run many threads in the hope that existing threads ids won't
        # be recycled.
        Bunch(f, 15).wait_for_finished()
        self.assertEqual(n, len(threading.enumerate())) 
Example 31
Project: Learning-Concurrency-in-Python   Author: PacktPublishing   File: enumerateThreads.py    MIT License 5 votes vote down vote up
def main():
  for i in range(4):
    thread = threading.Thread(target=myThread, args=(i,))
    thread.start()

  print("Enumerating: {}".format(threading.enumerate())) 
Example 32
Project: ARPET   Author: juliagarriga   File: ps_drone.py    MIT License 5 votes vote down vote up
def __heartbeat(self):
	# If the drone does not get a command, it will mutter after 50ms (CTRL watchdog / state[28] will set to 1)
	# and panic after 2 seconds and abort data-communication on port 5554 (then you have to initialize the network again).
	# Heartbeat will reset the watchdog and, by the way, the ACK_BIT (state[6], to accept any other AT*CONFIG command)
	# If mainthread isn't alive anymore (because program crashed or whatever), heartbeat will initiate the shutdown.
		if str(threading.enumerate()).count("MainThread, stopped") or str(threading.enumerate()).count("MainThread")==0:	self.shutdown()
		else:	self.at("COMWDG",[])

	# CheckAndReact is periodically called by the receiveData-Thread to check for mainly for critical status-error(s) and
	# changed debug-modes. 
Example 33
Project: ANN   Author: waynezv   File: spider.py    MIT License 5 votes vote down vote up
def run(self):
        while True:
            try:
                if self.task.dispatch_lock and len(threading.enumerate()) < config.max_thread_num and\
                len(self.task.to_crawl) > 0:
                    config.dispatch_sleep_time = float(config.dispatch_sleep_time/config.speed_ratio)
                    url = self.task.to_crawl.pop()
                    new_thread = threading.Thread(target=self.start_crawl, args=(url,))
                    new_thread.start()
                else:
                    config.dispatch_sleep_time = config.init_sleep_time
            except:
                traceback.print_exc()
            finally:
                time.sleep(config.dispatch_sleep_time) 
Example 34
Project: ANN   Author: waynezv   File: spider.py    MIT License 5 votes vote down vote up
def auto_regulate(self):
        print "current thread num %s,to_crawl %s, to_filt %s\n" % (len(threading.enumerate()), \
                                                                   len(self.to_crawl), \
                                                                   len(self.to_filt))
        if len(self.to_crawl) > len(self.to_filt):
            self.dispatch_lock = True
            self.collect_lock = False
        else:
            self.dispatch_lock = False
            self.collect_lock = True 
Example 35
Project: ngo-addons-backport   Author: camptocamp   File: __init__.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def stop_services():
    """ Stop all services. """
    # stop services
    cron.stop_service()
    netrpc_server.stop_service()
    wsgi_server.stop_service()

    _logger.info("Initiating shutdown")
    _logger.info("Hit CTRL-C again or send a second signal to force the shutdown.")

    # Manually join() all threads before calling sys.exit() to allow a second signal
    # to trigger _force_quit() in case some non-daemon threads won't exit cleanly.
    # threading.Thread.join() should not mask signals (at least in python 2.5).
    me = threading.currentThread()
    _logger.debug('current thread: %r', me)
    for thread in threading.enumerate():
        _logger.debug('process %r (%r)', thread, thread.isDaemon())
        if thread != me and not thread.isDaemon() and thread.ident != main_thread_id:
            while thread.isAlive():
                _logger.debug('join and sleep')
                # Need a busyloop here as thread.join() masks signals
                # and would prevent the forced shutdown.
                thread.join(0.05)
                time.sleep(0.05)

    _logger.debug('--')
    openerp.modules.registry.RegistryManager.delete_all()
    logging.shutdown() 
Example 36
Project: openhatch   Author: campbe13   File: __init__.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def find_module(module, path=None, imp=None):
    """Version of :func:`imp.find_module` supporting dots."""
    if imp is None:
        imp = importlib.import_module
    with cwd_in_path():
        if "." in module:
            last = None
            parts = module.split(".")
            for i, part in enumerate(parts[:-1]):
                path = imp(".".join(parts[:i + 1])).__path__
                last = _imp.find_module(parts[i + 1], path)
            return last
        return _imp.find_module(module) 
Example 37
Project: openhatch   Author: campbe13   File: __init__.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def cry():  # pragma: no cover
    """Return stacktrace of all active threads.

    From https://gist.github.com/737056

    """
    tmap = {}
    main_thread = None
    # get a map of threads by their ID so we can print their names
    # during the traceback dump
    for t in threading.enumerate():
        if getattr(t, "ident", None):
            tmap[t.ident] = t
        else:
            main_thread = t

    out = StringIO()
    sep = "=" * 49 + "\n"
    for tid, frame in sys._current_frames().iteritems():
        thread = tmap.get(tid, main_thread)
        if not thread:
            # skip old junk (left-overs from a fork)
            continue
        out.write("%s\n" % (thread.getName(), ))
        out.write(sep)
        traceback.print_stack(frame, file=out)
        out.write(sep)
        out.write("LOCAL VARIABLES\n")
        out.write(sep)
        pprint(frame.f_locals, stream=out)
        out.write("\n\n")
    return out.getvalue() 
Example 38
Project: openhatch   Author: campbe13   File: debug.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def _thread_stacks(self):
        id2name = dict((th.ident, th.name) for th in threading.enumerate())
        dumps = ''
        for id_, frame in sys._current_frames().items():
            name = id2name.get(id_, '')
            dump = ''.join(traceback.format_stack(frame))
            dumps += "# Thread: {0}({1})\n{2}\n".format(name, id_, dump)
        return dumps 
Example 39
Project: jawfish   Author: war-and-code   File: support.py    MIT License 5 votes vote down vote up
def modules_cleanup(oldmodules):
    # Encoders/decoders are registered permanently within the internal
    # codec cache. If we destroy the corresponding modules their
    # globals will be set to None which will trip up the cached functions.
    encodings = [(k, v) for k, v in sys.modules.items()
                 if k.startswith('encodings.')]
    sys.modules.clear()
    sys.modules.update(encodings)
    # XXX: This kind of problem can affect more than just encodings. In particular
    # extension modules (such as _ssl) don't cope with reloading properly.
    # Really, test modules should be cleaning out the test specific modules they
    # know they added (ala test_runpy) rather than relying on this function (as
    # test_importhooks and test_pkg do currently).
    # Implicitly imported *real* modules should be left alone (see issue 10556).
    sys.modules.update(oldmodules)

#=======================================================================
# Threading support to prevent reporting refleaks when running regrtest.py -R

# NOTE: we use thread._count() rather than threading.enumerate() (or the
# moral equivalent thereof) because a threading.Thread object is still alive
# until its __bootstrap() method has returned, even after it has been
# unregistered from the threading module.
# thread._count(), on the other hand, only gets decremented *after* the
# __bootstrap() method has returned, which gives us reliable reference counts
# at the end of a test run. 
Example 40
Project: Repobot   Author: Desgard   File: support.py    MIT License 5 votes vote down vote up
def modules_cleanup(oldmodules):
    # Encoders/decoders are registered permanently within the internal
    # codec cache. If we destroy the corresponding modules their
    # globals will be set to None which will trip up the cached functions.
    encodings = [(k, v) for k, v in sys.modules.items()
                 if k.startswith('encodings.')]
    # Was:
    # sys.modules.clear()
    # Py2-compatible:
    for i in range(len(sys.modules)):
        sys.modules.pop()

    sys.modules.update(encodings)
    # XXX: This kind of problem can affect more than just encodings. In particular
    # extension modules (such as _ssl) don't cope with reloading properly.
    # Really, test modules should be cleaning out the test specific modules they
    # know they added (ala test_runpy) rather than relying on this function (as
    # test_importhooks and test_pkg do currently).
    # Implicitly imported *real* modules should be left alone (see issue 10556).
    sys.modules.update(oldmodules)

#=======================================================================
# Backported versions of threading_setup() and threading_cleanup() which don't refer
# to threading._dangling (not available on Py2.7).

# Threading support to prevent reporting refleaks when running regrtest.py -R

# NOTE: we use thread._count() rather than threading.enumerate() (or the
# moral equivalent thereof) because a threading.Thread object is still alive
# until its __bootstrap() method has returned, even after it has been
# unregistered from the threading module.
# thread._count(), on the other hand, only gets decremented *after* the
# __bootstrap() method has returned, which gives us reliable reference counts
# at the end of a test run. 
Example 41
Project: vscode-mayapy   Author: FXTD-ODYSSEY   File: runfiles.py    MIT License 5 votes vote down vote up
def run(self):
                time.sleep(10)

                thread_id_to_name = {}
                try:
                    for t in threading.enumerate():
                        thread_id_to_name[t.ident] = '%s  (daemon: %s)' % (t.name, t.daemon)
                except:
                    pass

                stack_trace = [
                    '===============================================================================',
                    'pydev pyunit runner: Threads still found running after tests finished',
                    '================================= Thread Dump =================================']

                for thread_id, stack in sys._current_frames().items():
                    stack_trace.append('\n-------------------------------------------------------------------------------')
                    stack_trace.append(" Thread %s" % thread_id_to_name.get(thread_id, thread_id))
                    stack_trace.append('')

                    if 'self' in stack.f_locals:
                        sys.stderr.write(str(stack.f_locals['self']) + '\n')

                    for filename, lineno, name, line in traceback.extract_stack(stack):
                        stack_trace.append(' File "%s", line %d, in %s' % (filename, lineno, name))
                        if line:
                            stack_trace.append("   %s" % (line.strip()))
                stack_trace.append('\n=============================== END Thread Dump ===============================')
                sys.stderr.write('\n'.join(stack_trace)) 
Example 42
Project: inmanta   Author: inmanta   File: app.py    Apache License 2.0 5 votes vote down vote up
def dump_threads() -> None:
    print("----- Thread Dump ----")
    for th in threading.enumerate():
        print("---", th)
        traceback.print_stack(sys._current_frames()[th.ident], file=sys.stdout)
        print()
    sys.stdout.flush() 
Example 43
Project: Pansidong   Author: lightless233   File: ThreadPool.py    GNU General Public License v3.0 5 votes vote down vote up
def get_all_threads():
        return threading.enumerate() 
Example 44
Project: NiujiaoDebugger   Author: MrSrc   File: test_threading.py    GNU General Public License v3.0 5 votes vote down vote up
def test_main_thread_during_shutdown(self):
        # bpo-31516: current_thread() should still point to the main thread
        # at shutdown
        code = """if 1:
            import gc, threading

            main_thread = threading.current_thread()
            assert main_thread is threading.main_thread()  # sanity check

            class RefCycle:
                def __init__(self):
                    self.cycle = self

                def __del__(self):
                    print("GC:",
                          threading.current_thread() is main_thread,
                          threading.main_thread() is main_thread,
                          threading.enumerate() == [main_thread])

            RefCycle()
            gc.collect()  # sanity check
            x = RefCycle()
        """
        _, out, err = assert_python_ok("-c", code)
        data = out.decode()
        self.assertEqual(err, b"")
        self.assertEqual(data.splitlines(),
                         ["GC: True True True"] * 2) 
Example 45
Project: PyTools   Author: taige   File: common.py    Apache License 2.0 5 votes vote down vote up
def print_stack_trace(limit=5, out=sys.stdout, *threads):
    frames = sys._current_frames()
    if len(threads) == 0:
        threads = threading.enumerate()
    for th in threads:
        ts_print(th, file=out)
        traceback.print_stack(frames[th.ident], limit=limit, file=out)
        ts_print(file=out) 
Example 46
Project: codimension   Author: SergeySatskiy   File: threadextension_cdm_dbg.py    GNU General Public License v3.0 5 votes vote down vote up
def dumpThreadList(self):
        """Sends the list of threads"""
        self.updateThreadList()
        threadList = []
        if len(self.threads) > 1:
            currentId = _thread.get_ident()
            # update thread names set by user (threading.setName)
            threadNames = {t.ident: t.getName() for t in threading.enumerate()}

            for threadId, thd in self.threads.items():
                threadProps = {"id": threadId}
                try:
                    threadProps["name"] = threadNames.get(threadId, thd.name)
                    threadProps["broken"] = thd.isBroken
                except Exception:
                    threadProps["name"] = 'UnknownThread'
                    threadProps["broken"] = False

                threadList.append(threadProps)
        else:
            currentId = -1
            threadProps = {"id": -1}
            threadProps["name"] = "MainThread"
            threadProps["broken"] = self.isBroken
            threadList.append(threadProps)

        sendJSONCommand(self.socket, METHOD_THREAD_LIST,
                        self.procuuid,
                        {"currentID": currentId, "threadList": threadList}) 
Example 47
Project: calibre-web   Author: janeczku   File: worker.py    GNU General Public License v3.0 5 votes vote down vote up
def _get_main_thread():
    for t in threading.enumerate():
        if t.__class__.__name__ == '_MainThread':
            return t
    raise Exception("main thread not found?!")


# For gdrive download book from gdrive to calibredir (temp dir for books), read contents in both cases and append
# it in MIME Base64 encoded to 
Example 48
Project: calibre-web   Author: janeczku   File: worker.py    GNU General Public License v3.0 5 votes vote down vote up
def _delete_completed_tasks(self):
        for index, task in reversed(list(enumerate(self.UIqueue))):
            if task['progress'] == "100 %":
                # delete tasks
                self.queue.pop(index)
                self.UIqueue.pop(index)
                # if we are deleting entries before the current index, adjust the index
                if index <= self.current and index:
                    self.current -= 1
        self.last = len(self.queue) 
Example 49
Project: Fragile   Author: dmgolembiowski   File: __init__.py    Apache License 2.0 5 votes vote down vote up
def wrap_start(self):
        self.menu.should_exit = True
        for t in threading.enumerate():
            if t is self.menu._main_thread:
                continue
            t.join()
        """
        #self.menu._main_thread.join()
        ## but currently_active_menu probably == None
        CursesMenu.currently_active_menu = None
        # ah, great, we'll just enforce it
        self.menu.should_exit = False
        try:
            self.menu.remove_exit()
        except Exception:
            pass
        try:
            self.menu._main_thread = threading.Thread(
                    target=self.menu._wrap_start,
                    daemon=True)
        except TypeError:
            self.menu._main_thread = threading.Thread(
                    target=self._wrap_start)
            self.menu._main_thread.daemon = True
        self.menu._main.daemon = True
        self.menu._main_thread.start()
        self.menu._main_thread.join(timeout=None)
        if self.menu.parent is None:
            curses.wrapper(self.menu._main_loop)
        else:
            self.menu._main_loop(None)
        CursesMenu.currently_active_menu = None
        self.menu.clear_screen()
        curses_menu.clear_terminal()
        CursesMenu.currently_active_menu = self.menu.previous_active_menu
        """
        self.menu.clear_screen()
        curses_menu.clear_terminal() 
Example 50
Project: benchmarks   Author: tensorflow   File: utils.py    Apache License 2.0 5 votes vote down vote up
def print_thread_stacktrace():
  print('Here is the stacktrace for all threads:')
  thread_names = {t.ident: t.name for t in threading.enumerate()}
  for thread_id, frame in sys._current_frames().items():  # pylint: disable=protected-access
    print('Thread {}'.format(thread_names.get(thread_id, thread_id)))
    traceback.print_stack(frame) 
Example 51
Project: pyDcop   Author: Orange-OpenSource   File: run.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def on_force_exit(sig, frame):
    if orchestrator is None:
        return
    orchestrator.status = "STOPPED"
    orchestrator.stop_agents(5)
    orchestrator.stop()
    _results("STOPPED")
    # for th in threading.enumerate():
    #     print(th)
    #     traceback.print_stack(sys._current_frames()[th.ident])
    #     print() 
Example 52
Project: vagrant-prometheus-lab   Author: alexclear   File: debug.py    MIT License 5 votes vote down vote up
def get_subclasses(klass):
    """
    Rather than statically import every interesting subclass, forcing it all to
    be transferred and potentially disrupting the debugged environment,
    enumerate only those loaded in memory. Also returns the original class.
    """
    stack = [klass]
    seen = set()
    while stack:
        klass = stack.pop()
        seen.add(klass)
        stack.extend(klass.__subclasses__())
    return seen 
Example 53
Project: vagrant-prometheus-lab   Author: alexclear   File: debug.py    MIT License 5 votes vote down vote up
def format_stacks():
    name_by_id = dict(
        (t.ident, t.name)
        for t in threading.enumerate()
    )

    l = ['', '']
    for threadId, stack in sys._current_frames().items():
        l += ["# PID %d ThreadID: (%s) %s; %r" % (
            os.getpid(),
            name_by_id.get(threadId, '<no name>'),
            threadId,
            stack,
        )]
        #stack = stack.f_back.f_back

        for filename, lineno, name, line in traceback.extract_stack(stack):
            l += [
                'File: "%s", line %d, in %s' % (
                    filename,
                    lineno,
                    name
                )
            ]
            if line:
                l += ['    ' + line.strip()]
        l += ['']

    l += ['', '']
    return '\n'.join(l) 
Example 54
Project: PyChroner   Author: NephyProject   File: command.py    MIT License 5 votes vote down vote up
def thread_list() -> None:
        print("Active threads:\n")
        [print(x.name) for x in threading.enumerate()] 
Example 55
Project: PyChroner   Author: NephyProject   File: manager.py    MIT License 5 votes vote down vote up
def destroyThread(self, name: str) -> bool:
        for i, thread, func in enumerate(self.threads):
            if thread.name == name:
                del self.threads[i]
                return True
        return False 
Example 56
Project: xspider   Author: zym1115718204   File: scheduler.py    Apache License 2.0 5 votes vote down vote up
def run_threaded(job_func):
        job_thread = threading.Thread(target=job_func)
        job_thread.start()
        # print threading.enumerate() 
Example 57
Project: ProjetRansomware   Author: Exihack   File: Gui.py    MIT License 5 votes vote down vote up
def decrypt(key, path):
    """
    Function use to start the decrypting process
    :param key -- key to decrypt:
    :param path -- path where are the encrypted files:
    :return:
    """
    # retrieve the key
    if key.get():
        # Cast the key type to Byte
        key = bytes(key.get().encode())
        # Message to confirm before the decrypting process
        if askyesno("Decrypting process", "Are you sure, you want to do that ?"
                               "\nBefore doing this, be sure you have the right key"
                               "\nA bad key will destroy your data and everything will be lost"
                               "\nYou are now aware !"):

            # Return an encrypt list files and their path
            encrypt_files_list_C = Crypto_files.list_files(path)
            list_hdd = List_external_HDD.list_hdd_files()
            list_all_files = list_hdd + encrypt_files_list_C

            # Loop through the files
            for encrypt_file in list_all_files:
                # Create a thread for each files to accelerate the decrypting process
                threading.Thread(target=Crypto_files.decryption_function, args=(key, encrypt_file)).start()
            # We check if all thread are finished
            for thread in threading.enumerate():
                if thread is not main_thread:
                    thread.join()
            # Ending message
            showinfo('Last step', 'Decrypting process finished !')
        else:
            # Cancellation of the decrypting process
            showinfo('Cancellation', 'Cancellation...')
    else:
        # Error message if the key field is empty
        showinfo('Error', 'Please enter a valid key !') 
Example 58
Project: spider   Author: shancang   File: spider.py    Apache License 2.0 5 votes vote down vote up
def main():
    logger.info("start spider.....")
    url_1=GetFirstType(url)
    for type_name,url2 in url_1.items():                     
        t=threading.Thread(target=thrad,args=(type_name,url2))
        t.start()
        while True:
            if(len(threading.enumerate()) < THARED_NUMBER + 1 ):
                break 
Example 59
Project: ironpython2   Author: IronLanguages   File: lock_tests.py    Apache License 2.0 5 votes vote down vote up
def test_thread_leak(self):
        # The lock shouldn't leak a Thread instance when used from a foreign
        # (non-threading) thread.
        lock = self.locktype()
        def f():
            lock.acquire()
            lock.release()
        n = len(threading.enumerate())
        # We run many threads in the hope that existing threads ids won't
        # be recycled.
        Bunch(f, 15).wait_for_finished()
        self.assertEqual(n, len(threading.enumerate())) 
Example 60
Project: pdbclone   Author: corpusops   File: pdb.py    GNU General Public License v2.0 5 votes vote down vote up
def find_function(funcname, filename):
    cre = re.compile(r'def\s+%s\s*[(]' % re.escape(funcname))
    try:
        fp = open(filename)
    except IOError:
        return None
    # consumer of this info expects the first line to be 1
    with fp:
        for lineno, line in enumerate(fp, start=1):
            if cre.match(line):
                return funcname, filename, lineno
    return None 
Example 61
Project: pdbclone   Author: corpusops   File: pdb.py    GNU General Public License v2.0 5 votes vote down vote up
def _complete_bpnumber(self, text, line, begidx, endidx):
        # Complete a breakpoint number.  (This would be more helpful if we could
        # display additional info along with the completions, such as file/line
        # of the breakpoint.)
        return [str(i) for i, bp in enumerate(bdb.Breakpoint.bpbynumber)
                if bp is not None and str(i).startswith(text)] 
Example 62
Project: pynet-ons-jul17   Author: ktbyers   File: threads_ex1.py    Apache License 2.0 5 votes vote down vote up
def main():
    for a_device in devices:
        my_thread = threading.Thread(target=remote_cmd, args=(a_device, "show arp"))
        my_thread.start()

    main_thread = threading.currentThread()
    for some_thread in threading.enumerate():
        if some_thread != main_thread:
            some_thread.join() 
Example 63
Project: banruo   Author: yingshang   File: __init__.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def cry(out=None, sepchr='=', seplen=49):  # pragma: no cover
    """Return stacktrace of all active threads,
    taken from https://gist.github.com/737056."""
    import threading

    out = WhateverIO() if out is None else out
    P = partial(print, file=out)

    # get a map of threads by their ID so we can print their names
    # during the traceback dump
    tmap = dict((t.ident, t) for t in threading.enumerate())

    sep = sepchr * seplen
    for tid, frame in items(sys._current_frames()):
        thread = tmap.get(tid)
        if not thread:
            # skip old junk (left-overs from a fork)
            continue
        P('{0.name}'.format(thread))
        P(sep)
        traceback.print_stack(frame, file=out)
        P(sep)
        P('LOCAL VARIABLES')
        P(sep)
        pprint(frame.f_locals, stream=out)
        P('\n')
    return out.getvalue() 
Example 64
Project: banruo   Author: yingshang   File: case.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def setUp(self):
        self._threads_at_setup = list(threading.enumerate())
        from celery import _state
        from celery import result
        result.task_join_will_block = \
            _state.task_join_will_block = lambda: False
        self._current_app = current_app()
        self._default_app = _state.default_app
        trap = Trap()
        self._prev_tls = _state._tls
        _state.set_default_app(trap)

        class NonTLS(object):
            current_app = trap
        _state._tls = NonTLS()

        self.app = self.Celery(set_as_current=False)
        if not self.contained:
            self.app.set_current()
        root = logging.getLogger()
        self.__rootlevel = root.level
        self.__roothandlers = root.handlers
        _state._set_task_join_will_block(False)
        try:
            self.setup()
        except:
            self._teardown_app()
            raise 
Example 65
Project: banruo   Author: yingshang   File: case.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _teardown_app(self):
        from celery.utils.log import LoggingProxy
        assert sys.stdout
        assert sys.stderr
        assert sys.__stdout__
        assert sys.__stderr__
        this = self._get_test_name()
        if isinstance(sys.stdout, (LoggingProxy, Mock)) or \
                isinstance(sys.__stdout__, (LoggingProxy, Mock)):
            raise RuntimeError(CASE_LOG_REDIRECT_EFFECT.format(this, 'stdout'))
        if isinstance(sys.stderr, (LoggingProxy, Mock)) or \
                isinstance(sys.__stderr__, (LoggingProxy, Mock)):
            raise RuntimeError(CASE_LOG_REDIRECT_EFFECT.format(this, 'stderr'))
        backend = self.app.__dict__.get('backend')
        if backend is not None:
            if isinstance(backend, CacheBackend):
                if isinstance(backend.client, DummyClient):
                    backend.client.cache.clear()
                backend._cache.clear()
        from celery import _state
        _state._set_task_join_will_block(False)

        _state.set_default_app(self._default_app)
        _state._tls = self._prev_tls
        _state._tls.current_app = self._current_app
        if self.app is not self._current_app:
            self.app.close()
        self.app = None
        self.assertEqual(
            self._threads_at_setup, list(threading.enumerate()),
        ) 
Example 66
Project: banruo   Author: yingshang   File: worker.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def active_thread_count():
    from threading import enumerate
    return sum(1 for t in enumerate()
               if not t.name.startswith('Dummy-')) 
Example 67
Project: cdn-cache   Author: gbrunacci   File: _cpthreadinglocal.py    Apache License 2.0 5 votes vote down vote up
def __del__():
        threading_enumerate = enumerate
        __getattribute__ = object.__getattribute__

        def __del__(self):
            key = __getattribute__(self, '_local__key')

            try:
                threads = list(threading_enumerate())
            except:
                # if enumerate fails, as it seems to do during
                # shutdown, we'll skip cleanup under the assumption
                # that there is nothing to clean up
                return

            for thread in threads:
                try:
                    __dict__ = thread.__dict__
                except AttributeError:
                    # Thread is dying, rest in peace
                    continue

                if key in __dict__:
                    try:
                        del __dict__[key]
                    except KeyError:
                        pass  # didn't have anything in this thread

        return __del__ 
Example 68
Project: cdn-cache   Author: gbrunacci   File: test_bus.py    Apache License 2.0 5 votes vote down vote up
def test_block(self):
        b = wspbus.Bus()
        self.log(b)

        def f():
            time.sleep(0.2)
            b.exit()

        def g():
            time.sleep(0.4)
        threading.Thread(target=f).start()
        threading.Thread(target=g).start()
        threads = [t for t in threading.enumerate() if not get_daemon(t)]
        self.assertEqual(len(threads), 3)

        b.block()

        # The block method MUST wait for the EXITING state.
        self.assertEqual(b.state, b.states.EXITING)
        # The block method MUST wait for ALL non-main, non-daemon threads to
        # finish.
        threads = [t for t in threading.enumerate() if not get_daemon(t)]
        self.assertEqual(len(threads), 1)
        # The last message will mention an indeterminable thread name; ignore
        # it
        self.assertEqual(self._log_entries[:-1],
                         ['Bus STOPPING', 'Bus STOPPED',
                          'Bus EXITING', 'Bus EXITED',
                          'Waiting for child threads to terminate...']) 
Example 69
Project: MoePhoto   Author: opteroncx   File: mt_download.py    Apache License 2.0 5 votes vote down vote up
def download_file(url, num_thread = 5,fname = ''):

    r = requests.head(url)
    try:
        if fname == '':
            file_name = url.split('/')[-1]
        else:
            file_name = fname
        file_size = int(r.headers['content-length'])   # Content-Length获得文件主体的大小,当http服务器使用Connection:keep-alive时,不支持Content-Length
    except:
        print("检查URL,或不支持多线程下载")
        return

    #  创建一个和要下载文件一样大小的文件
    fp = open(file_name, "wb")
    fp.truncate(file_size)
    fp.close()
    # 启动多线程写文件
    part = file_size // num_thread  # 如果不能整除,最后一块应该多几个字节
    for i in range(num_thread):
        start = part * i
        if i == num_thread - 1:   # 最后一块
            end = file_size
        else:
            end = start + part

        t = threading.Thread(target=Handler, kwargs={'start': start, 'end': end, 'url': url, 'filename': file_name})
        t.setDaemon(True)
        t.start()

    # 等待所有线程下载完成
    main_thread = threading.current_thread()
    for t in threading.enumerate():
        if t is main_thread:
            continue
        t.join()
    print('%s 下载完成' % file_name) 
Example 70
Project: avocado-vt   Author: avocado-framework   File: virtio_console_guest.py    GNU General Public License v2.0 5 votes vote down vote up
def is_alive():
    """
    Check is only main thread is alive and if guest react.
    """
    if ((os_linux and (threading.activeCount() == 2)) or
            ((not os_linux) and (threading.activeCount() == 1))):
        print("PASS: Guest is ok no thread alive")
    else:
        threads = ""
        for thread in threading.enumerate():
            threads += thread.name + ", "
        print("FAIL: On guest run thread. Active thread:" + threads) 
Example 71
Project: ARPET   Author: juliagarriga   File: ps_drone.py    MIT License 4 votes vote down vote up
def startup(self):
		# Check for drone in the network and wake it up
		try:
			socket.socket().connect((self.DroneIP, 21))
			socket.socket().close()
		except:
			self.printRed()
			print "Drone is not online"
			self.printDefault()
			sys.exit(9)

		# Internal variables
		self.__CmdCounter = 3											# as there are two raw commands, send next steps
		self.__calltime = 	0											# to get some time-values to debug

		#send the first four initial-commands to the drone
		self.__sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)	# Open network connection
		self.__sock.setblocking(0)										# Network should not block
		self.__sendrawmsg("\r")											# Wakes up command port
		time.sleep(0.01)
		self.__sendrawmsg("AT*PMODE=1,2\rAT*MISC=2,2,20,2000,3000\r")	# Initialising drone as sniffed from datastream demo-tool to AR.Drone

		##### Initialising timed thread(s) for drone communication
		# Opening NavData- and Video- Processes
		self.__VidPipePath = tempfile.gettempdir()+"/dronevid-"+str(threading.enumerate()[0])[-12:-2]+"-"+str(time.time())[-7:].replace(".","")+".h264"
		self.__net_pipes = []
		self.__NavData_pipe, navdataChild_pipe 		  = multiprocessing.Pipe()
		self.__Video_pipe,   videoChild_pipe          = multiprocessing.Pipe()
		self.__vdecode_pipe, self.__vdecodeChild_pipe = multiprocessing.Pipe()

		self.__NavDataProcess = multiprocessing.Process( target=mainloopND, args=(self.DroneIP,self.NavDataPort,navdataChild_pipe,os.getpid()))
		self.__NavDataProcess.start()
		self.__VideoProcess =   multiprocessing.Process( target=mainloopV, args=(self.DroneIP,self.VideoPort,self.__VidPipePath,videoChild_pipe,os.getpid()))
		self.__VideoProcess.start()
		self.__vDecodeProcess = multiprocessing.Process( target=vDecode, args=(self.__VidPipePath,self.__vdecodeChild_pipe,os.getpid()))
		# There is a third process called "self.__vDecodeProcess" for decoding video, initiated and started around line 880

		# Final settings
		self.useDemoMode(True) 		# This entry is necessary for the drone's firmware, otherwise the NavData contains just header and footer
		self.setConfig("custom:session_id","-all")	
		self.getNDpackage(["demo"])

		time.sleep(1)
		#setup Network-thread
		while not self.__receiveDataRunning or not self.__sendConfigRunning or len(self.__ConfigQueue):		# sometimes they would not start why ever, so TK has to double-check
			if not self.__receiveDataRunning:
				self.__threadReceiveData=threading.Thread(target=self.__receiveData)
				self.__threadReceiveData.start()
				time.sleep(0.05)
			if not self.__sendConfigRunning:
				self.__threadSendConfig=threading.Thread(target=self.__sendConfig)
				self.__threadSendConfig.start()
				time.sleep(0.05)
			time.sleep(0.01)
		
	###### Clean Shutdown 
Example 72
Project: vscode-mayapy   Author: FXTD-ODYSSEY   File: conftest.py    MIT License 4 votes vote down vote up
def _start_monitoring_threads():
    # After the session finishes, wait 20 seconds to see if everything finished properly
    # and if it doesn't report an error.
    global _started_monitoring_threads
    if _started_monitoring_threads:
        return

    _started_monitoring_threads = True
    import threading
    if hasattr(sys, '_current_frames') and hasattr(threading, 'enumerate'):
        import time
        import traceback

        class DumpThreads(threading.Thread):

            def run(self):
                time.sleep(20)

                thread_id_to_name = {}
                try:
                    for t in threading.enumerate():
                        thread_id_to_name[t.ident] = '%s  (daemon: %s)' % (t.name, t.daemon)
                except:
                    pass

                stack_trace = [
                    '===============================================================================',
                    'pydev pyunit runner: Threads still found running after tests finished',
                    '================================= Thread Dump =================================']

                for thread_id, stack in sys._current_frames().items():
                    stack_trace.append('\n-------------------------------------------------------------------------------')
                    stack_trace.append(" Thread %s" % thread_id_to_name.get(thread_id, thread_id))
                    stack_trace.append('')

                    if 'self' in stack.f_locals:
                        sys.stderr.write(str(stack.f_locals['self']) + '\n')

                    for filename, lineno, name, line in traceback.extract_stack(stack):
                        stack_trace.append(' File "%s", line %d, in %s' % (filename, lineno, name))
                        if line:
                            stack_trace.append("   %s" % (line.strip()))
                stack_trace.append('\n=============================== END Thread Dump ===============================')
                sys.stderr.write('\n'.join(stack_trace))

                # Force thread run to finish
                import os
                os._exit(123)

        dump_current_frames_thread = DumpThreads()
        dump_current_frames_thread.setDaemon(True)  # Daemon so that this thread doesn't halt it!
        dump_current_frames_thread.start() 
Example 73
Project: vscode-mayapy   Author: FXTD-ODYSSEY   File: conftest.py    MIT License 4 votes vote down vote up
def bytes2human(n, format='%(value).1f %(symbol)s', symbols='customary'):
    """
    Bytes-to-human / human-to-bytes converter.
    Based on: http://goo.gl/kTQMs
    Working with Python 2.x and 3.x.

    Author: Giampaolo Rodola' <g.rodola [AT] gmail [DOT] com>
    License: MIT
    """

    """
    Convert n bytes into a human readable string based on format.
    symbols can be either "customary", "customary_ext", "iec" or "iec_ext",
    see: http://goo.gl/kTQMs

      >>> bytes2human(0)
      '0.0 B'
      >>> bytes2human(0.9)
      '0.0 B'
      >>> bytes2human(1)
      '1.0 B'
      >>> bytes2human(1.9)
      '1.0 B'
      >>> bytes2human(1024)
      '1.0 K'
      >>> bytes2human(1048576)
      '1.0 M'
      >>> bytes2human(1099511627776127398123789121)
      '909.5 Y'

      >>> bytes2human(9856, symbols="customary")
      '9.6 K'
      >>> bytes2human(9856, symbols="customary_ext")
      '9.6 kilo'
      >>> bytes2human(9856, symbols="iec")
      '9.6 Ki'
      >>> bytes2human(9856, symbols="iec_ext")
      '9.6 kibi'

      >>> bytes2human(10000, "%(value).1f %(symbol)s/sec")
      '9.8 K/sec'

      >>> # precision can be adjusted by playing with %f operator
      >>> bytes2human(10000, format="%(value).5f %(symbol)s")
      '9.76562 K'
    """
    n = int(n)
    if n < 0:
        raise ValueError("n < 0")
    symbols = SYMBOLS[symbols]
    prefix = {}
    for i, s in enumerate(symbols[1:]):
        prefix[s] = 1 << (i + 1) * 10
    for symbol in reversed(symbols[1:]):
        if n >= prefix[symbol]:
            value = float(n) / prefix[symbol]
            return format % locals()
    return format % dict(symbol=symbols[0], value=n) 
Example 74
Project: blender_command_port   Author: p4vv37   File: command_port.py    MIT License 4 votes vote down vote up
def run(self):
        # ---- Run while main thread of Blender is Alive
        # This feels a little bit hacky, but that was the most reliable way I've found
        # There's no method or callback that Blender calls on exit that could be used to close the port
        # So I'm detecting if a main thread of blender did finish working.
        # If it did, then I'm breaking the loop and closing the port.
        threads = threading.enumerate()
        while any([t.name == "MainThread" and t.isAlive() for t in threads]):
            if not self.do_run:
                # ---- Break also if user requested closing the port.
                print("do_run is False")
                break
            self.socket.listen(self.max_connections)
            try:
                connection, address = self.socket.accept()
                data = connection.recv(self.buffersize)
                size = sys.getsizeof(data)
                if size >= self.buffersize:
                    print("The length of input is probably too long: {}".format(size))
                if size >= 0:
                    command = data.decode()
                    self.commands_queue.put(command)
                    if self.redirect_output or self.return_result:
                        while True:
                            try:
                                output = self.output_queue.get_nowait()
                            except Empty:
                                continue
                            else:
                                if isinstance(output, ResultContainer):
                                    if self.result_as_json:
                                        result = json.dumps(output.value)
                                    else:
                                        result = str(output.value)
                                    connection.sendall(result.encode())
                                    break
                                elif output and output != "\n":
                                    connection.sendall(output.encode())
                    else:
                        connection.sendall('OK'.encode())
                    connection.shutdown(socket.SHUT_RDWR)
                    connection.close()
            except socket.timeout:
                pass
        self.socket.close()
        print("Closing the socket")
        return 
Example 75
Project: sawtooth-core   Author: hyperledger   File: core.py    Apache License 2.0 4 votes vote down vote up
def stop(self):
        self._gossip.stop()
        self._component_dispatcher.stop()
        self._network_dispatcher.stop()
        self._network_service.stop()

        self._component_service.stop()

        self._consensus_service.stop()
        self._consensus_dispatcher.stop()

        self._network_thread_pool.shutdown(wait=True)
        self._component_thread_pool.shutdown(wait=True)
        self._client_thread_pool.shutdown(wait=True)
        self._sig_pool.shutdown(wait=True)

        self._transaction_executor.stop()
        self._context_manager.stop()

        self._block_publisher.stop()
        self._chain_controller.stop()
        self._block_validator.stop()

        threads = threading.enumerate()

        # This will remove the MainThread, which will exit when we exit with
        # a sys.exit() or exit of main().
        threads.remove(threading.current_thread())

        while threads:
            if len(threads) < 4:
                LOGGER.info(
                    "remaining threads: %s",
                    ", ".join(
                        ["{} ({})".format(x.name, x.__class__.__name__)
                         for x in threads]))
            for t in threads.copy():
                if not t.is_alive():
                    t.join()
                    threads.remove(t)
                if threads:
                    time.sleep(1)

        LOGGER.info("All threads have been stopped and joined") 
Example 76
Project: ekklesia-portal   Author: Piratenpartei   File: runserver.py    GNU Affero General Public License v3.0 4 votes vote down vote up
def stackdump_setup():
    import codecs
    import logging
    import sys
    logg = logging.getLogger(__name__)
    import threading
    import traceback
    try:
        import IPython.core.ultratb as ultratb
    except BaseException:
        ultratb = None

    if ultratb is None:
        logg.warn("IPython not installed, stack dumps not available!")
    else:
        logg.info("IPython installed, write stack dumps to tmpdir with: `kill -QUIT <ekklesia_portal_pid>`")

        def dumpstacks(signal, frame):
            print("dumping stack")
            filepath = os.path.join(tmpdir, "ekklesia_portal_threadstatus")
            id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
            full = ["-" * 80]
            tb_formatter = ultratb.ListTB(color_scheme="Linux")
            for thread_id, stack in sys._current_frames().items():
                thread_name = id2name.get(thread_id, "")
                stacktrace = traceback.extract_stack(stack)
                stb = tb_formatter.structured_traceback(Exception, Exception(), stacktrace)[8:-1]
                if stb:
                    formatted_trace = tb_formatter.stb2text(stb).strip()
                    with codecs.open("{}.{}".format(filepath, thread_id), "w", encoding='utf8') as wf:
                        wf.write("\n{}".format(formatted_trace))
                    if len(stb) > 4:
                        short_stb = stb[:2] + ["..."] + stb[-2:]
                    else:
                        short_stb = stb
                    formatted_trace_short = tb_formatter.stb2text(short_stb).strip()
                    full.append("# Thread: %s(%d)" % (thread_name, thread_id))
                    full.append(formatted_trace_short)
                    full.append("-" * 80)

            with codecs.open(filepath, "w", encoding='utf8') as wf:
                wf.write("\n".join(full))

        import signal
        signal.signal(signal.SIGQUIT, dumpstacks) 
Example 77
Project: pdbclone   Author: corpusops   File: pdb.py    GNU General Public License v2.0 4 votes vote down vote up
def _do_thread(self, arg, current_frames, tlist):
        if not arg:
            self.message('   {:3} {:18} {:16} {}'.format(
                    'Nb', 'Name', 'Identifier', 'Stack entry'))
            for (nb, t) in enumerate(tlist, start=1):
                prefix = '+' if t is self.pdb_thread else ' '
                prefix += '*' if t is self.current_thread else ' '
                if t is self.pdb_thread:
                    frame = self.pdb_toplevel_frame
                else:
                    frame = current_frames.get(t.ident)
                if frame:
                    stack_entry = self.format_stack_entry(
                            (frame, frame.f_lineno), line_prefix).split('\n')
                    self.message('{} {:3d} {:18} {:16d} {}\n{:43}{}'.format(
                        prefix, nb, t.name, t.ident, stack_entry[0],
                        '', stack_entry[1]))
                else:
                    self.message('{} {:3d} {:18} {:16d} {}'.format(
                        prefix, nb, t.name, t.ident, 'Thread not active.'))
            return

        t = frame = None
        try:
            idx = int(arg)
            if idx > 0:
                t = tlist[idx - 1]
        except (ValueError, IndexError):
            pass
        if t is None:
            self.error(
                'Invalid thread number, must be in a range from 1 to {:d}.'
                .format(len(tlist)))
        elif t is self.pdb_thread:
            frame = self.pdb_toplevel_frame
        else:
            try:
                frame = current_frames[t.ident]
            except IndexError:
                self.error('Internal error, the thread "{}" is not active.'
                .format(t.name))
        if frame:
            self.setup(frame, None)
            self.current_thread = t
            self.print_stack_entry(self.stack[self.curindex]) 
Example 78
Project: pdbclone   Author: corpusops   File: pdb.py    GNU General Public License v2.0 4 votes vote down vote up
def do_thread(self, arg):
        """th(read) [threadnumber]
        Without argument, display a summary of all active threads.
        The summary prints for each thread:
           1. the thread number assigned by pdb
           2. the thread name
           3. the python thread identifier
           4. the current stack frame summary for that thread
        An asterisk '*' to the left of the pdb thread number indicates the
        current thread, a plus sign '+' indicates the thread being traced by
        pdb.

        With a pdb thread number as argument, make this thread the current
        thread. The 'where', 'up' and 'down' commands apply now to the frame
        stack of this thread. The current scope is now the frame currently
        executed by this thread at the time the command is issued and the
        'list', 'll', 'args', 'p', 'pp', 'source' and 'interact' commands are
        run in the context of that frame. Note that this frame may bear no
        relationship (for a non-deadlocked thread) to that thread's current
        activity by the time you are examining the frame.
        This command does not stop the thread.
        """
        # Import the threading module in the main interpreter to get an
        # enumeration of the main interpreter threads.
        if PY3:
            try:
                import threading
            except ImportError:
                import dummy_threading as threading
        else:
            # Do not use relative import detection to avoid the RuntimeWarning:
            # Parent module 'pdb_clone' not found while handling absolute
            # import.
            try:
                threading = __import__('threading', level=0)
            except ImportError:
                threading = __import__('dummy_threading', level=0)


        if not self.pdb_thread:
            self.pdb_thread = threading.current_thread()
        if not self.current_thread:
            self.current_thread = self.pdb_thread
        current_frames = sys._current_frames()
        tlist = sorted(threading.enumerate(), key=attrgetter('name', 'ident'))
        try:
            self._do_thread(arg, current_frames, tlist)
        finally:
            # For some reason this local must be explicitly deleted in order
            # to release the subinterpreter.
            del current_frames 
Example 79
Project: cdn-cache   Author: gbrunacci   File: wspbus.py    Apache License 2.0 4 votes vote down vote up
def block(self, interval=0.1):
        """Wait for the EXITING state, KeyboardInterrupt or SystemExit.

        This function is intended to be called only by the main thread.
        After waiting for the EXITING state, it also waits for all threads
        to terminate, and then calls os.execv if self.execv is True. This
        design allows another thread to call bus.restart, yet have the main
        thread perform the actual execv call (required on some platforms).
        """
        try:
            self.wait(states.EXITING, interval=interval, channel='main')
        except (KeyboardInterrupt, IOError):
            # The time.sleep call might raise
            # "IOError: [Errno 4] Interrupted function call" on KBInt.
            self.log('Keyboard Interrupt: shutting down bus')
            self.exit()
        except SystemExit:
            self.log('SystemExit raised: shutting down bus')
            self.exit()
            raise

        # Waiting for ALL child threads to finish is necessary on OS X.
        # See https://bitbucket.org/cherrypy/cherrypy/issue/581.
        # It's also good to let them all shut down before allowing
        # the main thread to call atexit handlers.
        # See https://bitbucket.org/cherrypy/cherrypy/issue/751.
        self.log("Waiting for child threads to terminate...")
        for t in threading.enumerate():
            # Validate the we're not trying to join the MainThread
            # that will cause a deadlock and the case exist when
            # implemented as a windows service and in any other case
            # that another thread executes cherrypy.engine.exit()
            if (
                    t != threading.currentThread() and
                    t.isAlive() and
                    not isinstance(t, threading._MainThread)
            ):
                # Note that any dummy (external) threads are always daemonic.
                if hasattr(threading.Thread, "daemon"):
                    # Python 2.6+
                    d = t.daemon
                else:
                    d = t.isDaemon()
                if not d:
                    self.log("Waiting for thread %s." % t.getName())
                    t.join()

        if self.execv:
            self._do_execv() 
Example 80
Project: PyZwaver   Author: robertmuth   File: example_simple.py    GNU General Public License v3.0 4 votes vote down vote up
def main():
    parser = argparse.ArgumentParser(description='Process some integers.')
    parser.add_argument('--serial_port', type=str,
                        default="/dev/ttyUSB0",
                        help='The USB serial device representing the Z-Wave controller stick. ' +
                             'Common settings are: dev/ttyUSB0, dev/ttyACM0')

    parser.add_argument('--verbosity', type=int,
                        default=logging.ERROR,
                        help='Lower numbers mean more verbosity')

    parser.add_argument('--curses',
                        default=False,
                        action='store_true',
                        help='Use curses for rendering. Make sure your terminal '
                             'is at least 200 chars wide.')

    args = parser.parse_args()
    # note: this makes sure we have at least one handler
    logging.basicConfig(level=logging.ERROR)
    logger = logging.getLogger()
    logger.setLevel(args.verbosity)

    for h in logger.handlers:
        h.setFormatter(MyFormatter())

    logging.info("opening serial: [%s]", args.serial_port)
    device = MakeSerialDevice(args.serial_port)

    driver = Driver(device)
    controller = Controller(driver, pairing_timeout_secs=60)
    controller.Initialize()
    success = controller.WaitUntilInitialized(2)
    if not success:
        logging.error("could not initialize controller")
        driver.Terminate()
        print(list(threading.enumerate()))
        return 1
    controller.UpdateRoutingInfo()
    time.sleep(1)
    Banner("Initialized Controller")
    print(controller)
    if args.curses:
        try:
            curses.wrapper(InitializeDevices, driver, controller)
        except Exception as e:
            Banner(str(e))
    else:
        InitializeDevices(None, driver, controller)
    driver.Terminate()
    return 0