Python threading.BoundedSemaphore() Examples

The following are code examples for showing how to use threading.BoundedSemaphore(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: pyblish-win   Author: pyblish   File: test_threading.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_BoundedSemaphore_limit(self):
        # BoundedSemaphore should raise ValueError if released too often.
        for limit in range(1, 10):
            bs = threading.BoundedSemaphore(limit)
            threads = [threading.Thread(target=bs.acquire)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            threads = [threading.Thread(target=bs.release)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            self.assertRaises(ValueError, bs.release) 
Example 2
Project: NiujiaoDebugger   Author: MrSrc   File: test_threading.py    GNU General Public License v3.0 6 votes vote down vote up
def test_BoundedSemaphore_limit(self):
        # BoundedSemaphore should raise ValueError if released too often.
        for limit in range(1, 10):
            bs = threading.BoundedSemaphore(limit)
            threads = [threading.Thread(target=bs.acquire)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            threads = [threading.Thread(target=bs.release)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            self.assertRaises(ValueError, bs.release) 
Example 3
Project: ironpython2   Author: IronLanguages   File: test_threading.py    Apache License 2.0 6 votes vote down vote up
def test_BoundedSemaphore_limit(self):
        # BoundedSemaphore should raise ValueError if released too often.
        for limit in range(1, 10):
            bs = threading.BoundedSemaphore(limit)
            threads = [threading.Thread(target=bs.acquire)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            threads = [threading.Thread(target=bs.release)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            self.assertRaises(ValueError, bs.release) 
Example 4
Project: Blockly-rduino-communication   Author: technologiescollege   File: test_threading.py    GNU General Public License v3.0 6 votes vote down vote up
def test_BoundedSemaphore_limit(self):
        # BoundedSemaphore should raise ValueError if released too often.
        for limit in range(1, 10):
            bs = threading.BoundedSemaphore(limit)
            threads = [threading.Thread(target=bs.acquire)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            threads = [threading.Thread(target=bs.release)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            self.assertRaises(ValueError, bs.release) 
Example 5
Project: cqp-sdk-for-py37-native   Author: crud-boy   File: test_threading.py    GNU General Public License v2.0 6 votes vote down vote up
def test_BoundedSemaphore_limit(self):
        # BoundedSemaphore should raise ValueError if released too often.
        for limit in range(1, 10):
            bs = threading.BoundedSemaphore(limit)
            threads = [threading.Thread(target=bs.acquire)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            threads = [threading.Thread(target=bs.release)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            self.assertRaises(ValueError, bs.release) 
Example 6
Project: oss-ftp   Author: aliyun   File: test_threading.py    MIT License 6 votes vote down vote up
def test_BoundedSemaphore_limit(self):
        # BoundedSemaphore should raise ValueError if released too often.
        for limit in range(1, 10):
            bs = threading.BoundedSemaphore(limit)
            threads = [threading.Thread(target=bs.acquire)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            threads = [threading.Thread(target=bs.release)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            self.assertRaises(ValueError, bs.release) 
Example 7
Project: OpenDoor   Author: stanislav-web   File: worker.py    GNU General Public License v3.0 6 votes vote down vote up
def __init__(self, queue, num_threads, timeout=0):
        """
        Init thread worker
        :param Queue.Queue queue: simple queue object
        :param int num_threads: threads numbers
        :param int timeout: delay timeout
        """

        super(Worker, self).__init__()
        self.__semaphore = BoundedSemaphore(num_threads)
        self.__event = Event()
        self.__event.set()
        self.__empty = False
        self.__running = True
        self.__queue = queue
        self.__timeout = timeout
        self.counter = 0 
Example 8
Project: OOD   Author: BUPT-AC   File: test_crawler.py    GNU General Public License v2.0 6 votes vote down vote up
def main():
	t1 = task("http://www.laurentluce.com/posts/python-threads-synchronization-locks-rlocks-semaphores-conditions-events-and-queues/")
	t2 = task("http://stackoverflow.com/questions/15651128/in-this-semaphore-example-is-it-necessary-to-lock-for-refill-and-buy")
	t3 = task("http://bbs.byr.cn/")
	event = Event()
	tasks = TaskQueue(event)
	pages = TaskQueue(None)
	tasks.add(t1)
	tasks.add(t2)
	tasks.add(t3)

	taskLock = BoundedSemaphore(tasks.numOfNewTasks)
	pageLock = BoundedSemaphore(1)

	f = open("test.txt",'a')

	Crawler1 = Crawler(tasks,taskLock,pages,pageLock,event,f)
	Crawler2 = Crawler(tasks,taskLock,pages,pageLock,event,f)
	Crawler1.start()
	Crawler2.start()
	Crawler1.join()
	Crawler2.join()
	f.close() 
Example 9
Project: amazon-qldb-driver-python   Author: awslabs   File: pooled_qldb_driver.py    Apache License 2.0 6 votes vote down vote up
def __init__(self, ledger_name, retry_limit=4, read_ahead=0, executor=None, region_name=None, verify=None,
                 endpoint_url=None, aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None,
                 config=None, boto3_session=None, pool_limit=0, timeout=DEFAULT_TIMEOUT_SECONDS):
        super().__init__(ledger_name, retry_limit, read_ahead, executor, region_name, verify, endpoint_url,
                         aws_access_key_id, aws_secret_access_key, aws_session_token, config, boto3_session)

        if pool_limit < 0:
            raise ValueError('Value for pool_limit cannot be negative.')
        if timeout < 0:
            raise ValueError('Value for timeout cannot be negative.')

        client_pool_limit = self._config.max_pool_connections
        if pool_limit == 0:
            self._pool_limit = client_pool_limit
        else:
            self._pool_limit = pool_limit

        if self._pool_limit > client_pool_limit:
            raise ValueError('The session pool limit given, {}, exceeds the limit set by the client, {}. Please lower '
                             'the limit and retry.'.format(str(self._pool_limit), str(client_pool_limit)))

        self._pool_permits = BoundedSemaphore(self._pool_limit)
        self._pool_permits_counter = AtomicInteger(self._pool_limit)
        self._pool = Queue()
        self._timeout = timeout 
Example 10
Project: senaite.impress   Author: senaite   File: decorators.py    GNU General Public License v2.0 6 votes vote down vote up
def synchronized(max_connections=2, verbose=0):
    """Synchronize function call via semaphore
    """
    semaphore = threading.BoundedSemaphore(max_connections, verbose=verbose)

    def inner(func):
        logger.debug("Semaphore for {} -> {}".format(func, semaphore))
        @wraps(func)
        def wrapper(*args, **kwargs):
            try:
                logger.info("==> {}::Acquire Semaphore ...".format(
                    func.__name__))
                semaphore.acquire()
                return func(*args, **kwargs)
            finally:
                logger.info("<== {}::Release Semaphore ...".format(
                    func.__name__))
                semaphore.release()

        return wrapper
    return inner 
Example 11
Project: aws-with   Author: aws-samples   File: workplan.py    Apache License 2.0 6 votes vote down vote up
def execute_work_plan(logger, options, commands_list):
    """ run through commands_list and run various commands in the thread pool """
    logger.info("Executing work plan across a thread pool of size: %s", options.threads)
    utils.GLOBALS["main_thread_lock"] = threading.Lock()
    utils.GLOBALS["thread_pool_lock"] = threading.BoundedSemaphore(options.threads)
    utils.GLOBALS["thread_count"] = len(commands_list)
    logger.debug("Locks created, task list size = %s", utils.GLOBALS["thread_count"])

    # obtain the main thread lock...
    logger.debug("Acquiring main thread lock")
    utils.GLOBALS["main_thread_lock"].acquire()

    for cmd in commands_list:
        logger.debug("waiting for next thread to be available")
        utils.GLOBALS["thread_pool_lock"].acquire()
        logger.debug("thread is available, starting thread")
        threading.Thread(target=commands.run_command, args=(logger, options, cmd, )).start()

    # block on the main thread lock being released...
    logger.debug("Blocking main thread, waiting on commands to finish")
    utils.GLOBALS["main_thread_lock"].acquire()
    logger.debug("Main thread lock released, working on output") 
Example 12
Project: oneshellcrack   Author: L-codes   File: oneshellcrack.py    GNU General Public License v3.0 6 votes vote down vote up
def main():
    global semaphore, sends

    signal.signal(signal.SIGINT, interrupt_handler)
    args = commandline()
    print(' ( Shell:{shell}, Numbers:{max_request}, Threads:{max_threads}, Retry:{max_retry} )\n'.format(**args.__dict__))

    semaphore = BoundedSemaphore(value=args.max_threads)
    stopwatch_start = time.time()
    for i, payload in enumerate(create_payload(args), 1):
        if attack:
            sends = i
            semaphore.acquire()
            t = Thread(target=crack, args=(i, args, payload))
            t.setDaemon(True)
            t.start()

    for _ in range(args.max_threads):
        semaphore.acquire()

    stopwatch = time.time() - stopwatch_start
    words = args.max_request * sends if sends else pwd_total
    speed = words / stopwatch if stopwatch else 0
    msg = '[Success] Password: {}'.format(pwd) if pwd else '[Failed] No password found'
    print('\n\n{msg}\n[Finish] {words} words in {stopwatch:.3f} seconds. ({speed:.0f} w/s)'.format(**locals())) 
Example 13
Project: lox   Author: BrianPugh   File: thread.py    MIT License 6 votes vote down vote up
def __init__(self, func, n_workers=50, daemon=None):
        """
        Creates the callable object for the 'thread' decorator.

        Parameters
        ----------
        func : function
            Function handle that each thread worker will execute.

        n_workers : int
            Maximum number of threads to invoke.

        """
        super().__init__(n_workers, func)

        self.job_queue = queue.Queue()  # Queue to put jobs on
        self.jobs_complete_lock = Lock()
        # Used to determine if all jobs have been completed
        self.job_lightswitch = LightSwitch(self.jobs_complete_lock)

        self.workers_sem = BoundedSemaphore(self.n_workers)
        self.daemon = daemon
        self.enable_auto_unpacking()

        self.clear() 
Example 14
Project: vseutil   Author: xvass   File: vseutil.py    GNU General Public License v2.0 6 votes vote down vote up
def copy_file(DAT2val,ip,user,password,sourcefile,destfile,outfile):
	semaphore = threading.BoundedSemaphore()
	semaphore.acquire()
	wnet_connect(ip, user, password)
	try:
		shutil.copy2(sourcefile,'\\\\' + str(ip) + '\\' + str(destfile) + '\\')
		print Fore.WHITE +"[*] DAT "+ "latest version " + Fore.YELLOW + "%s.0000 " % DAT2val + Fore.WHITE + " uploded..."
		if (outfile != None):
			log_to_file("[*] DAT "+ "latest version %s.0000 " % DAT2val + " uploded...",outfile)
		print Fore.WHITE + '[*] file ' + Fore.YELLOW + sourcefile + Fore.WHITE + ' copied to C:\Program Files\Common Files\McAfee\%s' % destfile
		if (outfile != None):
			log_to_file('[*] file' + sourcefile + ' copied to C:\Program Files\Common Files\McAfee\%s' % destfile,outfile)
	except IOError, e:
		if e.errno == 22:   				
			print Fore.WHITE + '[x] file did not copied to C:\Program Files\Common Files\McAfee\%s.' % destfile
			if (outfile != None):
				log_to_file('[x] file did not copied to C:\Program Files\Common Files\McAfee\%s.' % destfile,outfile) 
Example 15
Project: vseutil   Author: xvass   File: vseutil.py    GNU General Public License v2.0 6 votes vote down vote up
def checkServices(ip):
	semaphore = threading.BoundedSemaphore()
	semaphore.acquire()
	status1 = svcStatus( "McShield", unicode(ip))
	status2 = svcStatus( "McAfeeFramework", unicode(ip))
	if status1 == STOPPED and status2 != STOPPED:
		arg="win32service.SERVICE_ALL_ACCESS"	
		svcStart( "McShield",arg, unicode(ip))
	elif status1 != STOPPED and status2 == STOPPED:
		arg="win32service.SERVICE_ALL_ACCESS"
		svcStart( "McAfeeFramework",arg, unicode(ip))
	elif status1 == STOPPED and status2 == STOPPED:
		arg="win32service.SERVICE_ALL_ACCESS"
		svcStart( "McAfeeFramework",arg, unicode(ip))
		svcStart( "McShield",arg, unicode(ip))
	semaphore.release()	
	
# This function updates DAT files inside the C:\Program Files\Common Files\McAfee\Engine file. It requires McShield and McAfeeFramework to stop. 
# After the files has been copied,start services again. 
Example 16
Project: laozhao-python-codes   Author: archtaurus   File: pycode0x002B.py    GNU General Public License v3.0 6 votes vote down vote up
def __init__(self, device, resolution, fps):
        # Initialize the camera
        self.camera = pygame.camera.Camera(device, resolution)
        self.camera.start()

        # Set up a CameraSurface Buffer
        self.camera_surface = pygame.Surface(resolution)
        self.jpeg = ""
        self.jpeg_sema = threading.BoundedSemaphore()
        self.period = 1 / float(fps)
        self.stop = True

        # Prepare conditions
        self.frame_count = 0
        self.frame_available = threading.Condition()

        # Kick it off
        self.start_capture() 
Example 17
Project: Project-New-Reign---Nemesis-Main   Author: ShikyoKira   File: test_threading.py    GNU General Public License v3.0 6 votes vote down vote up
def test_BoundedSemaphore_limit(self):
        # BoundedSemaphore should raise ValueError if released too often.
        for limit in range(1, 10):
            bs = threading.BoundedSemaphore(limit)
            threads = [threading.Thread(target=bs.acquire)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            threads = [threading.Thread(target=bs.release)
                       for _ in range(limit)]
            for t in threads:
                t.start()
            for t in threads:
                t.join()
            self.assertRaises(ValueError, bs.release) 
Example 18
Project: bounded_pool_executor   Author: mowshon   File: __init__.py    MIT License 5 votes vote down vote up
def __init__(self, max_workers=None):
        super().__init__(max_workers)
        self.semaphore = multiprocessing.BoundedSemaphore(max_workers) 
Example 19
Project: bounded_pool_executor   Author: mowshon   File: __init__.py    MIT License 5 votes vote down vote up
def __init__(self, max_workers=None):
        super().__init__(max_workers)
        self.semaphore = threading.BoundedSemaphore(max_workers) 
Example 20
Project: pyblish-win   Author: pyblish   File: test_threading.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_various_ops(self):
        # This takes about n/3 seconds to run (about n/3 clumps of tasks,
        # times about 1 second per clump).
        NUMTASKS = 10

        # no more than 3 of the 10 can run at once
        sema = threading.BoundedSemaphore(value=3)
        mutex = threading.RLock()
        numrunning = Counter()

        threads = []

        for i in range(NUMTASKS):
            t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
            threads.append(t)
            self.assertEqual(t.ident, None)
            self.assertTrue(re.match('<TestThread\(.*, initial\)>', repr(t)))
            t.start()

        if verbose:
            print 'waiting for all tasks to complete'
        for t in threads:
            t.join(NUMTASKS)
            self.assertTrue(not t.is_alive())
            self.assertNotEqual(t.ident, 0)
            self.assertFalse(t.ident is None)
            self.assertTrue(re.match('<TestThread\(.*, \w+ -?\d+\)>', repr(t)))
        if verbose:
            print 'all tasks done'
        self.assertEqual(numrunning.get(), 0) 
Example 21
Project: pyblish-win   Author: pyblish   File: test_contextlib.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def testWithBoundedSemaphore(self):
        lock = threading.BoundedSemaphore()
        def locked():
            if lock.acquire(False):
                lock.release()
                return False
            else:
                return True
        self.boilerPlate(lock, locked)

# This is needed to make the test actually run under regrtest.py! 
Example 22
Project: vulnerability-engine   Author: RedHatInsights   File: bounded_executor.py    GNU General Public License v2.0 5 votes vote down vote up
def __init__(self, max_queue_size, max_workers=None):
        self.semaphore = BoundedSemaphore(max_queue_size)
        self.executor = ThreadPoolExecutor(max_workers=max_workers) 
Example 23
Project: subtake   Author: kp625544   File: sublist3r.py    GNU General Public License v2.0 5 votes vote down vote up
def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
        subdomains = subdomains or []
        base_url = 'https://dnsdumpster.com/'
        self.live_subdomains = []
        self.engine_name = "DNSdumpster"
        self.threads = 70
        self.lock = threading.BoundedSemaphore(value=self.threads)
        self.q = q
        super(DNSdumpster, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
        return 
Example 24
Project: subtake   Author: kp625544   File: sublist3r.py    GNU General Public License v2.0 5 votes vote down vote up
def __init__(self, subdomains, ports):
        self.subdomains = subdomains
        self.ports = ports
        self.threads = 20
        self.lock = threading.BoundedSemaphore(value=self.threads) 
Example 25
Project: pySymProxy   Author: inbilla   File: objectpool.py    MIT License 5 votes vote down vote up
def __init__(self, maxSize, objectType, *args):
        self._semaphore = threading.BoundedSemaphore(maxSize)
        self._queue = Queue.Queue()

        for i in range(0, maxSize):
            self._queue.put(apply(objectType, args)) 
Example 26
Project: glastonbury-tickets   Author: danpilch   File: tickets.py    MIT License 5 votes vote down vote up
def __init__(self, base_url, proxy_file, thread_pool):
        self.threads = []
        self.thread_pool = thread_pool
        self.base_url = base_url
        self.proxy_list = [line.strip() for line in open(proxy_file, 'r')]
        # Shuffle proxies
        random.shuffle(self.proxy_list)
        self.proxy_efficieny = {}
        self.proxy_efficieny_semaphore = threading.BoundedSemaphore()
        self.found_valid_proxy = False
        with open("settings.yml", 'r') as f:
            self.settings = yaml.load(f) 
Example 27
Project: NiujiaoDebugger   Author: MrSrc   File: test_threading.py    GNU General Public License v3.0 5 votes vote down vote up
def test_various_ops(self):
        # This takes about n/3 seconds to run (about n/3 clumps of tasks,
        # times about 1 second per clump).
        NUMTASKS = 10

        # no more than 3 of the 10 can run at once
        sema = threading.BoundedSemaphore(value=3)
        mutex = threading.RLock()
        numrunning = Counter()

        threads = []

        for i in range(NUMTASKS):
            t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
            threads.append(t)
            self.assertIsNone(t.ident)
            self.assertRegex(repr(t), r'^<TestThread\(.*, initial\)>$')
            t.start()

        if verbose:
            print('waiting for all tasks to complete')
        for t in threads:
            t.join()
            self.assertFalse(t.is_alive())
            self.assertNotEqual(t.ident, 0)
            self.assertIsNotNone(t.ident)
            self.assertRegex(repr(t), r'^<TestThread\(.*, stopped -?\d+\)>$')
        if verbose:
            print('all tasks done')
        self.assertEqual(numrunning.get(), 0) 
Example 28
Project: NiujiaoDebugger   Author: MrSrc   File: test_contextlib.py    GNU General Public License v3.0 5 votes vote down vote up
def testWithBoundedSemaphore(self):
        lock = threading.BoundedSemaphore()
        def locked():
            if lock.acquire(False):
                lock.release()
                return False
            else:
                return True
        self.boilerPlate(lock, locked) 
Example 29
Project: simple_gpu_scheduler   Author: ExpectationMax   File: scheduler.py    GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, available_gpus):
        """Initialize GPU manager.

        Args:
            available_gpus: GPU ids of gpus that can be used.

        """
        self.semaphore = threading.BoundedSemaphore(len(available_gpus))
        self.gpu_list_lock = threading.Lock()
        self.available_gpus = list(available_gpus) 
Example 30
Project: Alexa-Ecovacs   Author: EtienneMD   File: stream.py    MIT License 5 votes vote down vote up
def __init__(self, xmpp, sid, block_size, jid, peer, window_size=1, use_messages=False):
        self.xmpp = xmpp
        self.sid = sid
        self.block_size = block_size
        self.window_size = window_size
        self.use_messages = use_messages

        if jid is None:
            jid = xmpp.boundjid
        self.self_jid = jid
        self.peer_jid = peer

        self.send_seq = -1
        self.recv_seq = -1

        self._send_seq_lock = threading.Lock()
        self._recv_seq_lock = threading.Lock()

        self.stream_started = threading.Event()
        self.stream_in_closed = threading.Event()
        self.stream_out_closed = threading.Event()

        self.recv_queue = Queue()

        self.send_window = threading.BoundedSemaphore(value=self.window_size)
        self.window_ids = set()
        self.window_empty = threading.Event()
        self.window_empty.set() 
Example 31
Project: Smhdk   Author: p4kl0nc4t   File: app.py    MIT License 5 votes vote down vote up
def before_req():
    if time.time() - app.init_time >= 2*60*60:
        app.cache = {}
        app.init_time = time.time()
    if f.request.endpoint in ['query', 'get_dl']:
        ip_addr = f.request.remote_addr
        if ip_addr not in app.client_bsemaphores:
            app.client_bsemaphores[ip_addr] = threading.BoundedSemaphore(
                3)
        app.client_bsemaphores[ip_addr].acquire() 
Example 32
Project: ironpython2   Author: IronLanguages   File: test_threading.py    Apache License 2.0 5 votes vote down vote up
def test_various_ops(self):
        # This takes about n/3 seconds to run (about n/3 clumps of tasks,
        # times about 1 second per clump).
        NUMTASKS = 10

        # no more than 3 of the 10 can run at once
        sema = threading.BoundedSemaphore(value=3)
        mutex = threading.RLock()
        numrunning = Counter()

        threads = []

        for i in range(NUMTASKS):
            t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
            threads.append(t)
            self.assertIsNone(t.ident)
            self.assertRegexpMatches(repr(t), r'^<TestThread\(.*, initial\)>$')
            t.start()

        if verbose:
            print 'waiting for all tasks to complete'
        for t in threads:
            t.join(NUMTASKS)
            self.assertFalse(t.is_alive())
            self.assertNotEqual(t.ident, 0)
            self.assertIsNotNone(t.ident)
            self.assertRegexpMatches(repr(t), r'^<TestThread\(.*, \w+ -?\d+\)>$')
        if verbose:
            print 'all tasks done'
        self.assertEqual(numrunning.get(), 0) 
Example 33
Project: ironpython2   Author: IronLanguages   File: test_contextlib.py    Apache License 2.0 5 votes vote down vote up
def testWithBoundedSemaphore(self):
        lock = threading.BoundedSemaphore()
        def locked():
            if lock.acquire(False):
                lock.release()
                return False
            else:
                return True
        self.boilerPlate(lock, locked)

# This is needed to make the test actually run under regrtest.py! 
Example 34
Project: NewBanchoo   Author: int-and-his-friends   File: dl.py    MIT License 5 votes vote down vote up
def test_mutil_down(sid, save_path):
    url = 'http://osu.ppy.sh/d/' + str(sid)
    # filename = url.split('/')[-1]
    filename = save_path + '/' + str(sid) + '.osz'
    if os.path.exists(filename):
        return filename
    urlr1 = requests.head(url,cookies=read_cookies(OSU_COOKIE))
    urlr = requests.head(urlr1.headers['Location'])
    print(urlr.headers)    
    filesize = int(urlr.headers['Content-Length'])

    url = urlr.url
    threadnum = 4
    threading.BoundedSemaphore(threadnum)
    step = filesize // threadnum
    mtd_list = []
    start = 0
    end = -1

    tempf = open(filename,'w')
    tempf.close()
    with open(filename,'rb+') as  f:
        fileno = f.fileno()
        while end < filesize -1:
            start = end +1
            end = start + step -1
            if end > filesize:
                end = filesize
            dup = os.dup(fileno)
            fd = os.fdopen(dup,'rb+',-1)
            t = MulThreadDownload(url,start,end,fd)
            t.start()
            mtd_list.append(t)

        for i in  mtd_list:
            i.join() 
Example 35
Project: Blockly-rduino-communication   Author: technologiescollege   File: test_threading.py    GNU General Public License v3.0 5 votes vote down vote up
def test_various_ops(self):
        # This takes about n/3 seconds to run (about n/3 clumps of tasks,
        # times about 1 second per clump).
        NUMTASKS = 10

        # no more than 3 of the 10 can run at once
        sema = threading.BoundedSemaphore(value=3)
        mutex = threading.RLock()
        numrunning = Counter()

        threads = []

        for i in range(NUMTASKS):
            t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
            threads.append(t)
            self.assertEqual(t.ident, None)
            self.assertTrue(re.match('<TestThread\(.*, initial\)>', repr(t)))
            t.start()

        if verbose:
            print('waiting for all tasks to complete')
        for t in threads:
            t.join()
            self.assertTrue(not t.is_alive())
            self.assertNotEqual(t.ident, 0)
            self.assertFalse(t.ident is None)
            self.assertTrue(re.match('<TestThread\(.*, stopped -?\d+\)>',
                                     repr(t)))
        if verbose:
            print('all tasks done')
        self.assertEqual(numrunning.get(), 0) 
Example 36
Project: Blockly-rduino-communication   Author: technologiescollege   File: test_contextlib.py    GNU General Public License v3.0 5 votes vote down vote up
def testWithBoundedSemaphore(self):
        lock = threading.BoundedSemaphore()
        def locked():
            if lock.acquire(False):
                lock.release()
                return False
            else:
                return True
        self.boilerPlate(lock, locked) 
Example 37
Project: Yuki-Chan-The-Auto-Pentest   Author: Yukinoshita47   File: sublist3r.py    MIT License 5 votes vote down vote up
def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
        subdomains = subdomains or []
        base_url = 'https://dnsdumpster.com/'
        self.live_subdomains = []
        self.engine_name = "DNSdumpster"
        self.threads = 70
        self.lock = threading.BoundedSemaphore(value=self.threads)
        self.q = q
        super(DNSdumpster, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
        return 
Example 38
Project: Yuki-Chan-The-Auto-Pentest   Author: Yukinoshita47   File: sublist3r.py    MIT License 5 votes vote down vote up
def __init__(self, subdomains, ports):
        self.subdomains = subdomains
        self.ports = ports
        self.threads = 20
        self.lock = threading.BoundedSemaphore(value=self.threads) 
Example 39
Project: ITWSV   Author: penetrate2hack   File: sublist3r.py    MIT License 5 votes vote down vote up
def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
        subdomains = subdomains or []
        base_url = 'https://dnsdumpster.com/'
        self.live_subdomains = []
        self.engine_name = "DNSdumpster"
        self.threads = 70
        self.lock = threading.BoundedSemaphore(value=self.threads)
        self.q = q
        super(DNSdumpster, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
        return 
Example 40
Project: ITWSV   Author: penetrate2hack   File: sublist3r.py    MIT License 5 votes vote down vote up
def __init__(self, subdomains, ports):
        self.subdomains = subdomains
        self.ports = ports
        self.threads = 20
        self.lock = threading.BoundedSemaphore(value=self.threads) 
Example 41
Project: cqp-sdk-for-py37-native   Author: crud-boy   File: test_threading.py    GNU General Public License v2.0 5 votes vote down vote up
def test_various_ops(self):
        # This takes about n/3 seconds to run (about n/3 clumps of tasks,
        # times about 1 second per clump).
        NUMTASKS = 10

        # no more than 3 of the 10 can run at once
        sema = threading.BoundedSemaphore(value=3)
        mutex = threading.RLock()
        numrunning = Counter()

        threads = []

        for i in range(NUMTASKS):
            t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
            threads.append(t)
            self.assertIsNone(t.ident)
            self.assertRegex(repr(t), r'^<TestThread\(.*, initial\)>$')
            t.start()

        if verbose:
            print('waiting for all tasks to complete')
        for t in threads:
            t.join()
            self.assertFalse(t.is_alive())
            self.assertNotEqual(t.ident, 0)
            self.assertIsNotNone(t.ident)
            self.assertRegex(repr(t), r'^<TestThread\(.*, stopped -?\d+\)>$')
        if verbose:
            print('all tasks done')
        self.assertEqual(numrunning.get(), 0) 
Example 42
Project: cqp-sdk-for-py37-native   Author: crud-boy   File: test_contextlib.py    GNU General Public License v2.0 5 votes vote down vote up
def testWithBoundedSemaphore(self):
        lock = threading.BoundedSemaphore()
        def locked():
            if lock.acquire(False):
                lock.release()
                return False
            else:
                return True
        self.boilerPlate(lock, locked) 
Example 43
Project: Host3r   Author: Rbcafe   File: host3r.py    GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, domain, subdomains=None, q=None, lock=threading.Lock()):
        subdomains = subdomains or []
        self.base_url = 'https://dnsdumpster.com/'
        self.domain = urlparse.urlparse(domain).netloc
        self.subdomains = []
        self.live_subdomains = []
        self.session = requests.Session()
        self.engine_name = "DNSdumpster"
        multiprocessing.Process.__init__(self)
        self.threads = 70
        self.lock = threading.BoundedSemaphore(value=self.threads)
        self.q = q
        self.timeout = 25
        self.print_banner()
        return 
Example 44
Project: oss-ftp   Author: aliyun   File: test_threading.py    MIT License 5 votes vote down vote up
def test_various_ops(self):
        # This takes about n/3 seconds to run (about n/3 clumps of tasks,
        # times about 1 second per clump).
        NUMTASKS = 10

        # no more than 3 of the 10 can run at once
        sema = threading.BoundedSemaphore(value=3)
        mutex = threading.RLock()
        numrunning = Counter()

        threads = []

        for i in range(NUMTASKS):
            t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
            threads.append(t)
            self.assertEqual(t.ident, None)
            self.assertTrue(re.match('<TestThread\(.*, initial\)>', repr(t)))
            t.start()

        if verbose:
            print 'waiting for all tasks to complete'
        for t in threads:
            t.join(NUMTASKS)
            self.assertTrue(not t.is_alive())
            self.assertNotEqual(t.ident, 0)
            self.assertFalse(t.ident is None)
            self.assertTrue(re.match('<TestThread\(.*, \w+ -?\d+\)>', repr(t)))
        if verbose:
            print 'all tasks done'
        self.assertEqual(numrunning.get(), 0) 
Example 45
Project: oss-ftp   Author: aliyun   File: test_contextlib.py    MIT License 5 votes vote down vote up
def testWithBoundedSemaphore(self):
        lock = threading.BoundedSemaphore()
        def locked():
            if lock.acquire(False):
                lock.release()
                return False
            else:
                return True
        self.boilerPlate(lock, locked)

# This is needed to make the test actually run under regrtest.py! 
Example 46
Project: Zydra   Author: hamedA2   File: Zydra.py    MIT License 5 votes vote down vote up
def __init__(self):
        self.start_time = time.monotonic()
        self.process_lock = BoundedSemaphore(value=cpu_count())
        self.counter_lock = threading.BoundedSemaphore(value=1)
        self.banner()
        self.stop = Queue(maxsize=1)
        self.stop.put(False)
        self.count = Queue(maxsize=1)
        self.threads = []
        self.name = Queue(maxsize=1)
        self.name.put(str("a"))
        self.process_count = 0
        self.limit_process = 500
        self.shot = 5000 
Example 47
Project: TFTHelper   Author: AliMZaini   File: test__threading_2.py    MIT License 5 votes vote down vote up
def test_various_ops(self):
        # This takes about n/3 seconds to run (about n/3 clumps of tasks,
        # times about 1 second per clump).
        NUMTASKS = 10

        # no more than 3 of the 10 can run at once
        sema = threading.BoundedSemaphore(value=3)
        mutex = threading.RLock()
        numrunning = Counter()

        threads = []

        for i in range(NUMTASKS):
            t = TestThread("<thread %d>" % i, self, sema, mutex, numrunning)
            threads.append(t)
            t.daemon = False # Under PYPY we get daemon by default?
            if hasattr(t, 'ident'):
                self.assertIsNone(t.ident)
                self.assertFalse(t.daemon)
                self.assertTrue(re.match(r'<TestThread\(.*, initial\)>', repr(t)))
            t.start()

        if verbose:
            print('waiting for all tasks to complete')
        for t in threads:
            t.join(NUMTASKS)
            self.assertFalse(t.is_alive())
            if hasattr(t, 'ident'):
                self.assertNotEqual(t.ident, 0)
                self.assertFalse(t.ident is None)
                self.assertTrue(re.match(r'<TestThread\(.*, \w+ -?\d+\)>', repr(t)))
        if verbose:
            print('all tasks done')
        self.assertEqual(numrunning.get(), 0) 
Example 48
Project: jjal_downloader   Author: tucuprum   File: implement.py    GNU General Public License v3.0 5 votes vote down vote up
def define_thread_num(value):
    global NUMBER_OF_THREAD
    global threadLimiter

    NUMBER_OF_THREAD = value
    threadLimiter = threading.BoundedSemaphore(value) 
Example 49
Project: OOD   Author: BUPT-AC   File: test.py    GNU General Public License v2.0 5 votes vote down vote up
def main():
	t1 = task("http://www.laurentluce.com/posts/python-threads-synchronization-locks-rlocks-semaphores-conditions-events-and-queues/")
	t2 = task("http://stackoverflow.com/questions/15651128/in-this-semaphore-example-is-it-necessary-to-lock-for-refill-and-buy")
	t3 = task("http://bbs.byr.cn/")
	event = Event()
	tasks = TaskQueue(event)
	pages = TaskQueue(None)
	tasks.add(t1)
	tasks.add(t2)
	tasks.add(t3)

	taskLock = BoundedSemaphore(tasks.numOfNewTasks)
	pageLock = BoundedSemaphore(1)
	f = open("test.txt",'w')
	Connector0 = Connector(tasks,taskLock,pages,pageLock,event,'',f, 3000)
	Connector1 = Connector(tasks,taskLock,pages,pageLock,event,'',f, 3001)
	Connector0.start()
	Connector1.start()

	Crawler0 = Crawler('',3000)
	Crawler1 = Crawler('',3001)

	Crawler0.start()
	Crawler1.start()

	Connector1.join()
	Connector0.join()
	Crawler0.join()
	Crawler1.join()
	f.close() 
Example 50
Project: pathspider   Author: mami-project   File: sync.py    GNU General Public License v2.0 5 votes vote down vote up
def acquire_n(self, value=1, blocking=True, timeout=None):
        """
        Acquire ``value`` number of tokens at once.

        The parameters ``blocking`` and ``timeout`` have the same semantics as
        :class:`BoundedSemaphore`.

        :returns: The same value as the last call to `BoundedSemaphore`'s
                  :meth:`acquire` if :meth:`acquire` were called ``value``
                  times instead of the call to this method.
        """
        ret = None
        for _ in range(value):
            ret = self.acquire(blocking=blocking, timeout=timeout)
        return ret 
Example 51
Project: Multiprocessing-and-Multithreading   Author: Ziang-Lu   File: sync_blocking.py    MIT License 5 votes vote down vote up
def func() -> None:
    """
    Dummy function.
    :return: None
    """
    th_name = current_thread().name
    # 请求Semaphore, 成功后计数器-1
    print(f'{th_name} acquiring semaphore...')
    # Note that BoundedSemaphore objects can be used in a traditional way, i.e.,
    # via acquire() and release() methods, but it can also simply be used as a
    # context manager, as a syntax sugar
    with bounded_sema:  # 释放Semaphore的时候, 计数器+1
        print(f'{th_name} gets semaphore')
        time.sleep(4) 
Example 52
Project: TOOLS   Author: DOWRIGHTTV   File: ssh_bruteforce_thread.py    GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, tCount=1):
        validIP = re.compile('^((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?).){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$')
        self.tCount = tCount
        self.threadLimiter = threading.BoundedSemaphore(self.tCount)
        self.i = 0
        self.total = 0 
Example 53
Project: Medusa   Author: Ascotbe   File: sublist3r.py    GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, domain, subdomains=None, q=None, silent=False, verbose=True):
        subdomains = subdomains or []
        base_url = 'https://dnsdumpster.com/'
        self.live_subdomains = []
        self.engine_name = "DNSdumpster"
        self.threads = 70
        self.lock = threading.BoundedSemaphore(value=self.threads)
        self.q = q
        super(DNSdumpster, self).__init__(base_url, self.engine_name, domain, subdomains, q=q, silent=silent, verbose=verbose)
        return 
Example 54
Project: Medusa   Author: Ascotbe   File: sublist3r.py    GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, subdomains, ports):
        self.subdomains = subdomains
        self.ports = ports
        self.threads = 20
        self.lock = threading.BoundedSemaphore(value=self.threads) 
Example 55
Project: luscan-devel   Author: blackye   File: networkmanager.py    GNU General Public License v2.0 5 votes vote down vote up
def __create_semaphore(self):
        return BoundedSemaphore(self.max_connections)


    #-------------------------------------------------------------------------- 
Example 56
Project: Hot   Author: dsolimando   File: test_threading.py    GNU General Public License v3.0 5 votes vote down vote up
def test_various_ops(self):
        # This takes about n/3 seconds to run (about n/3 clumps of tasks,
        # times about 1 second per clump).
        NUMTASKS = 10

        # no more than 3 of the 10 can run at once
        sema = threading.BoundedSemaphore(value=3)
        mutex = threading.RLock()
        numrunning = Counter()

        threads = []

        for i in range(NUMTASKS):
            t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
            threads.append(t)
            t.start()

        if verbose:
            print 'waiting for all tasks to complete'
        for t in threads:
            t.join(NUMTASKS)
            self.assert_(not t.isAlive())
        if verbose:
            print 'all tasks done'
        self.assertEqual(numrunning.get(), 0)

    # run with a small(ish) thread stack size (256kB) 
Example 57
Project: Hot   Author: dsolimando   File: test_contextlib.py    GNU General Public License v3.0 5 votes vote down vote up
def testWithBoundedSemaphore(self):
        lock = threading.BoundedSemaphore()
        def locked():
            if lock.acquire(False):
                lock.release()
                return False
            else:
                return True
        self.boilerPlate(lock, locked)

# This is needed to make the test actually run under regrtest.py! 
Example 58
Project: gallipy   Author: GeoHistoricalData   File: monadic.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def __init__(self, f):
    self.subscribers = []
    self.cache = nil
    self.semaphore = threading.BoundedSemaphore(1)
    f(self.callback)

  # pure :: a -> Future a 
Example 59
Project: odc-tools   Author: opendatacube   File: __init__.py    Apache License 2.0 5 votes vote down vote up
def pool_broadcast(pool, action, *args, **kwargs):
    """Broadcast action across thread pool.

       Will submit action(*args, **kwargs) N times to thread pool, making sure
       that no thread's action finishes before all other actions have started,
       hence forcing each incarnation of `action` to run in it's own thread.

       This function waits for all of these to complete and returns a list of results.
    """
    N = pool._max_workers

    s1 = BoundedSemaphore(N)
    s2 = BoundedSemaphore(N)

    def bcast_action():
        s1.release()                 # tell main thread we started
        x = action(*args, **kwargs)
        s2.acquire()                 # wait for all threads to start
        return x

    for _ in range(N):
        s1.acquire()
        s2.acquire()

    rr = [pool.submit(bcast_action) for _ in range(N)]

    # wait for all to start
    for _ in range(N):
        s1.acquire()

    # allow all to continue
    for _ in range(N):
        s2.release()

    # Extract results (this might block if action is long running)
    # TODO: deal with possible failures
    return [r.result() for r in rr] 
Example 60
Project: vseutil   Author: xvass   File: vseutil.py    GNU General Public License v2.0 5 votes vote down vote up
def unzip(DAT,ip,destfile,outfile):
	semaphore = threading.BoundedSemaphore()
	semaphore.acquire()
	zipp = zipfile.ZipFile('\\\\' + str(ip) + '\\' + str(destfile) + '\\' + DAT)
	zipp.extractall('\\\\' + str(ip) + '\\' + str(destfile) + '\\')
	print Fore.WHITE + "[*] files have been extracted to C:\Program Files\Common Files\McAfee\%s" % destfile
	if (outfile != None):
		log_to_file("[*] files have been extracted to C:\Program Files\Common Files\McAfee\%s" % destfile,outfile)
	print Fore.WHITE + "[*] new DAT has been installed.."
	if (outfile != None):
		log_to_file("[*] new DAT has been installed..",outfile)
	semaphore.release() 
Example 61
Project: vseutil   Author: xvass   File: vseutil.py    GNU General Public License v2.0 5 votes vote down vote up
def deletefiles(ip,destfile,DAT,outfile):
	semaphore = threading.BoundedSemaphore()
	semaphore.acquire()
	os.remove('\\\\' + str(ip) + '\\' + str(destfile) + '\\' + DAT)
	os.remove('\\\\' + str(ip) + '\\' + str(destfile) + '\\' + "legal.txt")
	print Fore.WHITE + "[*] cleaning unwanted files at C:\Program Files\Common Files\McAfee\%s" % destfile
	if (outfile != None):
		log_to_file("[*] cleaning unwanted files at C:\Program Files\Common Files\McAfee\%s" % destfile,outfile)
	semaphore.release() 
Example 62
Project: vseutil   Author: xvass   File: vseutil.py    GNU General Public License v2.0 5 votes vote down vote up
def wmiconnect(ip,username,upass,outfile):
	semaphore = threading.BoundedSemaphore()
	semaphore.acquire()
	try:	
		c = wmi.WMI(computer=ip, user=username, password=upass, namespace="root/default").StdRegProv
		print "[*] Connected to host with IP: %s" % ip
		if (outfile != None):
			log_to_file("[*] Connected to host with IP: %s" % ip,outfile)
	except:
		c="not_connected"
		pass
	semaphore.release()
	return c 
Example 63
Project: vseutil   Author: xvass   File: vseutil.py    GNU General Public License v2.0 5 votes vote down vote up
def changeDATversion(DAT,ip,DAT2val,valnum,username,upass,sourcefile,destfile,outfile,arch,c,value,val,level):
	semaphore = threading.BoundedSemaphore()
	semaphore.acquire()
	try:
		if DAT2val != valnum and level=="down":
			status1,status2,state_value = update_vse(DAT,ip,DAT2val,valnum,username,upass,sourcefile,destfile,outfile)
			update_registry(status1,status2,state_value,arch,outfile,c,value,DAT2val,val)
		elif DAT2val != valnum and level == None and DAT2val > valnum:
			status1,status2,state_value = update_vse(DAT,ip,DAT2val,valnum,username,upass,sourcefile,destfile,outfile)
			update_registry(status1,status2,state_value,arch,outfile,c,value,DAT2val,val)
		elif DAT2val == valnum or (DAT2val < valnum and level==None):
			print Fore.WHITE + "[*] current %s " % (value) + "is " + Fore.YELLOW +  "%s " % (val)
			if (DAT2val < valnum):
				print Fore.WHITE + "[!] You are trying to install a lower DAT version..Use option '-d' to downgrade DAT version.."
			print Fore.WHITE + "[*] Exiting..."
			if (outfile != None):
				log_to_file("[*] current %s " % (value) + "is %s" % (val),outfile)
				log_to_file("[*] Exiting...",outfile)
	except:
		print Fore.RED + "[!] Check if DAT version you are trying to install exists.."
		print Fore.WHITE + "[*] Exiting..."
		if (outfile != None):
			log_to_file("[!] Check if DAT version you are trying to install exists..",outfile)
			log_to_file("[*] Exiting...",outfile)
		checkServices(ip)
	semaphore.release()
	return 
Example 64
Project: PixolutionImageDownloader   Author: pixolution   File: BoundedExecutor.py    Apache License 2.0 5 votes vote down vote up
def __enter__(self):
        self.executor = ThreadPoolExecutor(max_workers=self.max_workers)
        self.semaphore = BoundedSemaphore(self.bound + self.max_workers)
        return self 
Example 65
Project: python-threading   Author: loretoparisi   File: bounded_pool_executor.py    MIT License 5 votes vote down vote up
def __init__(self, max_workers=None):
        super().__init__(max_workers)
        self.semaphore = multiprocessing.BoundedSemaphore(max_workers) 
Example 66
Project: python-threading   Author: loretoparisi   File: bounded_pool_executor.py    MIT License 5 votes vote down vote up
def __init__(self, max_workers=None):
        super().__init__(max_workers)
        self.semaphore = threading.BoundedSemaphore(max_workers) 
Example 67
Project: Software-Architecture-with-Python   Author: PacktPublishing   File: thumbnail_url_controller.py    MIT License 5 votes vote down vote up
def __init__(self, limit=10):
        self.limit = limit
        self.counter = threading.BoundedSemaphore(value=limit)
        self.count = 0
        # Start time
        self.start = time.time()
        # Image saving rate
        self.rate = 0 
Example 68
Project: Software-Architecture-with-Python   Author: PacktPublishing   File: thumbnail_limit_sema.py    MIT License 5 votes vote down vote up
def __init__(self, limit=10):
        self.limit = limit
        self.counter = threading.BoundedSemaphore(value=limit)
        self.count = 0
        # Start time
        self.start = time.time()
        # Image saving rate
        self.rate = 0 
Example 69
Project: Project-New-Reign---Nemesis-Main   Author: ShikyoKira   File: test_threading.py    GNU General Public License v3.0 5 votes vote down vote up
def test_various_ops(self):
        # This takes about n/3 seconds to run (about n/3 clumps of tasks,
        # times about 1 second per clump).
        NUMTASKS = 10

        # no more than 3 of the 10 can run at once
        sema = threading.BoundedSemaphore(value=3)
        mutex = threading.RLock()
        numrunning = Counter()

        threads = []

        for i in range(NUMTASKS):
            t = TestThread("<thread %d>"%i, self, sema, mutex, numrunning)
            threads.append(t)
            self.assertIsNone(t.ident)
            self.assertRegex(repr(t), r'^<TestThread\(.*, initial\)>$')
            t.start()

        if verbose:
            print('waiting for all tasks to complete')
        for t in threads:
            t.join()
            self.assertFalse(t.is_alive())
            self.assertNotEqual(t.ident, 0)
            self.assertIsNotNone(t.ident)
            self.assertRegex(repr(t), r'^<TestThread\(.*, stopped -?\d+\)>$')
        if verbose:
            print('all tasks done')
        self.assertEqual(numrunning.get(), 0) 
Example 70
Project: python-netsurv   Author: sofia-netsurv   File: brain_multiprocessing.py    MIT License 4 votes vote down vote up
def _multiprocessing_managers_transform():
    return astroid.parse(
        """
    import array
    import threading
    import multiprocessing.pool as pool

    import six

    class Namespace(object):
        pass

    class Value(object):
        def __init__(self, typecode, value, lock=True):
            self._typecode = typecode
            self._value = value
        def get(self):
            return self._value
        def set(self, value):
            self._value = value
        def __repr__(self):
            return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
        value = property(get, set)

    def Array(typecode, sequence, lock=True):
        return array.array(typecode, sequence)

    class SyncManager(object):
        Queue = JoinableQueue = six.moves.queue.Queue
        Event = threading.Event
        RLock = threading.RLock
        BoundedSemaphore = threading.BoundedSemaphore
        Condition = threading.Condition
        Barrier = threading.Barrier
        Pool = pool.Pool
        list = list
        dict = dict
        Value = Value
        Array = Array
        Namespace = Namespace
        __enter__ = lambda self: self
        __exit__ = lambda *args: args

        def start(self, initializer=None, initargs=None):
            pass
        def shutdown(self):
            pass
    """
    ) 
Example 71
Project: python-netsurv   Author: sofia-netsurv   File: brain_multiprocessing.py    MIT License 4 votes vote down vote up
def _multiprocessing_managers_transform():
    return astroid.parse(
        """
    import array
    import threading
    import multiprocessing.pool as pool

    import six

    class Namespace(object):
        pass

    class Value(object):
        def __init__(self, typecode, value, lock=True):
            self._typecode = typecode
            self._value = value
        def get(self):
            return self._value
        def set(self, value):
            self._value = value
        def __repr__(self):
            return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
        value = property(get, set)

    def Array(typecode, sequence, lock=True):
        return array.array(typecode, sequence)

    class SyncManager(object):
        Queue = JoinableQueue = six.moves.queue.Queue
        Event = threading.Event
        RLock = threading.RLock
        BoundedSemaphore = threading.BoundedSemaphore
        Condition = threading.Condition
        Barrier = threading.Barrier
        Pool = pool.Pool
        list = list
        dict = dict
        Value = Value
        Array = Array
        Namespace = Namespace
        __enter__ = lambda self: self
        __exit__ = lambda *args: args

        def start(self, initializer=None, initargs=None):
            pass
        def shutdown(self):
            pass
    """
    ) 
Example 72
Project: conf   Author: XonqNopp   File: brain_multiprocessing.py    GNU General Public License v3.0 4 votes vote down vote up
def _multiprocessing_managers_transform():
    return astroid.parse('''
    import array
    import threading
    import multiprocessing.pool as pool

    import six

    class Namespace(object):
        pass

    class Value(object):
        def __init__(self, typecode, value, lock=True):
            self._typecode = typecode
            self._value = value
        def get(self):
            return self._value
        def set(self, value):
            self._value = value
        def __repr__(self):
            return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
        value = property(get, set)

    def Array(typecode, sequence, lock=True):
        return array.array(typecode, sequence)

    class SyncManager(object):
        Queue = JoinableQueue = six.moves.queue.Queue
        Event = threading.Event
        RLock = threading.RLock
        BoundedSemaphore = threading.BoundedSemaphore
        Condition = threading.Condition
        Barrier = threading.Barrier
        Pool = pool.Pool
        list = list
        dict = dict
        Value = Value
        Array = Array
        Namespace = Namespace
        __enter__ = lambda self: self
        __exit__ = lambda *args: args
        
        def start(self, initializer=None, initargs=None):
            pass
        def shutdown(self):
            pass
    ''') 
Example 73
Project: conf   Author: XonqNopp   File: brain_multiprocessing.py    GNU General Public License v3.0 4 votes vote down vote up
def _multiprocessing_managers_transform():
    return astroid.parse('''
    import array
    import threading
    import multiprocessing.pool as pool

    import six

    class Namespace(object):
        pass

    class Value(object):
        def __init__(self, typecode, value, lock=True):
            self._typecode = typecode
            self._value = value
        def get(self):
            return self._value
        def set(self, value):
            self._value = value
        def __repr__(self):
            return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
        value = property(get, set)

    def Array(typecode, sequence, lock=True):
        return array.array(typecode, sequence)

    class SyncManager(object):
        Queue = JoinableQueue = six.moves.queue.Queue
        Event = threading.Event
        RLock = threading.RLock
        BoundedSemaphore = threading.BoundedSemaphore
        Condition = threading.Condition
        Barrier = threading.Barrier
        Pool = pool.Pool
        list = list
        dict = dict
        Value = Value
        Array = Array
        Namespace = Namespace
        __enter__ = lambda self: self
        __exit__ = lambda *args: args
        
        def start(self, initializer=None, initargs=None):
            pass
        def shutdown(self):
            pass
    ''') 
Example 74
Project: vim-python-function-expander   Author: ColinKennedy   File: brain_multiprocessing.py    MIT License 4 votes vote down vote up
def _multiprocessing_managers_transform():
    return astroid.parse('''
    import array
    import threading
    import multiprocessing.pool as pool

    import six

    class Namespace(object):
        pass

    class Value(object):
        def __init__(self, typecode, value, lock=True):
            self._typecode = typecode
            self._value = value
        def get(self):
            return self._value
        def set(self, value):
            self._value = value
        def __repr__(self):
            return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
        value = property(get, set)

    def Array(typecode, sequence, lock=True):
        return array.array(typecode, sequence)

    class SyncManager(object):
        Queue = JoinableQueue = six.moves.queue.Queue
        Event = threading.Event
        RLock = threading.RLock
        BoundedSemaphore = threading.BoundedSemaphore
        Condition = threading.Condition
        Barrier = threading.Barrier
        Pool = pool.Pool
        list = list
        dict = dict
        Value = Value
        Array = Array
        Namespace = Namespace
        __enter__ = lambda self: self
        __exit__ = lambda *args: args
        
        def start(self, initializer=None, initargs=None):
            pass
        def shutdown(self):
            pass
    ''') 
Example 75
Project: pySINDy   Author: luckystarufo   File: brain_multiprocessing.py    MIT License 4 votes vote down vote up
def _multiprocessing_managers_transform():
    return astroid.parse(
        """
    import array
    import threading
    import multiprocessing.pool as pool

    import six

    class Namespace(object):
        pass

    class Value(object):
        def __init__(self, typecode, value, lock=True):
            self._typecode = typecode
            self._value = value
        def get(self):
            return self._value
        def set(self, value):
            self._value = value
        def __repr__(self):
            return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
        value = property(get, set)

    def Array(typecode, sequence, lock=True):
        return array.array(typecode, sequence)

    class SyncManager(object):
        Queue = JoinableQueue = six.moves.queue.Queue
        Event = threading.Event
        RLock = threading.RLock
        BoundedSemaphore = threading.BoundedSemaphore
        Condition = threading.Condition
        Barrier = threading.Barrier
        Pool = pool.Pool
        list = list
        dict = dict
        Value = Value
        Array = Array
        Namespace = Namespace
        __enter__ = lambda self: self
        __exit__ = lambda *args: args

        def start(self, initializer=None, initargs=None):
            pass
        def shutdown(self):
            pass
    """
    ) 
Example 76
Project: Light_control   Author: laurent-colas   File: brain_multiprocessing.py    The Unlicense 4 votes vote down vote up
def _multiprocessing_managers_transform():
    return astroid.parse('''
    import array
    import threading
    import multiprocessing.pool as pool

    import six

    class Namespace(object):
        pass

    class Value(object):
        def __init__(self, typecode, value, lock=True):
            self._typecode = typecode
            self._value = value
        def get(self):
            return self._value
        def set(self, value):
            self._value = value
        def __repr__(self):
            return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
        value = property(get, set)

    def Array(typecode, sequence, lock=True):
        return array.array(typecode, sequence)

    class SyncManager(object):
        Queue = JoinableQueue = six.moves.queue.Queue
        Event = threading.Event
        RLock = threading.RLock
        BoundedSemaphore = threading.BoundedSemaphore
        Condition = threading.Condition
        Barrier = threading.Barrier
        Pool = pool.Pool
        list = list
        dict = dict
        Value = Value
        Array = Array
        Namespace = Namespace
        __enter__ = lambda self: self
        __exit__ = lambda *args: args
        
        def start(self, initializer=None, initargs=None):
            pass
        def shutdown(self):
            pass
    ''') 
Example 77
Project: vseutil   Author: xvass   File: vseutil.py    GNU General Public License v2.0 4 votes vote down vote up
def update_registry(status1,status2,state_value,arch,outfile,c,value,DAT2val,current_val):
	semaphore = threading.BoundedSemaphore()
	semaphore.acquire()
	if status1 != STOPPED and status2 != STOPPED and state_value == "passed":									
		if(arch == 'x86'):
			print "[*] updating registry.."
			if (outfile != None):
				log_to_file("[*] updating registry..",outfile)
			datRelease,date_release = c.GetStringValue(hDefKey=win32con.HKEY_LOCAL_MACHINE,sSubKeyName="SOFTWARE\Network Associates\ePolicy Orchestrator\Application Plugins\VIRUSCAN8800",sValueName="DatDate")
			datInst,date_install = c.GetStringValue(hDefKey=win32con.HKEY_LOCAL_MACHINE,sSubKeyName="SOFTWARE\Network Associates\ePolicy Orchestrator\Application Plugins\VIRUSCAN8800",sValueName="DatInstallDate")
			DATrelease = DatReleaseDate(DAT2val,current_val,date_release)
			DATinstall = installDatDate(DATrelease,date_install)
			result, = c.SetStringValue(hDefKey=win32con.HKEY_LOCAL_MACHINE,sSubKeyName=r"SOFTWARE\Network Associates\ePolicy Orchestrator\Application Plugins\VIRUSCAN8800",sValueName=value,sValue=str(DAT2val) + '.0000')
			dateres, = c.SetStringValue(hDefKey=win32con.HKEY_LOCAL_MACHINE,sSubKeyName=r"SOFTWARE\Network Associates\ePolicy Orchestrator\Application Plugins\VIRUSCAN8800",sValueName='DatDate',sValue=DATrelease)
			dateinstalledres, = c.SetStringValue(hDefKey=win32con.HKEY_LOCAL_MACHINE,sSubKeyName=r"SOFTWARE\Network Associates\ePolicy Orchestrator\Application Plugins\VIRUSCAN8800",sValueName='DatInstallDate',sValue=DATinstall)
			res,val = c.GetStringValue(hDefKey=win32con.HKEY_LOCAL_MACHINE,sSubKeyName="SOFTWARE\Network Associates\ePolicy Orchestrator\Application Plugins\VIRUSCAN8800",sValueName=value)
			print "[*] registry updated successfully"
			if (outfile != None):
				log_to_file("[*] registry updated successfully",outfile)
		else:
			print "[*] updating registry.."
			if (outfile != None):
				log_to_file("[*] updating registry..",outfile)
			datRelease,date_release = c.GetStringValue(hDefKey=win32con.HKEY_LOCAL_MACHINE,sSubKeyName="SOFTWARE\Network Associates\ePolicy Orchestrator\Application Plugins\VIRUSCAN8800",sValueName="DatDate")
			datInst,date_install = c.GetStringValue(hDefKey=win32con.HKEY_LOCAL_MACHINE,sSubKeyName="SOFTWARE\Network Associates\ePolicy Orchestrator\Application Plugins\VIRUSCAN8800",sValueName="DatInstallDate")
			DATrelease = DatReleaseDate(DAT2val,current_val,date_release)
			DATinstall = installDatDate(DATrelease,date_install)
			result, = c.SetStringValue(hDefKey=win32con.HKEY_LOCAL_MACHINE,sSubKeyName=r"SOFTWARE\Wow6432Node\Network Associates\ePolicy Orchestrator\Application Plugins\VIRUSCAN8800",sValueName=value,sValue=str(DAT2val) + '.0000')
			dateres, = c.SetStringValue(hDefKey=win32con.HKEY_LOCAL_MACHINE,sSubKeyName=r"SOFTWARE\Wow6432Node\Network Associates\ePolicy Orchestrator\Application Plugins\VIRUSCAN8800",sValueName='DatDate',sValue=DATrelease)
			dateinstalledres, = c.SetStringValue(hDefKey=win32con.HKEY_LOCAL_MACHINE,sSubKeyName=r"SOFTWARE\Wow6432Node\Network Associates\ePolicy Orchestrator\Application Plugins\VIRUSCAN8800",sValueName='DatInstallDate',sValue=DATinstall)
			res,val = c.GetStringValue(hDefKey=win32con.HKEY_LOCAL_MACHINE,sSubKeyName="SOFTWARE\Wow6432Node\Network Associates\ePolicy Orchestrator\Application Plugins\VIRUSCAN8800",sValueName=value)
			print "[*] registry updated successfully"
			if (outfile != None):
				log_to_file("[*] registry updated successfully",outfile)
		print  Fore.WHITE + "[*] new current %s is" % (value) + Fore.YELLOW + " %s " % (val)
		print Fore.WHITE + "[*] Exiting...\n"
		if (outfile != None):
			log_to_file("[*] new current %s " % (value) + "is %s \n\n" % (val),outfile)
	elif state_value == "not_passed":
		print "[x] Registry cannot be updated. Please try again..\n"
		if (outfile != None):
			log_to_file("[x] Registry cannot be updated. Please try again..\n",outfile)
	semaphore.release()
	return 
Example 78
Project: Blackjack-Tracker   Author: martinabeleda   File: brain_multiprocessing.py    MIT License 4 votes vote down vote up
def _multiprocessing_managers_transform():
    return astroid.parse('''
    import array
    import threading
    import multiprocessing.pool as pool

    import six

    class Namespace(object):
        pass

    class Value(object):
        def __init__(self, typecode, value, lock=True):
            self._typecode = typecode
            self._value = value
        def get(self):
            return self._value
        def set(self, value):
            self._value = value
        def __repr__(self):
            return '%s(%r, %r)'%(type(self).__name__, self._typecode, self._value)
        value = property(get, set)

    def Array(typecode, sequence, lock=True):
        return array.array(typecode, sequence)

    class SyncManager(object):
        Queue = JoinableQueue = six.moves.queue.Queue
        Event = threading.Event
        RLock = threading.RLock
        BoundedSemaphore = threading.BoundedSemaphore
        Condition = threading.Condition
        Barrier = threading.Barrier
        Pool = pool.Pool
        list = list
        dict = dict
        Value = Value
        Array = Array
        Namespace = Namespace
        __enter__ = lambda self: self
        __exit__ = lambda *args: args
        
        def start(self, initializer=None, initargs=None):
            pass
        def shutdown(self):
            pass
    ''') 
Example 79
Project: folderclone   Author: Spazzlo   File: multifolderclone.py    GNU General Public License v3.0 4 votes vote down vote up
def clone(self):
        accounts = glob(self.path + '/*.json')
        if len(accounts) < 2:
            raise ValueError('The path provided (%s) has 1 or no accounts.' % self.path)

        check = build('drive','v3',credentials=Credentials.from_service_account_file(accounts[0]))

        try:
            root_dir = check.files().get(fileId=self.source,supportsAllDrives=True).execute()['name']
        except HttpError:
            raise ValueError('Source folder %s cannot be read or is invalid.' % self.source)

        dest_dict = {i:'' for i in self.dest}
        for key in list(dest_dict.keys()):
            try:
                dest_dir = check.files().get(fileId=key,supportsAllDrives=True).execute()['name']
                dest_dict[key] = dest_dir
            except HttpError:
                if not self.skip_bad_dests:
                    raise ValueError('Destination folder %s cannot be read or is invalid.' % key)
                else:
                    dest_dict.pop(key)

        print('Creating %d Drive Services' % len(accounts))
        drive = []
        for account in accounts:
            credentials = Credentials.from_service_account_file(account, scopes=[
                'https://www.googleapis.com/auth/drive'
            ])
            drive.append(build('drive', 'v3', credentials=credentials))
        if self.thread_count is not None and self.thread_count <= len(drive):
            self.threads = threading.BoundedSemaphore(self.thread_count)
            print('BoundedSemaphore with %d threads' % self.thread_count)
        elif self.thread_count is None:
            self.threads = threading.BoundedSemaphore(len(drive))
            print('BoundedSemaphore with %d threads' % len(drive))
        else:
            raise ValueError('More threads than there is service accounts.')

        for i, dest_dir in dest_dict.items():
            print('Copying from %s to %s.' % (root_dir,dest_dir))
            self._rcopy(drive,1,self.source,i,root_dir,'',self.width) 
Example 80
Project: findNeighbour2   Author: davidhwyllie   File: webservice-server.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def __init__(self,CONFIG):
		""" Using values in CONFIG, starts a server with CONFIG['NAME'] on port CONFIG['PORT'].
		
		CONFIG contains Configuration parameters relevant to the reference based compression system which lies
		at the core of the server.
		
		for explanations as to the meanings of these values, please see the documentation in  ewSetCore, to
		which the CONFIG dictionary gets passed.
		
		An example CONFIG is below:
		
		{			
		"DESCRIPTION":"A test server operating in ../unittest_tmp, only suitable for testing",
		"PORT":8184,
		"IP":"127.0.0.1",
		"INPUTREF":"../reference/TB-ref.fasta",
		"PERSISTENCEDIR":"../unittest_tmp/",
		"EXCLUDEFILE":"../reference/TB-exclude.txt",
		"SNPDIR":"../unittest_tmp",
		"DEBUGMODE":0,
		"SERVERNAME":"TBSNP",
		"EDGEDB_CONNSTRING":"sqlite:///<<DEFAULT>>/{0}.db",
		"FNPERSISTENCE_CONNSTRING":"sqlite:///<<DEFAULT>>/findNeighbour.db",
		"MAXN_STORAGE":100000,
		"MAXN_PROP_DEFAULT":0.70,
		"NCOMPRESSIONCUTOFF":100000,
		"LOGFILE":"../unittest_tmp/logfile.log",
		"LOGLEVEL":"INFO",
		"SNPCEILING": 20,
		"MULTIPROCESSING":0
		}

	
		"""
		
		## check input
		if isinstance(CONFIG, str):
			CONFIG=json.loads(CONFIG)	# assume JSON string; convert.
		
		# check it is a dictionary	
		if not isinstance(CONFIG, dict):
			raise KeyError("CONFIG must be either a dictionary or a JSON string encoding a dictionary.  It is: {0}".format(CONFIG))
		
		# check that the keys of config are as expected.
		required_keys=set(['IP','PORT','INPUTREF','PERSISTENCEDIR','EXCLUDEFILE','SNPDIR','DEBUGMODE','SERVERNAME',
						   'EDGEDB_CONNSTRING','MAXN_STORAGE', 'MAXN_PROP_DEFAULT', 'SNPCEILING', 'MULTIPROCESSING'])
		missing=required_keys-set(CONFIG.keys())
		if not missing == set([]):
			raise KeyError("Required keys were not found in CONFIG. Missing are {0}".format(missing))
		
		# start process
		self.write_semaphore = threading.BoundedSemaphore(1)
		self.CONFIG=CONFIG
		self.objExaminer=NucleicAcid()
		self.ewsc=ewSetCore(CONFIG=self.CONFIG)
		print ("findNeighbour2 Ready; ({0})".format(self.ewsc.sc.excluded_hash()))