Python threading.Semaphore() Examples

The following are 30 code examples for showing how to use threading.Semaphore(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module threading , or try the search function .

Example 1
Project: botbuilder-python   Author: microsoft   File: cosmosdb_storage.py    License: MIT License 6 votes vote down vote up
def __init__(
        self, config: CosmosDbConfig, client: cosmos_client.CosmosClient = None
    ):
        """Create the storage object.

        :param config:
        """
        super(CosmosDbStorage, self).__init__()
        self.config = config
        self.client = client or cosmos_client.CosmosClient(
            self.config.endpoint, {"masterKey": self.config.masterkey}
        )
        # these are set by the functions that check
        # the presence of the database and container or creates them
        self.database = None
        self.container = None
        self._database_creation_options = config.database_creation_options
        self._container_creation_options = config.container_creation_options
        self.__semaphore = Semaphore() 
Example 2
Project: misp42splunk   Author: remg427   File: thread.py    License: GNU Lesser General Public License v3.0 6 votes vote down vote up
def __init__(self, max_workers=None, thread_name_prefix=''):
        """Initializes a new ThreadPoolExecutor instance.

        Args:
            max_workers: The maximum number of threads that can be used to
                execute the given calls.
            thread_name_prefix: An optional name prefix to give our threads.
        """
        if max_workers is None:
            # Use this number because ThreadPoolExecutor is often
            # used to overlap I/O instead of CPU work.
            max_workers = (cpu_count() or 1) * 5
        if max_workers <= 0:
            raise ValueError("max_workers must be greater than 0")

        self._max_workers = max_workers
        self._work_queue = queue.Queue()
        self._idle_semaphore = threading.Semaphore(0)
        self._threads = set()
        self._shutdown = False
        self._shutdown_lock = threading.Lock()
        self._thread_name_prefix = (thread_name_prefix or
                                    ("ThreadPoolExecutor-%d" % self._counter())) 
Example 3
Project: browserscope   Author: elsigh   File: adaptive_thread_pool.py    License: Apache License 2.0 6 votes vote down vote up
def __init__(self,
               num_threads,
               sleep=InterruptibleSleep):
    """Constructor for ThreadGate instances.

    Args:
      num_threads: The total number of threads using this gate.
      sleep: Used for dependency injection.
    """
    self.__enabled_count = 1

    self.__lock = threading.Lock()

    self.__thread_semaphore = threading.Semaphore(self.__enabled_count)
    self.__num_threads = num_threads
    self.__backoff_time = 0
    self.__sleep = sleep 
Example 4
Project: ACE   Author: IntegralDefense   File: network_semaphore.py    License: Apache License 2.0 6 votes vote down vote up
def initialize_fallback_semaphores():
    """This needs to be called once at the very beginning of starting ACE."""

    # we need some fallback functionality for when the network semaphore server is down
    # these semaphores serve that purpose
    global_engine_instance_count = saq.CONFIG['global'].getint('global_engine_instance_count')
    for key in saq.CONFIG['network_semaphore'].keys():
        if key.startswith('semaphore_'):
            semaphore_name = key[len('semaphore_'):]
            # the configuration settings for the network semaphore specify how many connections
            # are allowed to a specific resource at once, globally
            # so if we unable to coordinate globally, the fall back is to divide the available
            # number of resources between all the engines evenly
            # that's what this next equation is for
            fallback_limit = int(floor(saq.CONFIG['network_semaphore'].getfloat(key) / float(global_engine_instance_count)))
            # we allow a minimum of one per engine
            if fallback_limit < 1:
                fallback_limit = 1

            logging.debug("fallback semaphore count for {0} is {1}".format(semaphore_name, fallback_limit))
            fallback_semaphores[semaphore_name] = LoggingSemaphore(fallback_limit)
            #fallback_semaphores[semaphore_name] = multiprocessing.Semaphore(fallback_limit)
            #fallback_semaphores[semaphore_name].semaphore_name = 'fallback {0}'.format(semaphore_name) 
Example 5
Project: Yuki-Chan-The-Auto-Pentest   Author: Yukinoshita47   File: Fuzzer.py    License: MIT License 6 votes vote down vote up
def start(self):
        # Setting up testers
        self.setupScanners()
        # Setting up threads
        self.setupThreads()
        self.index = 0
        self.dictionary.reset()
        self.runningThreadsCount = len(self.threads)
        self.running = True
        self.playEvent = threading.Event()
        self.pausedSemaphore = threading.Semaphore(0)
        self.playEvent.clear()
        self.exit = False
        for thread in self.threads:
            thread.start()
        self.play() 
Example 6
Project: visual_foresight   Author: SudeepDasari   File: wsg50_gripper.py    License: MIT License 6 votes vote down vote up
def __init__(self):
        super(WSG50Gripper, self).__init__()
        self.max_release = 0
        self.sem_list = [Semaphore(value = 0)]
        self._status_mutex = Lock()

        self._desired_gpos = GRIPPER_OPEN
        self._gripper_speed = 300

        self._force_counter = 0
        self._integrate_gripper_force, self._last_integrate = 0., None
        self._last_status_t = time.time()
        self.num_timeouts = 0

        self.gripper_pub = rospy.Publisher('/wsg_50_driver/goal_position', Cmd, queue_size=10)
        rospy.Subscriber("/wsg_50_driver/status", Status, self._gripper_callback)
        logging.getLogger('robot_logger').info("waiting for first status")
        self.sem_list[0].acquire()
        logging.getLogger('robot_logger').info('gripper initialized!')

        self._bg = Thread(target=self._background_monitor)
        self._bg.start() 
Example 7
Project: visual_foresight   Author: SudeepDasari   File: wsg50_gripper.py    License: MIT License 6 votes vote down vote up
def _set_gripper(self, command_pos, wait=False):
        self._status_mutex.acquire()
        self._desired_gpos = command_pos
        if wait:
            if self.num_timeouts > MAX_TIMEOUT:
                rospy.signal_shutdown("MORE THAN {} GRIPPER TIMEOUTS".format(MAX_TIMEOUT))

            sem = Semaphore(value=0)  # use of semaphore ensures script will block if gripper dies during execution
            self.sem_list.append(sem)
            self._status_mutex.release()

            start = rospy.get_time()
            logging.getLogger('robot_logger').debug("gripper sem acquire, list len-{}".format(len(self.sem_list)))
            sem.acquire()
            logging.getLogger('robot_logger').debug("waited on gripper for {} seconds".format(rospy.get_time() - start))
        else:
            self._status_mutex.release() 
Example 8
Project: visual_foresight   Author: SudeepDasari   File: camera_recorder.py    License: MIT License 6 votes vote down vote up
def __init__(self, topic_data, opencv_tracking=False, save_videos=False):
        self._tracking_enabled, self._save_vides = opencv_tracking, save_videos

        self._latest_image = LatestObservation(self._tracking_enabled, self._save_vides)

        self._is_tracking = False
        if self._tracking_enabled:
            self.box_height = 80

        self.bridge = CvBridge()
        self._is_first_status, self._status_sem = True, Semaphore(value=0)
        self._cam_height, self._cam_width = None, None
        self._last_hash, self._num_repeats = None, 0
        if self._save_vides:
            self._buffers = []
            self._saving = False

        self._topic_data = topic_data
        self._image_dtype = topic_data.dtype
        rospy.Subscriber(topic_data.name, Image_msg, self.store_latest_im)
        logging.getLogger('robot_logger').debug('downing sema on topic: {}'.format(topic_data.name))
        self._status_sem.acquire()
        logging.getLogger('robot_logger').info("Cameras {} subscribed: stream is {}x{}".format(self._topic_data.name, self._cam_width, self._cam_height)) 
Example 9
Project: rtkbase   Author: Stefal   File: RtkController.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def __init__(self, rtklib_path, config_path):

        self.bin_path = rtklib_path
        self.config_path = config_path

        self.child = 0

        self.status = {}
        self.obs_rover = {}
        self.obs_base = {}
        self.info = {}
        self.semaphore = Semaphore()

        self.started = False
        self.launched = False
        self.current_config = "" 
Example 10
Project: minion-manager   Author: keikoproj   File: aws_minion_manager.py    License: Apache License 2.0 6 votes vote down vote up
def set_semaphore(self, asg_meta):
        """
        Update no of instances can be terminated based on percentage.
        """
        asg_name = asg_meta.get_name()
        asg_semaphore = 'semaphore' + asg_name
        resp = self._ac_client.describe_auto_scaling_groups(AutoScalingGroupNames=[asg_name])
        desired_instances = resp["AutoScalingGroups"][0]["DesiredCapacity"]
        if self.terminate_percentage > 100:
            self.terminate_percentage = 100
        elif self.terminate_percentage <= 0:
            self.terminate_percentage = 1
        # Get no of instance can parallel be rotated
        svalue = int(round(desired_instances * (self.terminate_percentage/100.0)))
        if svalue == 0:
            svalue = 1
        logger.info("Maximum %d instance will be rotated at a time for ASG %s", svalue, asg_name)
        asg_semaphore = Semaphore(value=svalue)
        return asg_semaphore 
Example 11
Project: INGInious   Author: UCL-INGI   File: client_sync.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def new_job(self, priority, task, inputdata, launcher_name="Unknown", debug=False):
        """
            Runs a new job.
            It works exactly like the Client class, instead that there is no callback and directly returns result, in the form of a tuple
            (result, grade, problems, tests, custom, archive).
        """
        job_semaphore = threading.Semaphore(0)

        def manage_output(result, grade, problems, tests, custom, state, archive, stdout, stderr):
            """ Manages the output of this job """
            manage_output.job_return = (result, grade, problems, tests, custom, state, archive, stdout, stderr)
            job_semaphore.release()

        manage_output.job_return = None

        self._client.new_job(priority, task, inputdata, manage_output, launcher_name, debug)
        job_semaphore.acquire()
        job_return = manage_output.job_return
        return job_return 
Example 12
Project: Fluid-Designer   Author: Microvellum   File: test_logging.py    License: GNU General Public License v3.0 6 votes vote down vote up
def setUp(self):
        """Set up a TCP server to receive log messages, and a SocketHandler
        pointing to that server's address and port."""
        BaseTest.setUp(self)
        self.server = server = self.server_class(self.address,
                                                 self.handle_socket, 0.01)
        server.start()
        server.ready.wait()
        hcls = logging.handlers.SocketHandler
        if isinstance(server.server_address, tuple):
            self.sock_hdlr = hcls('localhost', server.port)
        else:
            self.sock_hdlr = hcls(server.server_address, None)
        self.log_output = ''
        self.root_logger.removeHandler(self.root_logger.handlers[0])
        self.root_logger.addHandler(self.sock_hdlr)
        self.handled = threading.Semaphore(0) 
Example 13
Project: multitasking   Author: ranaroussi   File: __init__.py    License: Apache License 2.0 6 votes vote down vote up
def createPool(name="main", threads=None, engine=None):

    config["POOL_NAME"] = name

    try:
        threads = int(threads)
    except Exception:
        threads = config["MAX_THREADS"]
    if threads < 2:
        threads = 0

    engine = engine if engine is not None else config["ENGINE"]

    config["MAX_THREADS"] = threads
    config["ENGINE"] = engine

    config["POOLS"][config["POOL_NAME"]] = {
        "pool": Semaphore(threads) if threads > 0 else None,
        "engine": Process if "process" in engine.lower() else Thread,
        "name": name,
        "threads": threads
    } 
Example 14
Project: ironpython3   Author: IronLanguages   File: test_logging.py    License: Apache License 2.0 6 votes vote down vote up
def setUp(self):
        """Set up a TCP server to receive log messages, and a SocketHandler
        pointing to that server's address and port."""
        BaseTest.setUp(self)
        self.server = server = self.server_class(self.address,
                                                 self.handle_socket, 0.01)
        server.start()
        server.ready.wait()
        hcls = logging.handlers.SocketHandler
        if isinstance(server.server_address, tuple):
            self.sock_hdlr = hcls('localhost', server.port)
        else:
            self.sock_hdlr = hcls(server.server_address, None)
        self.log_output = ''
        self.root_logger.removeHandler(self.root_logger.handlers[0])
        self.root_logger.addHandler(self.sock_hdlr)
        self.handled = threading.Semaphore(0) 
Example 15
Project: zoe   Author: DistributedSystemsGroup   File: elastic_scheduler.py    License: Apache License 2.0 6 votes vote down vote up
def __init__(self, state: SQLManager, policy, metrics: StatsManager):
        if policy not in ('FIFO', 'SIZE', 'DYNSIZE'):
            raise UnsupportedSchedulerPolicyError
        self.metrics = metrics
        self.trigger_semaphore = threading.Semaphore(0)
        self.policy = policy
        self.queue = []
        self.queue_running = []
        self.queue_termination = []
        self.additional_exec_state = {}
        self.loop_quit = False
        self.loop_th = threading.Thread(target=self.loop_start_th, name='scheduler')
        self.core_limit_recalc_trigger = threading.Event()
        self.core_limit_th = threading.Thread(target=self._adjust_core_limits, name='adjust_core_limits')
        self.state = state
        for execution in self.state.executions.select(status='running'):
            if execution.all_services_running:
                self.queue_running.append(execution)
            else:
                self.queue.append(execution)
                self.additional_exec_state[execution.id] = ExecutionProgress()
        self.loop_th.start()
        self.core_limit_th.start() 
Example 16
Project: ReachView   Author: emlid   File: RtkController.py    License: GNU General Public License v3.0 6 votes vote down vote up
def __init__(self, rtklib_path):

        self.bin_path = rtklib_path + "/app/rtkrcv/gcc"
        self.config_path = rtklib_path + "/app/rtkrcv"

        self.child = 0

        self.status = {}
        self.obs_rover = {}
        self.obs_base = {}
        self.info = {}
        self.semaphore = Semaphore()

        self.started = False
        self.launched = False
        self.current_config = "" 
Example 17
Project: addon   Author: alfa-addon   File: thread.py    License: GNU General Public License v3.0 6 votes vote down vote up
def __init__(self, max_workers=None, thread_name_prefix=''):
        """Initializes a new ThreadPoolExecutor instance.

        Args:
            max_workers: The maximum number of threads that can be used to
                execute the given calls.
            thread_name_prefix: An optional name prefix to give our threads.
        """
        if max_workers is None:
            # Use this number because ThreadPoolExecutor is often
            # used to overlap I/O instead of CPU work.
            max_workers = (cpu_count() or 1) * 5
        if max_workers <= 0:
            raise ValueError("max_workers must be greater than 0")

        self._max_workers = max_workers
        self._work_queue = queue.Queue()
        self._idle_semaphore = threading.Semaphore(0)
        self._threads = set()
        self._shutdown = False
        self._shutdown_lock = threading.Lock()
        self._thread_name_prefix = (thread_name_prefix or
                                    ("ThreadPoolExecutor-%d" % self._counter())) 
Example 18
Project: RacingRobot   Author: sergionr2   File: main.py    License: MIT License 5 votes vote down vote up
def forceStop(command_queue, n_received_semaphore):
    """
    Stop The car
    :param command_queue: (CustomQueue) Queue for sending orders to the Arduino
    :param n_received_semaphore: (threading.Semaphore) Semaphore to regulate orders sent to the Arduino
    """
    command_queue.clear()
    n_received_semaphore.release()
    n_received_semaphore.release()
    command_queue.put((Order.MOTOR, 0))
    command_queue.put((Order.SERVO, int((THETA_MIN + THETA_MAX) / 2))) 
Example 19
Project: raveberry   Author: raveberry   File: playback.py    License: GNU Lesser General Public License v3.0 5 votes vote down vote up
def start(self) -> None:
        self.queue.delete_placeholders()
        Playback.queue_semaphore = Semaphore(self.queue.count())

        with self.mopidy_command(important=True):
            self.player.playback.stop()
            self.player.tracklist.clear()
            # make songs disappear from tracklist after being played
            self.player.tracklist.set_consume(True)
        self._loop() 
Example 20
Project: instavpn   Author: sockeye44   File: pastee.py    License: Apache License 2.0 5 votes vote down vote up
def __init__(self, url=PASTEE_URL):
        """Constructor.

        Args:
          url: URL to Pastee installation (defaults to https://pastee.org)
        """
        parse = urlparse.urlsplit(url)
        self._scheme = parse[0]
        self._netloc = parse[1]
        self._lock = threading.Semaphore() 
Example 21
Project: bamf   Author: malwaredllc   File: bamf.py    License: GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, shodan_api=None):
        """
        Initialize a new Bamf instance

        `Optional`
        :param str shodan_api:  Shodan API key

        """
        mechanize.Browser.__init__(self)
        self._models = {}
        self._targets = {}
        self._devices = []
        self._threads = []
        self._backdoors = []
        self._queue = Queue.Queue()
        self._query = 'alphanetworks/2.23'
        self._ports = [8000, 8080, 8888]
        self._semaphore = threading.Semaphore(value=1)
        self._database = sqlite3.connect('database.db')
        self._database.executescript(self.__tbl_config)
        self._database.executescript(self.__tbl_routers)
        self._database.executescript(self.__tbl_devices)
        self._shodan = self._init_shodan(shodan_api)
        self.addheaders = [('User-Agent', 'xmlset_roodkcableoj28840ybtide')]
        self.set_handle_robots(False)
        self.set_handle_redirect(True)
        self.set_handle_refresh(True)
        self.set_handle_equiv(True)
        self.set_handle_referer(True)
        self.set_debug_http(False)
        self.set_debug_responses(False) 
Example 22
Project: mars   Author: mars-project   File: executor.py    License: Apache License 2.0 5 votes vote down vote up
def semaphore(cls, value):
        return threading.Semaphore(value) 
Example 23
Project: mars   Author: mars-project   File: executor.py    License: Apache License 2.0 5 votes vote down vote up
def semaphore(cls, value):
        # as gevent threadpool is the **real** thread, so use threading.Semaphore
        return threading.Semaphore(value) 
Example 24
Project: GTDWeb   Author: lanbing510   File: synch.py    License: GNU General Public License v2.0 5 votes vote down vote up
def __init__(self):
        self.mutex = threading.RLock()
        self.can_read = threading.Semaphore(0)
        self.can_write = threading.Semaphore(0)
        self.active_readers = 0
        self.active_writers = 0
        self.waiting_readers = 0
        self.waiting_writers = 0 
Example 25
Project: lock_files   Author: jlinoff   File: lock_files.py    License: MIT License 5 votes vote down vote up
def main():
    '''
    main
    '''
    opts = getopts()
    password = get_password(opts)

    stats = {
        'locked': 0,
        'unlocked': 0,
        'skipped': 0,
        'files': 0,
        'dirs': 0,
        'read': 0,
        'written': 0,
        }

    # Use the mutex for I/O to avoid interspersed output.
    # Use the semaphore to limit the number of active threads.
    global th_semaphore
    th_semaphore = Semaphore(opts.jobs)

    try:
        run(opts, password, stats)
        wait_for_threads()
    except KeyboardInterrupt:
        abort_threads()
        _println('', sys.stderr)
        errn('^C detected, cleaning up threads, please wait\n')
        wait_for_threads()

    summary(opts, stats)
    if th_abort == True:
        sys.exit(1) 
Example 26
Project: ironpython2   Author: IronLanguages   File: test_contextlib.py    License: Apache License 2.0 5 votes vote down vote up
def testWithSemaphore(self):
        lock = threading.Semaphore()
        def locked():
            if lock.acquire(False):
                lock.release()
                return False
            else:
                return True
        self.boilerPlate(lock, locked) 
Example 27
Project: python-mpv   Author: jaseg   File: mpv.py    License: GNU Affero General Public License v3.0 5 votes vote down vote up
def wait_for_property(self, name, cond=lambda val: val, level_sensitive=True):
        """Waits until ``cond`` evaluates to a truthy value on the named property. This can be used to wait for
        properties such as ``idle_active`` indicating the player is done with regular playback and just idling around
        """
        sema = threading.Semaphore(value=0)
        def observer(name, val):
            if cond(val):
                sema.release()
        self.observe_property(name, observer)
        if not level_sensitive or not cond(getattr(self, name.replace('-', '_'))):
            sema.acquire()
        self.unobserve_property(name, observer) 
Example 28
Project: respeaker_python_library   Author: respeaker   File: pyusb_backend.py    License: Apache License 2.0 5 votes vote down vote up
def __init__(self):
        super(PyUSB, self).__init__()
        self.ep_out = None
        self.ep_in = None
        self.dev = None
        self.closed = False
        self.rcv_data = []
        self.read_sem = threading.Semaphore(0) 
Example 29
Project: faces   Author: skarlekar   File: utils.py    License: GNU General Public License v2.0 5 votes vote down vote up
def __init__(self, count):
        """A semaphore for the purpose of limiting the number of tasks

        :param count: The size of semaphore
        """
        self._semaphore = threading.Semaphore(count) 
Example 30
Project: faces   Author: skarlekar   File: utils.py    License: GNU General Public License v2.0 5 votes vote down vote up
def __init__(self, count):
        """A semaphore for the purpose of limiting the number of tasks

        :param count: The size of semaphore
        """
        self._semaphore = threading.Semaphore(count)