Python time.time() Examples

The following are 50 code examples for showing how to use time.time(). They are extracted from open source Python projects. You can vote up the examples you like or vote down the exmaples you don't like. You can also save this page to your account.

Example 1
Project: dsq   Author: baverman   File: test_manager.py    (MIT License) View Source Project 8 votes vote down vote up
def test_worker_alarm(manager):
    called = []
    def handler(signal, frame):
        called.append(True)
    signal.signal(signal.SIGALRM, handler)

    @manager.task
    def foo(sleep):
        time.sleep(sleep)

    w = Worker(manager, task_timeout=1)
    w.process_one(make_task('foo', args=(0.1,)))
    assert not called

    w.process_one(make_task('foo', args=(1.1,)))
    assert called 
Example 2
Project: ArticleSpider   Author: mtianyan   File: zhihu_login_requests.py    (license) View Source Project 6 votes vote down vote up
def get_captcha():
    import time
    t = str(int(time.time()*1000))
    captcha_url = "https://www.zhihu.com/captcha.gif?r={0}&type=login".format(t)
    t = session.get(captcha_url, headers=header)
    with open("captcha.jpg","wb") as f:
        f.write(t.content)
        f.close()

    from PIL import Image
    try:
        im = Image.open('captcha.jpg')
        im.show()
        im.close()
    except:
        pass

    captcha = input("?????\n>")
    return captcha 
Example 3
Project: pycos   Author: pgiri   File: dispycos.py    (license) View Source Project 6 votes vote down vote up
def run_async_at(self, where, gen, *args, **kwargs):
        """Must be used with 'yield' as

        'rtask = yield computation.run_async_at(where, gen, ...)'

        Run given generator function 'gen' with arguments 'args' and 'kwargs' at
        remote server 'where'.  If the request is successful, 'rtask' will be a
        (remote) task; check result with 'isinstance(rtask,
        pycos.Task)'. The generator is supposed to be (mostly) I/O bound and
        not consume CPU time. Unlike other 'run' variants, tasks created
        with 'async' are not "tracked" by scheduler (see online documentation for
        more details).

        If 'where' is a string, it is assumed to be IP address of a node, in
        which case the task is scheduled at that node on a server at that
        node. If 'where' is a Location instance, it is assumed to be server
        location in which case the task is scheduled at that server.

        'gen' must be generator function, as it is used to run task at
        remote location.
        """
        yield self._run_request('run_async', where, 0, gen, *args, **kwargs) 
Example 4
Project: pycos   Author: pgiri   File: dispycos.py    (license) View Source Project 6 votes vote down vote up
def _pulse_proc(self, task=None):
        """For internal use only.
        """
        task.set_daemon()
        last_pulse = time.time()
        timeout = 2 * self._pulse_interval
        while 1:
            msg = yield task.receive(timeout=timeout)
            if msg == 'pulse':
                last_pulse = time.time()
            elif msg == 'quit':
                break
            elif msg is None and (time.time() - last_pulse) > (10 * self._pulse_interval):
                logger.warning('scheduler may have gone away!')
            else:
                logger.debug('ignoring invalid pulse message')
        self._pulse_task = None 
Example 5
Project: pycos   Author: pgiri   File: dispycos.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, name, addr):
            self.name = name
            self.addr = addr
            self.cpus_used = 0
            self.cpus = 0
            self.platform = None
            self.avail_info = None
            self.servers = {}
            self.disabled_servers = {}
            self.load = 0.0
            self.status = Scheduler.NodeClosed
            self.task = None
            self.last_pulse = time.time()
            self.lock = pycos.Lock()
            self.avail = pycos.Event()
            self.avail.clear() 
Example 6
Project: pycos   Author: pgiri   File: httpd.py    (license) View Source Project 6 votes vote down vote up
def shutdown(self, wait=True):
        """This method should be called by user program to close the
        http server. If 'wait' is True the server waits for poll_sec
        so the http client gets all the updates before server is
        closed.
        """
        if wait:
            pycos.logger.info('HTTP server waiting for %s seconds for client updates '
                              'before quitting', self._poll_sec)
            if pycos.Pycos().cur_task():
                def _shutdown(task=None):
                    yield task.sleep(self._poll_sec + 0.5)
                    self._server.shutdown()
                    self._server.server_close()
                pycos.Task(_shutdown)
            else:
                time.sleep(self._poll_sec + 0.5)
                self._server.shutdown()
                self._server.server_close()
        else:
            self._server.shutdown()
            self._server.server_close() 
Example 7
Project: pycos   Author: pgiri   File: dispycos.py    (license) View Source Project 6 votes vote down vote up
def _pulse_proc(self, task=None):
        """For internal use only.
        """
        task.set_daemon()
        last_pulse = time.time()
        timeout = 2 * self._pulse_interval
        while 1:
            msg = yield task.receive(timeout=timeout)
            if msg == 'pulse':
                last_pulse = time.time()
            elif msg == 'quit':
                break
            elif msg is None and (time.time() - last_pulse) > (10 * self._pulse_interval):
                logger.warning('scheduler may have gone away!')
            else:
                logger.debug('ignoring invalid pulse message')
        self._pulse_task = None 
Example 8
Project: pycos   Author: pgiri   File: dispycos.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, name, addr):
            self.name = name
            self.addr = addr
            self.cpus_used = 0
            self.cpus = 0
            self.platform = None
            self.avail_info = None
            self.servers = {}
            self.disabled_servers = {}
            self.load = 0.0
            self.status = Scheduler.NodeClosed
            self.task = None
            self.last_pulse = time.time()
            self.lock = pycos.Lock()
            self.avail = pycos.Event()
            self.avail.clear() 
Example 9
Project: pycos   Author: pgiri   File: httpd.py    (license) View Source Project 6 votes vote down vote up
def shutdown(self, wait=True):
        """This method should be called by user program to close the
        http server. If 'wait' is True the server waits for poll_sec
        so the http client gets all the updates before server is
        closed.
        """
        if wait:
            pycos.logger.info('HTTP server waiting for %s seconds for client updates '
                              'before quitting', self._poll_sec)
            if pycos.Pycos().cur_task():
                def _shutdown(task=None):
                    yield task.sleep(self._poll_sec + 0.5)
                    self._server.shutdown()
                    self._server.server_close()
                pycos.Task(_shutdown)
            else:
                time.sleep(self._poll_sec + 0.5)
                self._server.shutdown()
                self._server.server_close()
        else:
            self._server.shutdown()
            self._server.server_close() 
Example 10
Project: attract-repel   Author: nmrksic   File: attract-repel.py    (Apache License 2.0) View Source Project 6 votes vote down vote up
def create_vector_dictionary(self):
        """
        Extracts the current word vectors from TensorFlow embeddings and (if print_simlex=True) prints their SimLex scores. 
        """
        log_time = time.time()

        [current_vectors] = self.sess.run([self.W_dynamic])
        self.word_vectors = {}
        for idx in range(0, self.vocabulary_size):
            self.word_vectors[self.inverted_index[idx]] = normalise_vector(current_vectors[idx, :])

        if self.log_scores_over_time or self.print_simlex:
            (score_simlex, score_wordsim) = simlex_scores(self.word_vectors, self.print_simlex)
            return (score_simlex, score_wordsim)

        return (1.0, 1.0) 
Example 11
Project: charm-plumgrid-gateway   Author: openstack   File: utils.py    (Apache License 2.0) View Source Project 6 votes vote down vote up
def service_restarted(self, sentry_unit, service, filename,
                          pgrep_full=None, sleep_time=20):
        """Check if service was restarted.

           Compare a service's start time vs a file's last modification time
           (such as a config file for that service) to determine if the service
           has been restarted.
           """
        # /!\ DEPRECATION WARNING (beisner):
        # This method is prone to races in that no before-time is known.
        # Use validate_service_config_changed instead.

        # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
        # used instead of pgrep.  pgrep_full is still passed through to ensure
        # deprecation WARNS.  lp1474030
        self.log.warn('DEPRECATION WARNING:  use '
                      'validate_service_config_changed instead of '
                      'service_restarted due to known races.')

        time.sleep(sleep_time)
        if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
                self._get_file_mtime(sentry_unit, filename)):
            return True
        else:
            return False 
Example 12
Project: socialhome   Author: jaywink   File: streams.py    (GNU Affero General Public License v3.0) View Source Project 6 votes vote down vote up
def add_to_redis(content, through, keys):
    """Add content to a list of Redis ordered sets.

    :param content: Content object to add
    :param through: Content through object. For example on shares, this is the linked share content object
    :param keys: List of keys to add to
    """
    if not keys:
        return
    r = get_redis_connection()
    for key in keys:
        # Only add if not in the set already
        # This stops shares popping up more than once, for example
        if not r.zrank(key, content.id):
            r.zadd(key, int(time.time()), content.id)
            r.hset(BaseStream.get_throughs_key(key), content.id, through.id) 
Example 13
Project: kiota   Author: Morteo   File: Gateway.py    (MIT License) View Source Project 6 votes vote down vote up
def start(self):

    while True:
      try:
        self.connect()
        while True:
          #time.sleep(0.01) # attempt to reduce number of OSError: [Errno 104] ECONNRESET 
          self.client.check_msg()
          #time.sleep(0.01) # attempt to reduce number of OSError: [Errno 104] ECONNRESET 
          self.push()
          time.sleep(0.01)
      except OSError as e:
          Util.log(self,"failed to connect, retrying....", e)
          time.sleep(self.config["wait_to_reconnect"])

    self.client.disconnect() 
Example 14
Project: kuryr-kubernetes   Author: openstack   File: test_retry.py    (Apache License 2.0) View Source Project 6 votes vote down vote up
def _test_should_sleep(self, seconds_left, slept):
        attempt = 5
        timeout = 20
        interval = 3
        randint = 2
        deadline = self.now + seconds_left
        retry = h_retry.Retry(mock.Mock(), timeout=timeout, interval=interval)

        with mock.patch('random.randint') as m_randint, \
                mock.patch('time.sleep') as m_sleep:
            m_randint.return_value = randint

            ret = retry._sleep(deadline, attempt, _EX2())

            self.assertEqual(slept, ret)
            m_randint.assert_called_once_with(1, 2 ** attempt - 1)
            m_sleep.assert_called_once_with(slept) 
Example 15
Project: kuryr-kubernetes   Author: openstack   File: vif_pool.py    (Apache License 2.0) View Source Project 6 votes vote down vote up
def _populate_pool(self, pool_key, pod, subnets):
        # REVISIT(ltomasbo): Drop the subnets parameter and get the information
        # from the pool_key, which will be required when multi-network is
        # supported
        now = time.time()
        if (now - oslo_cfg.CONF.vif_pool.ports_pool_update_frequency <
                self._last_update.get(pool_key, 0)):
            LOG.info("Not enough time since the last pool update")
            return
        self._last_update[pool_key] = now

        pool_size = self._get_pool_size(pool_key)
        if pool_size < oslo_cfg.CONF.vif_pool.ports_pool_min:
            num_ports = max(oslo_cfg.CONF.vif_pool.ports_pool_batch,
                            oslo_cfg.CONF.vif_pool.ports_pool_min - pool_size)
            vifs = self._drv_vif.request_vifs(
                pod=pod,
                project_id=pool_key[1],
                subnets=subnets,
                security_groups=list(pool_key[2]),
                num_ports=num_ports)
            for vif in vifs:
                self._existing_vifs[vif.id] = vif
                self._available_ports_pools.setdefault(pool_key,
                                                       []).append(vif.id) 
Example 16
Project: toxxmlrpc   Author: merlink01   File: tox_xmlrpc_client.py    (GNU General Public License v2.0) View Source Project 6 votes vote down vote up
def start(self):
        self.client.start()
        #~ if not self.disable_auto_login:
            #~ while self.client.status == 'offline':
                #~ time.sleep(1)
            #~ logger.info('Client: %s'%self.client.status)
        if self.server_id:
            already_added = False
            for f in self.client.get_friend_list():
                if self.client.friend_get_public_key(f) in self.server_id:
                    already_added = True
                    logger.info('Server already in added')
                    break
            if not already_added:
                self.client.friend_add_with_request(self.server_id,self.password)
                logger.info('Started Friend request to Server')
        else:
            logger.info('No Server ID given') 
Example 17
Project: toxxmlrpc   Author: merlink01   File: tox_xmlrpc_client.py    (GNU General Public License v2.0) View Source Project 6 votes vote down vote up
def __request(self,methodname,args):
        logger.info('Execute: %s%s'%(methodname,repr(args)))
        data = xmlrpclib.dumps(args,methodname,allow_none=True)

        self.exec_lock.acquire()
        if not self.client.data_send(0,data,self.timeout):
            logger.warning('Raising Error, Timeout reached')
            self.exec_lock.release()
            raise IOError, 'Timeout'
        recdata = None

        time_to_wait = int(time.time()) + self.timeout
        while not recdata:
            timenow = int(time.time())
            if timenow > time_to_wait:
                logger.warning('Raising Error, Timeout reached')
                self.exec_lock.release()
                raise IOError, 'Timeout'
            recdata = self.client.data_recv()
            time.sleep(0.1)
        self.exec_lock.release()

        returndata = xmlrpclib.loads(recdata['data'],use_datetime=True)
        logger.info('got %s'%str(returndata))
        return returndata[0][0] 
Example 18
Project: drl.pth   Author: seba-1511   File: training.py    (license) View Source Project 6 votes vote down vote up
def test(args, env, agent):
    if args.record:
        if 'env' in vars(args):
            env = wrappers.Monitor(env, './videos/' + args.env + str(time()) + '/')
        else:
            env = wrappers.Monitor(env, './videos/' + str(time()) + '/')
    test_rewards = []
    test_start = time()
    test_steps = 0
    for iteration in range(1, 1 + args.n_test_iter):
        state = env.reset()
        iter_rewards = 0.0
        done = False
        while not done:
            test_steps += 1
            action, _ = agent.forward(state)
            state, reward, done, _ = env.step(action)
            iter_rewards += reward
        test_rewards.append(iter_rewards)
    print_stats('Test', test_rewards, args.n_test_iter,
                time() - test_start, test_steps, 0, agent)
    return test_rewards 
Example 19
Project: DREAM   Author: LaceyChen17   File: eval.py    (MIT License) View Source Project 6 votes vote down vote up
def eval_pred(dr_model, ub):
    '''
        evaluate dream model for predicting next basket on all training users
        in batches
    '''
    item_embedding = dr_model.encode.weight
    dr_model.eval()
    dr_hidden = dr_model.init_hidden(dr_model.config.batch_size)
    start_time = time()
    id_u, score_u = [], [] # user's id, user's score
    num_batchs = ceil(len(ub) / dr_model.config.batch_size)
    for i,x in enumerate(batchify(ub, dr_model.config.batch_size)):
        print(i)
        baskets, lens, uids = x
        _, dynamic_user, _ = dr_model(baskets, lens, dr_hidden)# shape: batch_size, max_len, embedding_size
        dr_hidden = repackage_hidden(dr_hidden)
        for i,l,du in zip(uids, lens, dynamic_user):
            du_latest = du[l - 1].unsqueeze(0) # shape: 1, embedding_size
            score_up = torch.mm(du_latest, item_embedding.t()) # shape: 1, num_item
            score_u.append(score_up.cpu().data.numpy())
            id_u.append(i)
    elapsed = time() - start_time 
    print('[Predicting] Elapsed: {02.2f}'.format(elapsed))
    return score_ub, id_u 
Example 20
Project: DREAM   Author: LaceyChen17   File: train.py    (MIT License) View Source Project 6 votes vote down vote up
def evaluate_dream():
    dr_model.eval()
    dr_hidden = dr_model.init_hidden(dr_config.batch_size) 
    
    total_loss = 0
    start_time = time()
    num_batchs = ceil(len(test_ub) / dr_config.batch_size)
    for i,x in enumerate(batchify(test_ub, dr_config.batch_size)):
        baskets, lens, _ = x
        dynamic_user, _  = dr_model(baskets, lens, dr_hidden)
        loss = bpr_loss(baskets, dynamic_user, dr_model.encode.weight, dr_config)
        dr_hidden = repackage_hidden(dr_hidden)
        total_loss += loss.data
        
    # Logging
    elapsed = (time() - start_time) * 1000 / num_batchs
    total_loss = total_loss[0] / num_batchs
    print('[Evaluation]| Epochs {:3d} | Elapsed {:02.2f} | Loss {:05.2f} |'.format(epoch, elapsed, total_loss))
    return total_loss 
Example 21
Project: Starfish   Author: BillWang139967   File: nbNetBase.py    (GNU General Public License v3.0) View Source Project 6 votes vote down vote up
def check_fd(self):
        '''??fd??
        ??read ?????????????????
        ?????????????
        '''
        while True:
            for fd in self.conn_state.keys():
                sock_state = self.conn_state[fd]
                # fd?read???? read_time ???
                # ???fd?epoll?????????????????
                if sock_state.state == "read" and sock_state.read_stime \
                    and (time.time() - sock_state.read_stime) >= sock_state.read_itime:
                    # ??????????fd
                    sock_state.state = "closing"
                    self.state_machine(fd)
            # ??????
            time.sleep(60)

#}}}
#{{{fork_processes 
Example 22
Project: Starfish   Author: BillWang139967   File: nbNetBase.py    (GNU General Public License v3.0) View Source Project 6 votes vote down vote up
def check_fd(self):
        '''??fd??
        ??read ?????????????????
        ?????????????
        '''
        while True:
            for fd in self.conn_state.keys():
                sock_state = self.conn_state[fd]
                # fd?read???? read_time ???
                # ???fd?epoll?????????????????
                if sock_state.state == "read" and sock_state.read_stime \
                    and (time.time() - sock_state.read_stime) >= sock_state.read_itime:
                    # ??????????fd
                    sock_state.state = "closing"
                    self.state_machine(fd)
            # ??????
            time.sleep(60)

#}}}
#{{{fork_processes 
Example 23
Project: Starfish   Author: BillWang139967   File: si.py    (GNU General Public License v3.0) View Source Project 6 votes vote down vote up
def uptime(self):
        with open('/proc/uptime', 'r') as f:
            uptime, idletime = f.readline().split()
            up_seconds = int(float(uptime))
            idle_seconds = int(float(idletime))
            # in some machine like Linode VPS, idle time may bigger than up time
            if idle_seconds > up_seconds:
                cpu_count = multiprocessing.cpu_count()
                idle_seconds = idle_seconds/cpu_count
                # in some VPS, this value may still bigger than up time
                # may be the domain 0 machine has more cores
                # we calclate approximately for it
                if idle_seconds > up_seconds:
                    for n in range(2,10):
                        if idle_seconds/n < up_seconds:
                            idle_seconds = idle_seconds/n
                            break
            fmt = '{days} ? {hours} ?? {minutes} ? {seconds} ?'
            uptime_string = strfdelta(datetime.timedelta(seconds = up_seconds), fmt)
            idletime_string = strfdelta(datetime.timedelta(seconds = idle_seconds), fmt)
        return {
            'up': uptime_string,
            'idle': idletime_string,
            'idle_rate': div_percent(idle_seconds, up_seconds),
        } 
Example 24
Project: piksi_ros   Author: uscresl   File: piksi_driver.py    (MIT License) View Source Project 6 votes vote down vote up
def setup_pubsub(self):

        freq_params = diagnostic_updater.FrequencyStatusParam({'min':self.diag_update_freq, 'max':self.diag_update_freq}, self.diag_freq_tolerance, self.diag_window_size)
        time_params = diagnostic_updater.TimeStampStatusParam(self.diag_min_delay, self.diag_max_delay)

        self.pub_fix = rospy.Publisher("~fix", NavSatFix, queue_size=1000)

        self.pub_spp_fix = diagnostic_updater.DiagnosedPublisher(rospy.Publisher("~spp_fix", NavSatFix, queue_size=1000), self.diag_updater, freq_params, time_params)

        self.pub_rtk_fix = diagnostic_updater.DiagnosedPublisher(rospy.Publisher("~rtk_fix", NavSatFix, queue_size=1000), self.diag_updater, freq_params, time_params)
        #self.pub_rtk = diagnostic_updater.DiagnosedPublisher(rospy.Publisher("~rtk_odom", Odometry, queue_size=1000), self.diag_updater, freq_params, time_params)
        self.pub_odom = diagnostic_updater.DiagnosedPublisher(rospy.Publisher("~odom", Odometry, queue_size=1000), self.diag_updater, freq_params, time_params)
        self.pub_time = diagnostic_updater.DiagnosedPublisher(rospy.Publisher("~time", TimeReference, queue_size=1000), self.diag_updater, freq_params, time_params)

        if self.publish_utm_rtk_tf or self.publish_rtk_child_tf:
            self.tf_br = tf2_ros.TransformBroadcaster()

        if self.publish_ephemeris:
            self.pub_eph = rospy.Publisher("~ephemeris", Ephemeris, queue_size=1000)

        if self.publish_observations:
            self.pub_obs = rospy.Publisher('~observations', Observations, queue_size=1000) 
Example 25
Project: pyOSC3   Author: Qirky   File: OSC3.py    (license) View Source Project 6 votes vote down vote up
def append(self, argument, typehint = None):
		"""Appends data to the bundle, creating an OSCMessage to encapsulate
		the provided argument unless this is already an OSCMessage.
		Any newly created OSCMessage inherits the OSCBundle's address at the time of creation.
		If 'argument' is an iterable, its elements will be encapsuated by a single OSCMessage.
		Finally, 'argument' can be (or contain) a dict, which will be 'converted' to an OSCMessage;
		  - if 'addr' appears in the dict, its value overrides the OSCBundle's address
		  - if 'args' appears in the dict, its value(s) become the OSCMessage's arguments
		"""
		if isinstance(argument, OSCMessage):
			binary = OSCBlob(argument.getBinary())
		else:
			msg = OSCMessage(self.address)
			if type(argument) == dict:
				if 'addr' in argument:
					msg.setAddress(argument['addr'])
				if 'args' in argument:
					msg.append(argument['args'], typehint)
			else:
				msg.append(argument, typehint)
			
			binary = OSCBlob(msg.getBinary())

		self.message += binary
		self.typetags += 'b' 
Example 26
Project: pyOSC3   Author: Qirky   File: OSC3.py    (license) View Source Project 6 votes vote down vote up
def OSCTimeTag(time):
	"""Convert a time in floating seconds to its
	OSC binary representation
	"""
	if time > 0:
		fract, secs = math.modf(time)
		secs = secs - NTP_epoch
		binary = struct.pack('>LL', int(secs), int(fract * NTP_units_per_second))
	else:
		binary = struct.pack('>LL', 0, 1)

	return binary

######
#
# OSCMessage decoding functions
#
###### 
Example 27
Project: pyOSC3   Author: Qirky   File: OSC2.py    (license) View Source Project 6 votes vote down vote up
def append(self, argument, typehint = None):
		"""Appends data to the bundle, creating an OSCMessage to encapsulate
		the provided argument unless this is already an OSCMessage.
		Any newly created OSCMessage inherits the OSCBundle's address at the time of creation.
		If 'argument' is an iterable, its elements will be encapsuated by a single OSCMessage.
		Finally, 'argument' can be (or contain) a dict, which will be 'converted' to an OSCMessage;
		  - if 'addr' appears in the dict, its value overrides the OSCBundle's address
		  - if 'args' appears in the dict, its value(s) become the OSCMessage's arguments
		"""
		if isinstance(argument, OSCMessage):
			binary = OSCBlob(argument.getBinary())
		else:
			msg = OSCMessage(self.address)
			if type(argument) == types.DictType:
				if 'addr' in argument:
					msg.setAddress(argument['addr'])
				if 'args' in argument:
					msg.append(argument['args'], typehint)
			else:
				msg.append(argument, typehint)
			
			binary = OSCBlob(msg.getBinary())

		self.message += binary
		self.typetags += 'b' 
Example 28
Project: pyOSC3   Author: Qirky   File: OSC2.py    (license) View Source Project 6 votes vote down vote up
def OSCTimeTag(time):
	"""Convert a time in floating seconds to its
	OSC binary representation
	"""
	if time > 0:
		fract, secs = math.modf(time)
		secs = secs - NTP_epoch
		binary = struct.pack('>LL', long(secs), long(fract * NTP_units_per_second))
	else:
		binary = struct.pack('>LL', 0L, 1L)

	return binary

######
#
# OSCMessage decoding functions
#
###### 
Example 29
Project: dsq   Author: baverman   File: cli.py    (MIT License) View Source Project 6 votes vote down vote up
def forwarder(tasks, interval, batch_size, source, dest):
    '''Forward items from one storage to another.'''
    from .utils import RunFlag, load_manager, redis_client
    from .store import QueueStore
    log = logging.getLogger('dsq.forwarder')

    if not tasks and not source:
        print('--tasks or --source must be provided')
        sys.exit(1)

    s = QueueStore(redis_client(source)) if source else load_manager(tasks).queue
    d = QueueStore(redis_client(dest))
    run = RunFlag()
    while run:
        batch = s.take_many(batch_size)
        if batch['schedule'] or batch['queues']:
            try:
                d.put_many(batch)
            except Exception:
                s.put_many(batch)
                log.exception('Forward error')
                raise
        else:
            time.sleep(interval) 
Example 30
Project: dsq   Author: baverman   File: worker.py    (MIT License) View Source Project 6 votes vote down vote up
def process(self, queue_list, burst=False):  # pragma: no cover
        signal.signal(signal.SIGALRM, self.alarm_handler)

        run = RunFlag()
        start = time()
        while run:
            task = self.manager.pop(queue_list, 1)
            if task:
                try:
                    self.process_one(task)
                except StopWorker:
                    break
            elif burst:
                break

            if self.lifetime and time() - start > self.lifetime:
                break

        self.manager.close() 
Example 31
Project: dsq   Author: baverman   File: store.py    (MIT License) View Source Project 6 votes vote down vote up
def reschedule(self, now=None):
        now = now or time()
        items, _, size = (self.client.pipeline()
                          .zrangebyscore(SCHEDULE_KEY, '-inf', now)
                          .zremrangebyscore(SCHEDULE_KEY, '-inf', now)
                          .zcard(SCHEDULE_KEY)
                          .execute())

        for chunk in iter_chunks(items, 5000):
            pipe = self.client.pipeline(False)
            for r in chunk:
                queue, _, task = r.partition(b':')
                pipe.rpush(rqname(queue), task)
            pipe.execute()

        return size 
Example 32
Project: tree-gen   Author: friggog   File: gen.py    (GNU General Public License v3.0) View Source Project 6 votes vote down vote up
def make(self):
        """make the tree"""
        start_time = time()
        if __logging__:
            print('** Generating Tree **')
        # create parent object
        self.tree_obj = bpy.data.objects.new('Tree', None)
        bpy.context.scene.objects.link(self.tree_obj)
        bpy.context.scene.objects.active = self.tree_obj
        # create branches
        self.create_branches()
        # create leaf mesh if needed
        self.create_leaf_mesh()
        g_time = time() - start_time
        if __logging__:
            print('Tree generated in %f seconds' % g_time) 
Example 33
Project: python-   Author: secondtonone1   File: ui.py    (license) View Source Project 6 votes vote down vote up
def __init__(self, *args, **kwargs):
        # The Windows terminal does not support the hide/show cursor ANSI codes
        # even with colorama. So we'll ensure that hide_cursor is False on
        # Windows.
        # This call neds to go before the super() call, so that hide_cursor
        # is set in time. The base progress bar class writes the "hide cursor"
        # code to the terminal in its init, so if we don't set this soon
        # enough, we get a "hide" with no corresponding "show"...
        if WINDOWS and self.hide_cursor:
            self.hide_cursor = False

        super(WindowsMixin, self).__init__(*args, **kwargs)

        # Check if we are running on Windows and we have the colorama module,
        # if we do then wrap our file with it.
        if WINDOWS and colorama:
            self.file = colorama.AnsiToWin32(self.file)
            # The progress code expects to be able to call self.file.isatty()
            # but the colorama.AnsiToWin32() object doesn't have that, so we'll
            # add it.
            self.file.isatty = lambda: self.file.wrapped.isatty()
            # The progress code expects to be able to call self.file.flush()
            # but the colorama.AnsiToWin32() object doesn't have that, so we'll
            # add it.
            self.file.flush = lambda: self.file.wrapped.flush() 
Example 34
Project: python-   Author: secondtonone1   File: random.py    (license) View Source Project 6 votes vote down vote up
def _test_generator(n, func, args):
    import time
    print(n, 'times', func.__name__)
    total = 0.0
    sqsum = 0.0
    smallest = 1e10
    largest = -1e10
    t0 = time.time()
    for i in range(n):
        x = func(*args)
        total += x
        sqsum = sqsum + x*x
        smallest = min(x, smallest)
        largest = max(x, largest)
    t1 = time.time()
    print(round(t1-t0, 3), 'sec,', end=' ')
    avg = total/n
    stddev = _sqrt(sqsum/n - avg*avg)
    print('avg %g, stddev %g, min %g, max %g\n' % \
              (avg, stddev, smallest, largest)) 
Example 35
Project: python-   Author: secondtonone1   File: random.py    (license) View Source Project 6 votes vote down vote up
def _test(N=2000):
    _test_generator(N, random, ())
    _test_generator(N, normalvariate, (0.0, 1.0))
    _test_generator(N, lognormvariate, (0.0, 1.0))
    _test_generator(N, vonmisesvariate, (0.0, 1.0))
    _test_generator(N, gammavariate, (0.01, 1.0))
    _test_generator(N, gammavariate, (0.1, 1.0))
    _test_generator(N, gammavariate, (0.1, 2.0))
    _test_generator(N, gammavariate, (0.5, 1.0))
    _test_generator(N, gammavariate, (0.9, 1.0))
    _test_generator(N, gammavariate, (1.0, 1.0))
    _test_generator(N, gammavariate, (2.0, 1.0))
    _test_generator(N, gammavariate, (20.0, 1.0))
    _test_generator(N, gammavariate, (200.0, 1.0))
    _test_generator(N, gauss, (0.0, 1.0))
    _test_generator(N, betavariate, (3.0, 3.0))
    _test_generator(N, triangular, (0.0, 1.0, 1.0/3.0))

# Create one instance, seeded from current time, and export its methods
# as module-level functions.  The functions share state across all uses
#(both in the user's code and in the Python libraries), but that's fine
# for most programs and is easier for the casual user than making them
# instantiate their own Random() instance. 
Example 36
Project: PGO-mapscan-opt   Author: seikur0   File: main0.py    (license) View Source Project 6 votes vote down vote up
def do_full_login(account):
    lock_network.acquire()
    time.sleep(locktime)
    lock_network.release()
    if account['type'] == 'ptc':
        login_ptc(account)
    elif account['type'] == 'google':
        login_google(account)
        new_session(account)
    else:
        lprint('[{}] Error: Login type should be either ptc or google.'.format(account['num']))
        sys.exit()

    cursor_accs = db_accs.cursor()
    while True:
        try:
            cursor_accs.execute("INSERT OR REPLACE INTO accounts VALUES(?,?,?,?,?,?,?)", [account['user'], account['access_token'], account['access_expire_timestamp'], account['api_url'], 0, '0', '0'])
            db_accs.commit()
            return
        except sqlite3.OperationalError as e:
            lprint('[-] Sqlite operational error: {}, account: {} Retrying...'.format(e, account['user']))
        except sqlite3.InterfaceError as e:
            lprint('[-] Sqlite interface error: {}, account: {} Retrying...'.format(e, account['user'])) 
Example 37
Project: PGO-mapscan-opt   Author: seikur0   File: main0.py    (license) View Source Project 6 votes vote down vote up
def update_data():
    timenow = int(round(time.time(),0))
    cursor_data = db_data.cursor()
    for l in range(0,len(data_buffer)):
        [pokeid, spawnid, latitude, longitude, expiretime, addinfo] = data_buffer.pop()
        db_repeat = True
        while db_repeat:
            try:
                cursor_data.execute("INSERT OR REPLACE INTO spawns VALUES(?,?,?,?,?,?,?,?)", [spawnid, round(latitude, 5), round(longitude, 5), addinfo, pokeid, expiretime, timenow, wID])
                db_repeat = False
            except sqlite3.OperationalError as e:
                lprint('[-] Sqlite operational error: {} Retrying...'.format(e))

    while True:
        try:
            db_data.commit()
            return
        except sqlite3.OperationalError as e:
            lprint('[-] Sqlite operational error: {} Retrying...'.format(e)) 
Example 38
Project: nova   Author: hubblestack   File: cve_scan_v2.py    (license) View Source Project 6 votes vote down vote up
def _get_cache(ttl, cache_path):
    '''
    If url contains valid cache, returns it, else returns empty list.
    '''
    # Check if we have a valid cached version.
    try:
        cached_time = os.path.getmtime(cache_path)
    except OSError:
        return []
    if current_time() - cached_time < ttl:
        log.debug('%s is less than ttl', cache_path)
        try:
            with open(cache_path) as json_file:
                loaded_json = json.load(json_file)
                return loaded_json
        except IOError:
            return []
        except ValueError:
            log.error('%s was not json formatted', cache_path)
            return []
    else:
        log.debug('%s was older than ttl', cache_path)
        return [] 
Example 39
Project: rca-evaluation   Author: sieve-microservices   File: rca.py    (license) View Source Project 6 votes vote down vote up
def run_demo(updated_services):

    # build both graphs from the edge descriptions
    graph_a = Graph()
    build_dummy_graph(GRAPH_A, graph_a)

    graph_b = Graph()
    build_dummy_graph(GRAPH_B, graph_b)

    # print them as .dot files for reference
    graph_a.print_graph("graph-a", True, True)
    graph_b.print_graph("graph-b", False, True)

    # generate the differences by considering an update on service 'a'
    start_time = time.time()
    diff_graph = graph_a.get_diff(graph_b, updated_services)
    print("--- graph differences calculated in %s seconds --- " % (time.time() - start_time))
    diff_graph.print_graph("graph-diff", False, True) 
Example 40
Project: ArticleSpider   Author: mtianyan   File: zhihu.py    (license) View Source Project 5 votes vote down vote up
def login(self, response):
        response_text = response.text
        match_obj = re.match('.*name="_xsrf" value="(.*?)"', response_text, re.DOTALL)
        xsrf = ''
        if match_obj:
            xsrf = (match_obj.group(1))

        if xsrf:
            post_url = "https://www.zhihu.com/login/phone_num"
            post_data = {
                "_xsrf": xsrf,
                "phone_num": "18487255487",
                "password": "ty158917",
                "captcha": ""
            }

            import time
            t = str(int(time.time() * 1000))
            captcha_url = "https://www.zhihu.com/captcha.gif?r={0}&type=login".format(t)
            yield scrapy.Request(captcha_url, headers=self.headers, meta={"post_data":post_data}, callback=self.login_after_captcha) 
Example 41
Project: pycos   Author: pgiri   File: tasks.py    (license) View Source Project 5 votes vote down vote up
def pycos_proc(n, task=None):
    s = random.uniform(0.5, 3)
    print('%f: process %d sleeping for %f seconds' % (time.time(), n, s))
    yield task.sleep(s)
    print('%f: process %d terminating' % (time.time(), n))

# create 10 clients 
Example 42
Project: pycos   Author: pgiri   File: dispycos.py    (license) View Source Project 5 votes vote down vote up
def _run_request(self, request, where, cpu, gen, *args, **kwargs):
        """Internal use only.
        """
        if isinstance(gen, str):
            name = gen
        else:
            name = gen.func_name

        if name in self._xfer_funcs:
            code = None
        else:
            # if not inspect.isgeneratorfunction(gen):
            #     logger.warning('"%s" is not a valid generator function', name)
            #     raise StopIteration([])
            code = inspect.getsource(gen).lstrip()

        def _run_req(task=None):
            msg = {'req': 'job', 'auth': self._auth,
                   'job': _DispycosJob_(request, task, name, where, cpu, code, args, kwargs)}
            if (yield self.scheduler.deliver(msg, timeout=MsgTimeout)) == 1:
                reply = yield task.receive()
                if isinstance(reply, Task):
                    if self.status_task:
                        msg = DispycosTaskInfo(reply, args, kwargs, time.time())
                        self.status_task.send(DispycosStatus(Scheduler.TaskCreated, msg))
                if not request.endswith('async'):
                    reply = yield task.receive()
            else:
                reply = None
            raise StopIteration(reply)

        yield Task(_run_req).finish() 
Example 43
Project: pycos   Author: pgiri   File: tasks.py    (license) View Source Project 5 votes vote down vote up
def pycos_proc(n, task=None):
    s = random.uniform(0.5, 3)
    print('%f: process %d sleeping for %f seconds' % (time.time(), n, s))
    yield task.sleep(s)
    print('%f: process %d terminating' % (time.time(), n))

# create 10 clients 
Example 44
Project: pycos   Author: pgiri   File: httpd.py    (license) View Source Project 5 votes vote down vote up
def __init__(self, name, addr):
            self.name = name
            self.addr = addr
            self.status = None
            self.servers = {}
            self.update_time = time.time()
            self.tasks_submitted = 0
            self.tasks_done = 0
            self.avail_info = None 
Example 45
Project: pycos   Author: pgiri   File: __init__.py    (license) View Source Project 5 votes vote down vote up
def _add_timeout(self, fd):
                if fd._timeout:
                    self._lock.acquire()
                    fd._timeout_id = _time() + fd._timeout + 0.0001
                    i = bisect_left(self._timeouts, (fd._timeout_id, fd))
                    self._timeouts.insert(i, (fd._timeout_id, fd))
                    if self._polling:
                        self.interrupt()
                    self._lock.release()
                else:
                    fd._timeout_id = None 
Example 46
Project: pycos   Author: pgiri   File: __init__.py    (license) View Source Project 5 votes vote down vote up
def _add_timeout(self, fd):
            if fd._timeout:
                fd._timeout_id = _time() + fd._timeout + 0.0001
                i = bisect_left(self._timeouts, (fd._timeout_id, fd))
                self._timeouts.insert(i, (fd._timeout_id, fd))
            else:
                fd._timeout_id = None 
Example 47
Project: pycos   Author: pgiri   File: __init__.py    (license) View Source Project 5 votes vote down vote up
def wait(self, timeout=None):
        """Must be used with 'yield' as 'yield cv.wait()'.
        """
        task = Pycos.cur_task(self._scheduler)
        if self._owner != task:
            raise RuntimeError('"%s"/%s: invalid lock release - owned by "%s"/%s' %
                               (task._name, task._id, self._owner._name, self._owner._id))
        assert self._depth > 0
        depth = self._depth
        self._depth = 0
        self._owner = None
        if self._waitlist:
            wake = self._waitlist.pop(0)
            wake._proceed_(True)
        self._notifylist.append(task)
        start = _time()
        if (yield task._await_(timeout)) is None:
            try:
                self._notifylist.remove(task)
            except ValueError:
                pass
            raise StopIteration(False)
        while self._owner is not None:
            self._waitlist.insert(0, task)
            if timeout is not None:
                timeout -= (_time() - start)
                if timeout <= 0:
                    raise StopIteration(False)
                start = _time()
            if (yield task._await_(timeout)) is None:
                try:
                    self._waitlist.remove(task)
                except ValueError:
                    pass
                raise StopIteration(False)
        assert self._depth == 0
        self._owner = task
        self._depth = depth
        raise StopIteration(True) 
Example 48
Project: pycos   Author: pgiri   File: __init__.py    (license) View Source Project 5 votes vote down vote up
def receive(self, category=None, timeout=None, alarm_value=None):
        """Similar to 'receive' of Task, except it retrieves (waiting, if
        necessary) messages in given 'category'.
        """
        # assert Pycos.cur_task() == self._task
        c = self._categories.get(category, None)
        if c:
            msg = c.popleft()
            raise StopIteration(msg)
        if timeout:
            start = _time()
        while 1:
            msg = yield self._task.receive(timeout=timeout, alarm_value=alarm_value)
            if msg == alarm_value:
                raise StopIteration(msg)
            for categorize in self._categorize:
                c = categorize(msg)
                if c == category:
                    raise StopIteration(msg)
                if c is not None:
                    bucket = self._categories.get(c, None)
                    if not bucket:
                        bucket = self._categories[c] = collections.deque()
                    bucket.append(msg)
                    break
            else:
                self._categories[None].append(msg)
            if timeout:
                now = _time()
                timeout -= now - start
                start = now 
Example 49
Project: pycos   Author: pgiri   File: __init__.py    (license) View Source Project 5 votes vote down vote up
def _suspend(self, task, timeout, alarm_value, state):
        """Internal use only. See sleep/suspend in Task.
        """
        self._lock.acquire()
        if self.__cur_task != task:
            self._lock.release()
            logger.warning('invalid "suspend" - "%s" != "%s"', task, self.__cur_task)
            return -1
        tid = task._id
        if state == Pycos._AwaitMsg_ and task._msgs:
            s, update = task._msgs[0]
            if s == state:
                task._msgs.popleft()
                self._lock.release()
                return update
        if timeout is None:
            task._timeout = None
        else:
            if not isinstance(timeout, (float, int)):
                logger.warning('invalid timeout %s', timeout)
                self._lock.release()
                return -1
            if timeout <= 0:
                self._lock.release()
                return alarm_value
            else:
                task._timeout = _time() + timeout + 0.0001
                heappush(self._timeouts, (task._timeout, tid, alarm_value))
        self._scheduled.discard(tid)
        self._suspended.add(tid)
        task._state = state
        self._lock.release()
        return 0 
Example 50
Project: pycos   Author: pgiri   File: dispycos.py    (license) View Source Project 5 votes vote down vote up
def _run_request(self, request, where, cpu, gen, *args, **kwargs):
        """Internal use only.
        """
        if isinstance(gen, str):
            name = gen
        else:
            name = gen.__name__

        if name in self._xfer_funcs:
            code = None
        else:
            # if not inspect.isgeneratorfunction(gen):
            #     logger.warning('"%s" is not a valid generator function', name)
            #     raise StopIteration([])
            code = inspect.getsource(gen).lstrip()

        def _run_req(task=None):
            msg = {'req': 'job', 'auth': self._auth,
                   'job': _DispycosJob_(request, task, name, where, cpu, code, args, kwargs)}
            if (yield self.scheduler.deliver(msg, timeout=MsgTimeout)) == 1:
                reply = yield task.receive()
                if isinstance(reply, Task):
                    if self.status_task:
                        msg = DispycosTaskInfo(reply, args, kwargs, time.time())
                        self.status_task.send(DispycosStatus(Scheduler.TaskCreated, msg))
                if not request.endswith('async'):
                    reply = yield task.receive()
            else:
                reply = None
            raise StopIteration(reply)

        yield Task(_run_req).finish()