Python time.time() Examples

def __init__(self, name=""):
        self.barsToProgress = {}

        self.t0 = time.time()
        self.timeRemaining = "--"
        self.status = "+"
        self.name = name
        self.lastRedraw = time.time()
        self.isatty = sys.stdout.isatty()

        try:
            self.handleResize(None,None)
            signal.signal(signal.SIGWINCH, self.handleResize)
            self.signal_set = True
        except:
            self.term_width = 79 

def start(self):       
        
        ####################
        start = time.time()
        ####################         

        self._create_folder_structure()
        self._clear_previous_executions()        
        self._add_ipynb()  
        self._get_flow_results()
        self._add_network_context()
        self._add_geo_localization()
        self._add_reputation()        
        self._create_flow_scores()
        self._get_oa_details()
        self._ingest_summary()

        ##################
        end = time.time()
        print(end - start)
        ################## 

def start(self):

        ####################
        start = time.time()
        ####################

        self._create_folder_structure()
        self._clear_previous_executions()   
        self._add_ipynb()
        self._get_proxy_results()
        self._add_reputation() 
        self._add_iana()
        self._add_network_context() 
        self._create_proxy_scores_csv()
        self._get_oa_details()
        self._ingest_summary()


        ##################
        end = time.time()
        print(end - start)
        ################## 

def start(self):

        ####################
        start = time.time()
        ####################

        self._clear_previous_executions()
        self._create_folder_structure()
        self._add_ipynb()
        self._get_dns_results()
        self._add_tld_column()
        self._add_reputation()
        self._add_hh_column()
        self._add_iana()
        self._add_network_context()
        self._create_dns_scores()
        self._get_oa_details()
        self._ingest_summary()

        ##################
        end = time.time()
        print(end - start)
        ################## 

def _calc_it_per_sec(self):
        now = time.time()
        
        if not self.last_it:
            self.last_it.append(now)
        else:
            self.last_it.append(now)
            if len(self.last_it) >= 100:
                self.last_it = self.last_it[-100:]

            # smooth it/sec
            intervals = list(map(lambda t: t[1] - t[0], zip(self.last_it[:-1], self.last_it[1:])))
            delta = sum(intervals) / len(intervals)
            if delta != 0.:
                it_per_sec = 1.0 / delta
                if it_per_sec > 1_000_00:
                    self.it_per_sec = "{}M".format(int(it_per_sec/1_000_000))
                elif it_per_sec > 1_000:
                    self.it_per_sec = "{}K".format(int(it_per_sec/1_000))
                else:
                    self.it_per_sec = "{:.2f}".format(it_per_sec)
                # 6 characters so the progress bar is the same size
                self.it_per_sec = self.it_per_sec.rjust(6)
                # self.last_it = now 

def _startImageCapturingAtGraphNode(self):
        amntY = 0.3
        xdir = [-0.5, 0, 0.5]
        cameraId = 0

        for idx,amntX in enumerate(xdir):
            self._moveHead(amntX, amntY)
            time.sleep(0.1)
            self._checkIfAsthamaInEnvironment(cameraId)
            if self.pumpFound :
                # Setting rotation angle of body towards head
                theta = 0
                if idx == 0: # LEFT
                    theta = math.radians(-45)
                if idx == 2: # RIGHT
                    theta = math.radians(45)
                self.pumpAngleRotation = theta

                return "KILLING GRAPH SEARCH : PUMP FOUND"

        self._moveHead(0, 0) # Bringing to initial view

        return 

def _capture2dImage(self, cameraId):
        # Capture Image in RGB

        # WARNING : The same Name could be used only six time.
        strName = "capture2DImage_{}".format(random.randint(1,10000000000))

        clientRGB = self.video_service.subscribeCamera(strName, cameraId, AL_kVGA, 11, 10)
        imageRGB = self.video_service.getImageRemote(clientRGB)

        imageWidth   = imageRGB[0]
        imageHeight  = imageRGB[1]
        array        = imageRGB[6]
        image_string = str(bytearray(array))

        # Create a PIL Image from our pixel array.
        im = Image.frombytes("RGB", (imageWidth, imageHeight), image_string)

        # Save the image.
        image_name_2d = "images/img2d-" + str(self.imageNo2d) + ".png"
        im.save(image_name_2d, "PNG") # Stored in images folder in the pwd, if not present then create one
        self.imageNo2d += 1
        im.show()

        return 

def _capture3dImage(self):
        # Depth Image in RGB

        # WARNING : The same Name could be used only six time.
        strName = "capture3dImage_{}".format(random.randint(1,10000000000))


        clientRGB = self.video_service.subscribeCamera(strName, AL_kDepthCamera, AL_kQVGA, 11, 10)
        imageRGB = self.video_service.getImageRemote(clientRGB)

        imageWidth  = imageRGB[0]
        imageHeight = imageRGB[1]
        array       = imageRGB[6]
        image_string = str(bytearray(array))

        # Create a PIL Image from our pixel array.
        im = Image.frombytes("RGB", (imageWidth, imageHeight), image_string)
        # Save the image.
        image_name_3d = "images/img3d-" + str(self.imageNo3d) + ".png"
        im.save(image_name_3d, "PNG") # Stored in images folder in the pwd, if not present then create one
        self.imageNo3d += 1
        im.show()

        return 

def moveForward(motion_service):
    X = 1  # forward
    Y = 0.0
    Theta = 0.0

    print "Movement Starting"
    t0= time.time()

    # Blocking call
    possible = motion_service.moveTo(X, Y, Theta)

    t1 = time.time()

    timeDiff = t1 -t0
    timeDiff *= 1000

    if not possible:
        return -1

    # if not possible:
    #     print "Interruption case Time : ",timeDiff
    # else:
    #     print "Journey over Time : ",timeDiff

    return timeDiff 

def wait_for_port(host, port, timeout=600, print_progress=True):
    if print_progress:
        sys.stderr.write("Waiting for {}:{}...".format(host, port))
        sys.stderr.flush()
    start_time = time.time()
    while True:
        try:
            socket.socket().connect((host, port))
            if print_progress:
                sys.stderr.write(GREEN("OK") + "\n")
            return
        except Exception:
            time.sleep(1)
            if print_progress:
                sys.stderr.write(".")
                sys.stderr.flush()
            if time.time() - start_time > timeout:
                raise 

def __new__(cls, t, snap=0):
        if isinstance(t, (str, bytes)) and t.isdigit():
            t = int(t)
        if not isinstance(t, (str, bytes)):
            from dateutil.tz import tzutc
            return datetime.fromtimestamp(t // 1000, tz=tzutc())
        try:
            units = ["weeks", "days", "hours", "minutes", "seconds"]
            diffs = {u: float(t[:-1]) for u in units if u.startswith(t[-1])}
            if len(diffs) == 1:
                # Snap > 0 governs the rounding of units (hours, minutes and seconds) to 0 to improve cache performance
                snap_units = {u.rstrip("s"): 0 for u in units[units.index(list(diffs)[0]) + snap:]} if snap else {}
                snap_units.pop("day", None)
                snap_units.update(microsecond=0)
                ts = datetime.now().replace(**snap_units) + relativedelta(**diffs)
                cls._precision[ts] = snap_units
                return ts
            return dateutil_parse(t)
        except (ValueError, OverflowError, AssertionError):
            raise ValueError('Could not parse "{}" as a timestamp or time delta'.format(t)) 

def main(_):
  """Runs `text_utils.simplify_nq_example` over all shards of a split.

  Prints simplified examples to a single gzipped file in the same directory
  as the input shards.
  """
  split = os.path.basename(FLAGS.data_dir)
  outpath = os.path.join(FLAGS.data_dir,
                         "simplified-nq-{}.jsonl.gz".format(split))
  with gzip.open(outpath, "wb") as fout:
    num_processed = 0
    start = time.time()
    for inpath in glob.glob(os.path.join(FLAGS.data_dir, "nq-*-??.jsonl.gz")):
      print("Processing {}".format(inpath))
      with gzip.open(inpath, "rb") as fin:
        for l in fin:
          utf8_in = l.decode("utf8", "strict")
          utf8_out = json.dumps(
              text_utils.simplify_nq_example(json.loads(utf8_in))) + u"\n"
          fout.write(utf8_out.encode("utf8"))
          num_processed += 1
          if not num_processed % 100:
            print("Processed {} examples in {}.".format(num_processed,
                                                        time.time() - start)) 

def __on_event(self):
        event = self._queue.get()
        if event.type == Events.SCAN_ERROR:
            self.__on_scan_error(event.data)
        elif event.type == Events.SCAN_DATA:
            self.__on_scan_data(event.data)
        elif event.type == Events.SERVER_ERROR:
            self.__on_server_error(event.data)
        elif event.type == Events.GPS_ERROR:
            self.__std_err(event.data['msg'])
            self.__restart_gps()
        elif event.type == Events.GPS_WARN:
            self.__std_err(event.data['msg'])
        elif event.type == Events.GPS_TIMEOUT:
            self.__std_err(event.data['msg'])
            self.__restart_gps()
        elif event.type == Events.GPS_LOC:
            self._location = event.data['loc']
        elif event.type == Events.PUSH_ERROR:
            if not self._warnedPush:
                self._warnedPush = True
                self.__std_err('Push failed:\n\t' + event.data['msg'])
        else:
            time.sleep(0.01) 

def __on_rec(self, recording):
        timestamp = time.time()
        for monitor in self._monitors:
            if not recording:
                monitor.set_level(None, timestamp, None)
            monitor.set_recording(recording, timestamp)

        if recording:
            self.__on_start()
        else:
            while self._push.hasFailed():
                resp = wx.MessageBox('Web push has failed, retry?', 'Warning',
                                     wx.OK | wx.CANCEL | wx.ICON_WARNING)
                if resp == wx.OK:
                    busy = wx.BusyInfo('Pushing...', self)
                    self._push.send_failed(self._settings.get_push_uri())
                    del busy
                else:
                    self._push.clear_failed()

        self._warnedPush = False
        self.__set_timeline() 

def current_milli_time():
    return int(round(time.time() * 1000)) 

def load_all(self):
        """The function to load all data and labels

        Give:
        data: the list of raw data, needs to be decompressed
              (e.g., raw JPEG string)
        labels: numpy array, with each element is a string
        """
        start = time.time()
        print("Start Loading Data from BCF {}".format(
            'MEMORY' if self._bcf_mode == 'MEM' else 'FILE'))

        self._labels = np.loadtxt(self._label_fn).astype(str)

        if self._bcf.size() != self._labels.shape[0]:
            raise Exception("Number of samples in data"
                            "and labels are not equal")
        else:
            for idx in range(self._bcf.size()):
                datum_str = self._bcf.get(idx)
                self._data.append(datum_str)
        end = time.time()
        print("Loading {} samples Done: Time cost {} seconds".format(
            len(self._data), end - start))

        return self._data, self._labels 

def do_realign(dataHub, sample):
    processes = dataHub.args.processes
    if processes is None or processes == 0:
        # we don't really gain from using virtual cores, so try to figure out how many physical
        # cores we have
        processes = misc.cpu_count_physical()

    variant = dataHub.variant
    reads = sample.reads
    name = "{}:{{}}".format(sample.name[:15])

    t0 = time.time()
    refalignments, badReadsRef = do1remap(variant.chromParts("ref"), reads, processes, 
        jobName=name.format("ref"), tryExact=dataHub.args.fast)
    altalignments, badReadsAlt = do1remap(variant.chromParts("alt"), reads, processes, 
        jobName=name.format("alt"), tryExact=dataHub.args.fast)
    t1 = time.time()

    logging.debug(" Time to realign: {:.1f}s".format(t1-t0))

    badReads = badReadsRef.union(badReadsAlt)

    if len(badReads) > 0:
        logging.warn(" Alignment failed with {} reads (this is a known issue)".format(badReads))
        for badRead in badReads:
            refalignments.pop(badRead, None)
            altalignments.pop(badRead, None)

    assert set(refalignments.keys()) == set(altalignments.keys()), \
                    set(refalignments.keys()) ^ set(altalignments.keys())

    alnCollections = []
    for key in refalignments:
        alnCollection = AlignmentSetCollection(key)
        alnCollection.addSet(refalignments[key], "ref")
        alnCollection.addSet(altalignments[key], "alt")
        alnCollections.append(alnCollection)

    return alnCollections 

def getReads(variant, bam, minmapq, pair_minmapq, searchDistance, single_ended=False, include_supplementary=False, max_reads=None, sample_reads=None):
    t0 = time.time()
    searchRegions = variant.searchRegions(searchDistance)

    # This cludge tries the chromosomes as given ('chr4' or '4') and if that doesn't work
    # tries to switch to the other variation ('4' or 'chr4')
    try:
        reads, supplementaryAlignmentsFound = _getreads(searchRegions, bam, minmapq, pair_minmapq, single_ended, 
            include_supplementary, max_reads, sample_reads)
    except ValueError as e:
        oldchrom = searchRegions[0].chr()
        try:
            if "chr" in oldchrom:
                newchrom = oldchrom.replace("chr", "")
                searchRegions = [Locus(l.chr().replace("chr", ""), l.start(), l.end(), l.strand()) for l in searchRegions]
            else:
                newchrom = "chr{}".format(oldchrom)
                searchRegions = [Locus("chr{}".format(l.chr()), l.start(), l.end(), l.strand()) for l in searchRegions]

            logging.warn("  Couldn't find reads on chromosome '{}'; trying instead '{}'".format(oldchrom, newchrom))

            reads, supplementaryAlignmentsFound = _getreads(searchRegions, bam, minmapq, pair_minmapq, single_ended, 
                include_supplementary, max_reads, sample_reads)

        except ValueError:
            raise e
    t1 = time.time()

    if supplementaryAlignmentsFound:
        logging.warn("  ** Supplementary alignments found: these alignments (with sam flag 0x800) **\n"
                     "  ** are poorly documented among mapping software and may result in missing **\n"
                     "  ** portions of reads; consider using the --include-supplementary          **\n"
                     "  ** command line argument if you think this is happening                   **")
        
    logging.debug("  time to find reads and mates:{:.1f}s".format(t1 - t0))
    logging.info("  number of reads found: {}".format(len(reads)))

    return reads 

def batchDisambiguate(alnCollections, isd, expectedOrientations, singleEnded=False, flankingRegionCollection=None,
    maxMultimappingSimilarity=0.9, alnScoreDeltaThreshold=2):
    t0 = time.time()

    for alnCollection in alnCollections:
        scoreAlignmentSetCollection(alnCollection, isd, 0, expectedOrientations, singleEnded=singleEnded,
            flankingRegionCollection=flankingRegionCollection, maxMultimappingSimilarity=maxMultimappingSimilarity)

    for alnCollection in alnCollections:
        disambiguate(alnCollection, singleEnded=singleEnded, alnScoreDeltaThreshold=alnScoreDeltaThreshold)

    t1 = time.time()
    logging.info(" Time for disambiguation: {:.2f}s".format(t1-t0)) 

def domatching(self):
        t0 = None

        for i, read in enumerate(self.tomatch):#[:150]):
            if i % 10000 == 0:
                if t0 is None:
                    t0 = time.time()
                    elapsed = "Finding mate pairs..."
                else:
                    t1 = time.time()
                    elapsed = "t={:.1f}s".format(t1-t0)
                    t0 = t1
                logging.info("   {:,} of {:,} {}".format(i, len(self.tomatch), elapsed))
            if len(self.readsByID[read.qname].reads) < 2:
                self.findmatch(read) 

def updateTimeRemaining(self, completed, total):
        t = time.time()
        elapsed = t - self.t0

        if elapsed == 0 or completed == 0:
            return "--"
        rate = completed/float(elapsed)
        remaining = total - completed
        t_remaining = remaining / rate

        #print "\n", total, completed, elapsed, rate, remaining
        
        #return t_remaining # in seconds

        self.timeRemaining = formatTime(t_remaining) 

def finish(self):
        text = [" "]
        if len(self.name) > 0:
            text.append(self.name)
        text.append("[completed] time elapsed: {}".format(formatTime(time.time()-self.t0)))
        text = " ".join(text)
        text = text.ljust(self.term_width)
        
        sys.stderr.write(text+"\n") 

def redraw(self):
        if self.isatty or (time.time()-self.lastRedraw) > 30:
            overallTotal = sum(x[1] for x in list(self.barsToProgress.values()))
            overallCompleted = sum(x[0] for x in list(self.barsToProgress.values()))
            
            numBars = len(self.barsToProgress)+1
            
            barWidth = (self.term_width-40-len(self.name)) // numBars - 1

            if self.status == "+":
                self.status = " "
            else:
                self.status = "+"
                
            text = [" ", self.status]
            if len(self.name) > 0:
                text.append(self.name)

            text.append(self._getBar("total", overallCompleted, overallTotal, 25))

            text.append("left:%s"%self.timeRemaining)

            if barWidth >= 6:
                for barid in sorted(self.barsToProgress):
                    text.append(self._getBar(barid, self.barsToProgress[barid][0], self.barsToProgress[barid][1], barWidth))
            else:
                text.append("[processes=%d]"%len(self.barsToProgress))
                
            endmarker = "\n"
            if self.isatty:
                endmarker = "\r"

            sys.stderr.write(" ".join(text)+endmarker)

            self.lastRedraw = time.time() 

def methodASDSDG(self, arg):
            print(arg)
            time.sleep(5) 

def _runTest(fn, description):
    print("\n\n -- running {} --\n\n".format(description))
    try:
        t0 = time.time()
        result = fn()
        t1 = time.time()
        result = [result[0], result[1], t1-t0]
    except Exception as e:
        print(" ** error running {}: {} **".format(description, e))
        print(traceback.print_exc())
        result = [False, str(e), -1]

    return result 

def log(info):
    """simple log"""
    print time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), info
    sys.stdout.flush() 

def study(self, course_id):
        """study one course"""
        time_step = 180
        log('start course:%s' % course_id)
        self.http.post(self.apis['enter_course'] % course_id, json_ret=False)
        course_show_api = self.apis['course_show'] % course_id
        log('url:%s' % course_show_api)
        self.http.post(course_show_api, json_ret=False)
        items_list = self.get_course_items(course_id)
        log('*' * 50)
        log('共有 %s 个子课程' % len(items_list))
        for index, i in enumerate(items_list):
            log('%s、%s ' % (index + 1, i['name']))
        log('*' * 50)
        log('begin to start...')
        for index, i in enumerate(items_list):
            sco_id = i['scoId']
            log('begin to study:%s-%s %s' % (index + 1, i['name'], sco_id))
            location = self.select_score_item(course_id, sco_id)
            cnt = 0
            while True:
                location = location + time_step * cnt
                cnt += 1
                log('location: %s' % location)
                self.do_heartbeat()
                self.update_timestep()
                ret = self.do_save(course_id, sco_id, location)
                if ret:
                    log('%s-%s %s' % (course_id, sco_id, 'done, start next'))
                    break
                log('*********** study %ss then go on *************' % time_step)
                time.sleep(time_step)
        info = '\033[92m\tDONE COURSE, url:%s\033[0m' % course_show_api
        log(info) 

def run(self):
        """入口"""
        s = time.time()
        course_list = []
        with open('study.list') as f:
            for course in f:
                course_list.append(course.strip())
        self.do_login()
        for course_id in course_list:
            try:
                self.study(course_id)
            except Exception as e:
                log("exception occured, study next..")
        cost = int(time.time() - s)
        log('main end, cost: %ss' % cost) 

def _ingest_summary(self):
        # get date parameters.
        yr = self._date[:4]
        mn = self._date[4:6]
        dy = self._date[6:]

        self._logger.info("Getting ingest summary data for the day")
        
        ingest_summary_cols = ["date","total"]		
        result_rows = []        
        df_filtered =  pd.DataFrame()

        query_to_load = ("""
            SELECT frame_time, COUNT(*) as total FROM {0}.{1}
            WHERE y={2} AND m={3} AND d={4} AND unix_tstamp IS NOT NULL
            AND frame_time IS NOT NULL AND frame_len IS NOT NULL
            AND dns_qry_name IS NOT NULL AND ip_src IS NOT NULL
            AND (dns_qry_class IS NOT NULL AND dns_qry_type IS NOT NULL
            AND dns_qry_rcode IS NOT NULL ) GROUP BY frame_time;
        """).format(self._db,self._table_name, yr, mn, dy)

        results = impala.execute_query_as_list(query_to_load)
        df = pd.DataFrame(results)

        # Forms a new dataframe splitting the minutes from the time column
        df_new = pd.DataFrame([["{0}-{1}-{2} {3}:{4}".format(yr, mn, dy,\
            val['frame_time'].replace("  "," ").split(" ")[3].split(":")[0].zfill(2),\
            val['frame_time'].replace("  "," ").split(" ")[3].split(":")[1].zfill(2)),\
            int(val['total']) if not math.isnan(val['total']) else 0 ] for key,val in df.iterrows()],columns = ingest_summary_cols)

        #Groups the data by minute
        sf = df_new.groupby(by=['date'])['total'].sum()
        df_per_min = pd.DataFrame({'date':sf.index, 'total':sf.values})

        df_final = df_filtered.append(df_per_min, ignore_index=True).to_records(False,False)

        if len(df_final) > 0:
            query_to_insert=("""
                INSERT INTO {0}.dns_ingest_summary PARTITION (y={1}, m={2}, d={3}) VALUES {4};
            """).format(self._db, yr, mn, dy, tuple(df_final))
            impala.execute_query(query_to_insert) 

def to_unixtime(time_string, format_string):
    with _STRPTIME_LOCK:
        return int(calendar.timegm(time.strptime(str(time_string), format_string)))