Python time.time() Examples

The following are 30 code examples for showing how to use time.time(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may want to check out the right sidebar which shows the related API usage.

You may also want to check out all available functions/classes of the module time , or try the search function .

Example 1
Project: svviz   Author: svviz   File: multiprocessor.py    License: MIT License 6 votes vote down vote up
def __init__(self, name=""):
        self.barsToProgress = {}

        self.t0 = time.time()
        self.timeRemaining = "--"
        self.status = "+"
        self.name = name
        self.lastRedraw = time.time()
        self.isatty = sys.stdout.isatty()

        try:
            self.handleResize(None,None)
            signal.signal(signal.SIGWINCH, self.handleResize)
            self.signal_set = True
        except:
            self.term_width = 79 
Example 2
Project: incubator-spot   Author: apache   File: flow_oa.py    License: Apache License 2.0 6 votes vote down vote up
def start(self):       
        
        ####################
        start = time.time()
        ####################         

        self._create_folder_structure()
        self._clear_previous_executions()        
        self._add_ipynb()  
        self._get_flow_results()
        self._add_network_context()
        self._add_geo_localization()
        self._add_reputation()        
        self._create_flow_scores()
        self._get_oa_details()
        self._ingest_summary()

        ##################
        end = time.time()
        print(end - start)
        ################## 
Example 3
Project: incubator-spot   Author: apache   File: proxy_oa.py    License: Apache License 2.0 6 votes vote down vote up
def start(self):

        ####################
        start = time.time()
        ####################

        self._create_folder_structure()
        self._clear_previous_executions()   
        self._add_ipynb()
        self._get_proxy_results()
        self._add_reputation() 
        self._add_iana()
        self._add_network_context() 
        self._create_proxy_scores_csv()
        self._get_oa_details()
        self._ingest_summary()


        ##################
        end = time.time()
        print(end - start)
        ################## 
Example 4
Project: incubator-spot   Author: apache   File: dns_oa.py    License: Apache License 2.0 6 votes vote down vote up
def start(self):

        ####################
        start = time.time()
        ####################

        self._clear_previous_executions()
        self._create_folder_structure()
        self._add_ipynb()
        self._get_dns_results()
        self._add_tld_column()
        self._add_reputation()
        self._add_hh_column()
        self._add_iana()
        self._add_network_context()
        self._create_dns_scores()
        self._get_oa_details()
        self._ingest_summary()

        ##################
        end = time.time()
        print(end - start)
        ################## 
Example 5
Project: vergeml   Author: mme   File: display.py    License: MIT License 6 votes vote down vote up
def _calc_it_per_sec(self):
        now = time.time()
        
        if not self.last_it:
            self.last_it.append(now)
        else:
            self.last_it.append(now)
            if len(self.last_it) >= 100:
                self.last_it = self.last_it[-100:]

            # smooth it/sec
            intervals = list(map(lambda t: t[1] - t[0], zip(self.last_it[:-1], self.last_it[1:])))
            delta = sum(intervals) / len(intervals)
            if delta != 0.:
                it_per_sec = 1.0 / delta
                if it_per_sec > 1_000_00:
                    self.it_per_sec = "{}M".format(int(it_per_sec/1_000_000))
                elif it_per_sec > 1_000:
                    self.it_per_sec = "{}K".format(int(it_per_sec/1_000))
                else:
                    self.it_per_sec = "{:.2f}".format(it_per_sec)
                # 6 characters so the progress bar is the same size
                self.it_per_sec = self.it_per_sec.rjust(6)
                # self.last_it = now 
Example 6
Project: pepper-robot-programming   Author: maverickjoy   File: asthama_search.py    License: MIT License 6 votes vote down vote up
def _startImageCapturingAtGraphNode(self):
        amntY = 0.3
        xdir = [-0.5, 0, 0.5]
        cameraId = 0

        for idx,amntX in enumerate(xdir):
            self._moveHead(amntX, amntY)
            time.sleep(0.1)
            self._checkIfAsthamaInEnvironment(cameraId)
            if self.pumpFound :
                # Setting rotation angle of body towards head
                theta = 0
                if idx == 0: # LEFT
                    theta = math.radians(-45)
                if idx == 2: # RIGHT
                    theta = math.radians(45)
                self.pumpAngleRotation = theta

                return "KILLING GRAPH SEARCH : PUMP FOUND"

        self._moveHead(0, 0) # Bringing to initial view

        return 
Example 7
Project: pepper-robot-programming   Author: maverickjoy   File: asthama_search.py    License: MIT License 6 votes vote down vote up
def _capture2dImage(self, cameraId):
        # Capture Image in RGB

        # WARNING : The same Name could be used only six time.
        strName = "capture2DImage_{}".format(random.randint(1,10000000000))

        clientRGB = self.video_service.subscribeCamera(strName, cameraId, AL_kVGA, 11, 10)
        imageRGB = self.video_service.getImageRemote(clientRGB)

        imageWidth   = imageRGB[0]
        imageHeight  = imageRGB[1]
        array        = imageRGB[6]
        image_string = str(bytearray(array))

        # Create a PIL Image from our pixel array.
        im = Image.frombytes("RGB", (imageWidth, imageHeight), image_string)

        # Save the image.
        image_name_2d = "images/img2d-" + str(self.imageNo2d) + ".png"
        im.save(image_name_2d, "PNG") # Stored in images folder in the pwd, if not present then create one
        self.imageNo2d += 1
        im.show()

        return 
Example 8
Project: pepper-robot-programming   Author: maverickjoy   File: asthama_search.py    License: MIT License 6 votes vote down vote up
def _capture3dImage(self):
        # Depth Image in RGB

        # WARNING : The same Name could be used only six time.
        strName = "capture3dImage_{}".format(random.randint(1,10000000000))


        clientRGB = self.video_service.subscribeCamera(strName, AL_kDepthCamera, AL_kQVGA, 11, 10)
        imageRGB = self.video_service.getImageRemote(clientRGB)

        imageWidth  = imageRGB[0]
        imageHeight = imageRGB[1]
        array       = imageRGB[6]
        image_string = str(bytearray(array))

        # Create a PIL Image from our pixel array.
        im = Image.frombytes("RGB", (imageWidth, imageHeight), image_string)
        # Save the image.
        image_name_3d = "images/img3d-" + str(self.imageNo3d) + ".png"
        im.save(image_name_3d, "PNG") # Stored in images folder in the pwd, if not present then create one
        self.imageNo3d += 1
        im.show()

        return 
Example 9
Project: pepper-robot-programming   Author: maverickjoy   File: robot_velocity_caliberation.py    License: MIT License 6 votes vote down vote up
def moveForward(motion_service):
    X = 1  # forward
    Y = 0.0
    Theta = 0.0

    print "Movement Starting"
    t0= time.time()

    # Blocking call
    possible = motion_service.moveTo(X, Y, Theta)

    t1 = time.time()

    timeDiff = t1 -t0
    timeDiff *= 1000

    if not possible:
        return -1

    # if not possible:
    #     print "Interruption case Time : ",timeDiff
    # else:
    #     print "Journey over Time : ",timeDiff

    return timeDiff 
Example 10
Project: aegea   Author: kislyuk   File: __init__.py    License: Apache License 2.0 6 votes vote down vote up
def wait_for_port(host, port, timeout=600, print_progress=True):
    if print_progress:
        sys.stderr.write("Waiting for {}:{}...".format(host, port))
        sys.stderr.flush()
    start_time = time.time()
    while True:
        try:
            socket.socket().connect((host, port))
            if print_progress:
                sys.stderr.write(GREEN("OK") + "\n")
            return
        except Exception:
            time.sleep(1)
            if print_progress:
                sys.stderr.write(".")
                sys.stderr.flush()
            if time.time() - start_time > timeout:
                raise 
Example 11
Project: aegea   Author: kislyuk   File: __init__.py    License: Apache License 2.0 6 votes vote down vote up
def __new__(cls, t, snap=0):
        if isinstance(t, (str, bytes)) and t.isdigit():
            t = int(t)
        if not isinstance(t, (str, bytes)):
            from dateutil.tz import tzutc
            return datetime.fromtimestamp(t // 1000, tz=tzutc())
        try:
            units = ["weeks", "days", "hours", "minutes", "seconds"]
            diffs = {u: float(t[:-1]) for u in units if u.startswith(t[-1])}
            if len(diffs) == 1:
                # Snap > 0 governs the rounding of units (hours, minutes and seconds) to 0 to improve cache performance
                snap_units = {u.rstrip("s"): 0 for u in units[units.index(list(diffs)[0]) + snap:]} if snap else {}
                snap_units.pop("day", None)
                snap_units.update(microsecond=0)
                ts = datetime.now().replace(**snap_units) + relativedelta(**diffs)
                cls._precision[ts] = snap_units
                return ts
            return dateutil_parse(t)
        except (ValueError, OverflowError, AssertionError):
            raise ValueError('Could not parse "{}" as a timestamp or time delta'.format(t)) 
Example 12
Project: natural-questions   Author: google-research-datasets   File: simplify_nq_data.py    License: Apache License 2.0 6 votes vote down vote up
def main(_):
  """Runs `text_utils.simplify_nq_example` over all shards of a split.

  Prints simplified examples to a single gzipped file in the same directory
  as the input shards.
  """
  split = os.path.basename(FLAGS.data_dir)
  outpath = os.path.join(FLAGS.data_dir,
                         "simplified-nq-{}.jsonl.gz".format(split))
  with gzip.open(outpath, "wb") as fout:
    num_processed = 0
    start = time.time()
    for inpath in glob.glob(os.path.join(FLAGS.data_dir, "nq-*-??.jsonl.gz")):
      print("Processing {}".format(inpath))
      with gzip.open(inpath, "rb") as fin:
        for l in fin:
          utf8_in = l.decode("utf8", "strict")
          utf8_out = json.dumps(
              text_utils.simplify_nq_example(json.loads(utf8_in))) + u"\n"
          fout.write(utf8_out.encode("utf8"))
          num_processed += 1
          if not num_processed % 100:
            print("Processed {} examples in {}.".format(num_processed,
                                                        time.time() - start)) 
Example 13
Project: RF-Monitor   Author: EarToEarOak   File: cli.py    License: GNU General Public License v2.0 6 votes vote down vote up
def __on_event(self):
        event = self._queue.get()
        if event.type == Events.SCAN_ERROR:
            self.__on_scan_error(event.data)
        elif event.type == Events.SCAN_DATA:
            self.__on_scan_data(event.data)
        elif event.type == Events.SERVER_ERROR:
            self.__on_server_error(event.data)
        elif event.type == Events.GPS_ERROR:
            self.__std_err(event.data['msg'])
            self.__restart_gps()
        elif event.type == Events.GPS_WARN:
            self.__std_err(event.data['msg'])
        elif event.type == Events.GPS_TIMEOUT:
            self.__std_err(event.data['msg'])
            self.__restart_gps()
        elif event.type == Events.GPS_LOC:
            self._location = event.data['loc']
        elif event.type == Events.PUSH_ERROR:
            if not self._warnedPush:
                self._warnedPush = True
                self.__std_err('Push failed:\n\t' + event.data['msg'])
        else:
            time.sleep(0.01) 
Example 14
Project: RF-Monitor   Author: EarToEarOak   File: gui.py    License: GNU General Public License v2.0 6 votes vote down vote up
def __on_rec(self, recording):
        timestamp = time.time()
        for monitor in self._monitors:
            if not recording:
                monitor.set_level(None, timestamp, None)
            monitor.set_recording(recording, timestamp)

        if recording:
            self.__on_start()
        else:
            while self._push.hasFailed():
                resp = wx.MessageBox('Web push has failed, retry?', 'Warning',
                                     wx.OK | wx.CANCEL | wx.ICON_WARNING)
                if resp == wx.OK:
                    busy = wx.BusyInfo('Pushing...', self)
                    self._push.send_failed(self._settings.get_push_uri())
                    del busy
                else:
                    self._push.clear_failed()

        self._warnedPush = False
        self.__set_timeline() 
Example 15
Project: unicorn-hat-hd   Author: pimoroni   File: demo.py    License: MIT License 5 votes vote down vote up
def current_milli_time():
    return int(round(time.time() * 1000)) 
Example 16
Project: Caffe-Python-Data-Layer   Author: liuxianming   File: DataManager.py    License: BSD 2-Clause "Simplified" License 5 votes vote down vote up
def load_all(self):
        """The function to load all data and labels

        Give:
        data: the list of raw data, needs to be decompressed
              (e.g., raw JPEG string)
        labels: numpy array, with each element is a string
        """
        start = time.time()
        print("Start Loading Data from BCF {}".format(
            'MEMORY' if self._bcf_mode == 'MEM' else 'FILE'))

        self._labels = np.loadtxt(self._label_fn).astype(str)

        if self._bcf.size() != self._labels.shape[0]:
            raise Exception("Number of samples in data"
                            "and labels are not equal")
        else:
            for idx in range(self._bcf.size()):
                datum_str = self._bcf.get(idx)
                self._data.append(datum_str)
        end = time.time()
        print("Loading {} samples Done: Time cost {} seconds".format(
            len(self._data), end - start))

        return self._data, self._labels 
Example 17
Project: svviz   Author: svviz   File: remap.py    License: MIT License 5 votes vote down vote up
def do_realign(dataHub, sample):
    processes = dataHub.args.processes
    if processes is None or processes == 0:
        # we don't really gain from using virtual cores, so try to figure out how many physical
        # cores we have
        processes = misc.cpu_count_physical()

    variant = dataHub.variant
    reads = sample.reads
    name = "{}:{{}}".format(sample.name[:15])

    t0 = time.time()
    refalignments, badReadsRef = do1remap(variant.chromParts("ref"), reads, processes, 
        jobName=name.format("ref"), tryExact=dataHub.args.fast)
    altalignments, badReadsAlt = do1remap(variant.chromParts("alt"), reads, processes, 
        jobName=name.format("alt"), tryExact=dataHub.args.fast)
    t1 = time.time()

    logging.debug(" Time to realign: {:.1f}s".format(t1-t0))

    badReads = badReadsRef.union(badReadsAlt)

    if len(badReads) > 0:
        logging.warn(" Alignment failed with {} reads (this is a known issue)".format(badReads))
        for badRead in badReads:
            refalignments.pop(badRead, None)
            altalignments.pop(badRead, None)

    assert set(refalignments.keys()) == set(altalignments.keys()), \
                    set(refalignments.keys()) ^ set(altalignments.keys())

    alnCollections = []
    for key in refalignments:
        alnCollection = AlignmentSetCollection(key)
        alnCollection.addSet(refalignments[key], "ref")
        alnCollection.addSet(altalignments[key], "alt")
        alnCollections.append(alnCollection)

    return alnCollections 
Example 18
Project: svviz   Author: svviz   File: remap.py    License: MIT License 5 votes vote down vote up
def getReads(variant, bam, minmapq, pair_minmapq, searchDistance, single_ended=False, include_supplementary=False, max_reads=None, sample_reads=None):
    t0 = time.time()
    searchRegions = variant.searchRegions(searchDistance)

    # This cludge tries the chromosomes as given ('chr4' or '4') and if that doesn't work
    # tries to switch to the other variation ('4' or 'chr4')
    try:
        reads, supplementaryAlignmentsFound = _getreads(searchRegions, bam, minmapq, pair_minmapq, single_ended, 
            include_supplementary, max_reads, sample_reads)
    except ValueError as e:
        oldchrom = searchRegions[0].chr()
        try:
            if "chr" in oldchrom:
                newchrom = oldchrom.replace("chr", "")
                searchRegions = [Locus(l.chr().replace("chr", ""), l.start(), l.end(), l.strand()) for l in searchRegions]
            else:
                newchrom = "chr{}".format(oldchrom)
                searchRegions = [Locus("chr{}".format(l.chr()), l.start(), l.end(), l.strand()) for l in searchRegions]

            logging.warn("  Couldn't find reads on chromosome '{}'; trying instead '{}'".format(oldchrom, newchrom))

            reads, supplementaryAlignmentsFound = _getreads(searchRegions, bam, minmapq, pair_minmapq, single_ended, 
                include_supplementary, max_reads, sample_reads)

        except ValueError:
            raise e
    t1 = time.time()

    if supplementaryAlignmentsFound:
        logging.warn("  ** Supplementary alignments found: these alignments (with sam flag 0x800) **\n"
                     "  ** are poorly documented among mapping software and may result in missing **\n"
                     "  ** portions of reads; consider using the --include-supplementary          **\n"
                     "  ** command line argument if you think this is happening                   **")
        
    logging.debug("  time to find reads and mates:{:.1f}s".format(t1 - t0))
    logging.info("  number of reads found: {}".format(len(reads)))

    return reads 
Example 19
Project: svviz   Author: svviz   File: disambiguate.py    License: MIT License 5 votes vote down vote up
def batchDisambiguate(alnCollections, isd, expectedOrientations, singleEnded=False, flankingRegionCollection=None,
    maxMultimappingSimilarity=0.9, alnScoreDeltaThreshold=2):
    t0 = time.time()

    for alnCollection in alnCollections:
        scoreAlignmentSetCollection(alnCollection, isd, 0, expectedOrientations, singleEnded=singleEnded,
            flankingRegionCollection=flankingRegionCollection, maxMultimappingSimilarity=maxMultimappingSimilarity)

    for alnCollection in alnCollections:
        disambiguate(alnCollection, singleEnded=singleEnded, alnScoreDeltaThreshold=alnScoreDeltaThreshold)

    t1 = time.time()
    logging.info(" Time for disambiguation: {:.2f}s".format(t1-t0)) 
Example 20
Project: svviz   Author: svviz   File: pairfinder.py    License: MIT License 5 votes vote down vote up
def domatching(self):
        t0 = None

        for i, read in enumerate(self.tomatch):#[:150]):
            if i % 10000 == 0:
                if t0 is None:
                    t0 = time.time()
                    elapsed = "Finding mate pairs..."
                else:
                    t1 = time.time()
                    elapsed = "t={:.1f}s".format(t1-t0)
                    t0 = t1
                logging.info("   {:,} of {:,} {}".format(i, len(self.tomatch), elapsed))
            if len(self.readsByID[read.qname].reads) < 2:
                self.findmatch(read) 
Example 21
Project: svviz   Author: svviz   File: multiprocessor.py    License: MIT License 5 votes vote down vote up
def updateTimeRemaining(self, completed, total):
        t = time.time()
        elapsed = t - self.t0

        if elapsed == 0 or completed == 0:
            return "--"
        rate = completed/float(elapsed)
        remaining = total - completed
        t_remaining = remaining / rate

        #print "\n", total, completed, elapsed, rate, remaining
        
        #return t_remaining # in seconds

        self.timeRemaining = formatTime(t_remaining) 
Example 22
Project: svviz   Author: svviz   File: multiprocessor.py    License: MIT License 5 votes vote down vote up
def finish(self):
        text = [" "]
        if len(self.name) > 0:
            text.append(self.name)
        text.append("[completed] time elapsed: {}".format(formatTime(time.time()-self.t0)))
        text = " ".join(text)
        text = text.ljust(self.term_width)
        
        sys.stderr.write(text+"\n") 
Example 23
Project: svviz   Author: svviz   File: multiprocessor.py    License: MIT License 5 votes vote down vote up
def redraw(self):
        if self.isatty or (time.time()-self.lastRedraw) > 30:
            overallTotal = sum(x[1] for x in list(self.barsToProgress.values()))
            overallCompleted = sum(x[0] for x in list(self.barsToProgress.values()))
            
            numBars = len(self.barsToProgress)+1
            
            barWidth = (self.term_width-40-len(self.name)) // numBars - 1

            if self.status == "+":
                self.status = " "
            else:
                self.status = "+"
                
            text = [" ", self.status]
            if len(self.name) > 0:
                text.append(self.name)

            text.append(self._getBar("total", overallCompleted, overallTotal, 25))

            text.append("left:%s"%self.timeRemaining)

            if barWidth >= 6:
                for barid in sorted(self.barsToProgress):
                    text.append(self._getBar(barid, self.barsToProgress[barid][0], self.barsToProgress[barid][1], barWidth))
            else:
                text.append("[processes=%d]"%len(self.barsToProgress))
                
            endmarker = "\n"
            if self.isatty:
                endmarker = "\r"

            sys.stderr.write(" ".join(text)+endmarker)

            self.lastRedraw = time.time() 
Example 24
Project: svviz   Author: svviz   File: multiprocessor.py    License: MIT License 5 votes vote down vote up
def methodASDSDG(self, arg):
            print(arg)
            time.sleep(5) 
Example 25
Project: svviz   Author: svviz   File: runTests.py    License: MIT License 5 votes vote down vote up
def _runTest(fn, description):
    print("\n\n -- running {} --\n\n".format(description))
    try:
        t0 = time.time()
        result = fn()
        t1 = time.time()
        result = [result[0], result[1], t1-t0]
    except Exception as e:
        print(" ** error running {}: {} **".format(description, e))
        print(traceback.print_exc())
        result = [False, str(e), -1]

    return result 
Example 26
Project: 21tb_robot   Author: iloghyr   File: study_robot.py    License: MIT License 5 votes vote down vote up
def log(info):
    """simple log"""
    print time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), info
    sys.stdout.flush() 
Example 27
Project: 21tb_robot   Author: iloghyr   File: study_robot.py    License: MIT License 5 votes vote down vote up
def study(self, course_id):
        """study one course"""
        time_step = 180
        log('start course:%s' % course_id)
        self.http.post(self.apis['enter_course'] % course_id, json_ret=False)
        course_show_api = self.apis['course_show'] % course_id
        log('url:%s' % course_show_api)
        self.http.post(course_show_api, json_ret=False)
        items_list = self.get_course_items(course_id)
        log('*' * 50)
        log('共有 %s 个子课程' % len(items_list))
        for index, i in enumerate(items_list):
            log('%s、%s ' % (index + 1, i['name']))
        log('*' * 50)
        log('begin to start...')
        for index, i in enumerate(items_list):
            sco_id = i['scoId']
            log('begin to study:%s-%s %s' % (index + 1, i['name'], sco_id))
            location = self.select_score_item(course_id, sco_id)
            cnt = 0
            while True:
                location = location + time_step * cnt
                cnt += 1
                log('location: %s' % location)
                self.do_heartbeat()
                self.update_timestep()
                ret = self.do_save(course_id, sco_id, location)
                if ret:
                    log('%s-%s %s' % (course_id, sco_id, 'done, start next'))
                    break
                log('*********** study %ss then go on *************' % time_step)
                time.sleep(time_step)
        info = '\033[92m\tDONE COURSE, url:%s\033[0m' % course_show_api
        log(info) 
Example 28
Project: 21tb_robot   Author: iloghyr   File: study_robot.py    License: MIT License 5 votes vote down vote up
def run(self):
        """入口"""
        s = time.time()
        course_list = []
        with open('study.list') as f:
            for course in f:
                course_list.append(course.strip())
        self.do_login()
        for course_id in course_list:
            try:
                self.study(course_id)
            except Exception as e:
                log("exception occured, study next..")
        cost = int(time.time() - s)
        log('main end, cost: %ss' % cost) 
Example 29
Project: incubator-spot   Author: apache   File: dns_oa.py    License: Apache License 2.0 5 votes vote down vote up
def _ingest_summary(self):
        # get date parameters.
        yr = self._date[:4]
        mn = self._date[4:6]
        dy = self._date[6:]

        self._logger.info("Getting ingest summary data for the day")
        
        ingest_summary_cols = ["date","total"]		
        result_rows = []        
        df_filtered =  pd.DataFrame()

        query_to_load = ("""
            SELECT frame_time, COUNT(*) as total FROM {0}.{1}
            WHERE y={2} AND m={3} AND d={4} AND unix_tstamp IS NOT NULL
            AND frame_time IS NOT NULL AND frame_len IS NOT NULL
            AND dns_qry_name IS NOT NULL AND ip_src IS NOT NULL
            AND (dns_qry_class IS NOT NULL AND dns_qry_type IS NOT NULL
            AND dns_qry_rcode IS NOT NULL ) GROUP BY frame_time;
        """).format(self._db,self._table_name, yr, mn, dy)

        results = impala.execute_query_as_list(query_to_load)
        df = pd.DataFrame(results)

        # Forms a new dataframe splitting the minutes from the time column
        df_new = pd.DataFrame([["{0}-{1}-{2} {3}:{4}".format(yr, mn, dy,\
            val['frame_time'].replace("  "," ").split(" ")[3].split(":")[0].zfill(2),\
            val['frame_time'].replace("  "," ").split(" ")[3].split(":")[1].zfill(2)),\
            int(val['total']) if not math.isnan(val['total']) else 0 ] for key,val in df.iterrows()],columns = ingest_summary_cols)

        #Groups the data by minute
        sf = df_new.groupby(by=['date'])['total'].sum()
        df_per_min = pd.DataFrame({'date':sf.index, 'total':sf.values})

        df_final = df_filtered.append(df_per_min, ignore_index=True).to_records(False,False)

        if len(df_final) > 0:
            query_to_insert=("""
                INSERT INTO {0}.dns_ingest_summary PARTITION (y={1}, m={2}, d={3}) VALUES {4};
            """).format(self._db, yr, mn, dy, tuple(df_final))
            impala.execute_query(query_to_insert) 
Example 30
Project: jumpserver-python-sdk   Author: jumpserver   File: utils.py    License: GNU General Public License v2.0 5 votes vote down vote up
def to_unixtime(time_string, format_string):
    with _STRPTIME_LOCK:
        return int(calendar.timegm(time.strptime(str(time_string), format_string)))