Python time.time() Examples

The following are code examples for showing how to use time.time(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: s2g   Author: caesar0301   File: test.py    MIT License 6 votes vote down vote up
def test_point_projects_to_edge(self):
        # p = (114.83299055, 26.8892277)
        p = (121.428387, 31.027371)
        a = time.time()
        edges, segments = self.sg.point_projects_to_edges(p, 0.01)
        print(time.time() - a)

        if self.show_plots:
            plt.figure()
            s2g.plot_lines(MultiLineString(segments), color='orange')  # original roads
            for i in range(0, len(edges)):
                s, e = edges[i]
                sxy = self.sg.node_xy[s]
                exy = self.sg.node_xy[e]
                plt.plot([sxy[0], exy[0]], [sxy[1], exy[1]], color='green')  # graph edges
            plt.plot(p[0], p[1], color='red', markersize=12, marker='o')  # bridges
            plt.show() 
Example 2
Project: fs_image   Author: facebookincubator   File: repo_server.py    MIT License 6 votes vote down vote up
def read_snapshot_dir(path: Path):
    db_path = path / 'snapshot.sql3'
    assert os.path.exists(db_path), f'no {db_path}, use rpm_repo_snapshot()'
    location_to_obj = add_snapshot_db_objs(sqlite3.connect(db_path))
    for repo in os.listdir(path / 'repos'):
        # Make JSON metadata for the repo's GPG keys.
        key_dir = path / 'repos' / repo / 'gpg_keys'
        for key_filename in os.listdir(key_dir.decode()):
            with open(key_dir / key_filename, 'rb') as infile:
                key_content = infile.read()
            location_to_obj[os.path.join(repo.decode(), key_filename)] = {
                'size': len(key_content),
                # We don't have a good timestamp for these, so set it to
                # "now".  Caching efficiency losses should be negligible :)
                'build_timestamp': int(time.time()),
                'content_bytes': key_content,  # Instead of `storage_id`
            }
    return location_to_obj 
Example 3
Project: SyNEThesia   Author: RunOrVeith   File: session_management.py    MIT License 6 votes vote down vote up
def utilize_session(self, model_name, data_provider, **kwargs):
        with SessionHandler(model=self.model, model_name=model_name) as session_handler:
            session_handler.load_weights_or_init()
            start_time = time.time()
            step = session_handler.step
            available = locals()
            available.pop("self", None)
            available.update(kwargs)
            print(f"{'Resuming' if step > 0 else 'Starting'} {model_name}: at step {step}")

            for input_feature in data_provider:
                available["input_feature"] = input_feature
                for hook in self.hooks:
                    provided = hook(self=self, **available)
                    available.update(provided)

                    yield hook.get_yieldables(self=self, **available) 
Example 4
Project: Home_Surveillance_with_Python   Author: kalfasyan   File: base_camera.py    MIT License 6 votes vote down vote up
def set(self):
        """Invoked by the camera thread when a new frame is available."""
        now = time.time()
        remove = None
        for ident, event in self.events.items():
            if not event[0].isSet():
                # if this client's event is not set, then set it
                # also update the last set timestamp to now
                event[0].set()
                event[1] = now
            else:
                # if the client's event is already set, it means the client
                # did not process a previous frame
                # if the event stays set for more than 5 seconds, then assume
                # the client is gone and remove it
                if now - event[1] > 5:
                    remove = ident
        if remove:
            del self.events[remove] 
Example 5
Project: Home_Surveillance_with_Python   Author: kalfasyan   File: base_camera.py    MIT License 6 votes vote down vote up
def _thread(cls):
        """Camera background thread."""
        print('Starting camera thread.')
        frames_iterator = cls.frames()
        for frame in frames_iterator:
            BaseCamera.frame = frame
            BaseCamera.event.set()  # send signal to clients
            time.sleep(0)

            # if there hasn't been any clients asking for frames in
            # the last 10 seconds then stop the thread
            if time.time() - BaseCamera.last_access > 10*60*500:
                frames_iterator.close()
                print('Stopping camera thread due to inactivity.')
                break
        BaseCamera.thread = None 
Example 6
Project: PEAKachu   Author: tbischler   File: window.py    ISC License 6 votes vote down vote up
def generate_window_counts(self):
        self._generate_windows()
        print("** Window read counting started for {} libraries...".format(len(
            self._lib_dict)), flush=True)
        t_start = time()
        for lib_name, lib in self._lib_dict.items():
            print(lib_name, flush=True)
            for replicon in self._replicon_dict:
                lib.replicon_dict[replicon][
                    "window_list"] = self._replicon_dict[replicon][
                        "window_list"]
            lib.count_reads_for_windows()
        t_end = time()
        print("Window read counting finished in {} seconds.\n".format(
            t_end-t_start), flush=True)
        print("** Generating data frames and filtering windows...", flush=True)
        t_start = time()
        self._convert_to_data_frame()
        t_end = time()
        print("Data frame generation and filtering finished in {} seconds.\n"
              .format(t_end-t_start), flush=True) 
Example 7
Project: PEAKachu   Author: tbischler   File: window.py    ISC License 6 votes vote down vote up
def _plot_and_write_windows_gfold(self):
        # plot windows
        print("* Plotting normalized windows...", flush=True)
        t_start = time()
        sig_window_df = self._window_df[self._window_df.significant]
        unsig_window_df = self._initial_window_df[
            ~self._initial_window_df.index.isin(sig_window_df.index)]
        self._plot_initial_windows(unsig_window_df.base_means,
                                   unsig_window_df.fold_change,
                                   sig_window_df.base_means,
                                   sig_window_df.fold_change)
        t_end = time()
        print("Plotting took {} seconds.".format(t_end-t_start), flush=True)
        # write windows after prefiltering with test results
        self._window_df.to_csv(
            "{}/windows_after_prefiltering.csv".format(self._output_folder),
            sep='\t', index=False, encoding='utf-8') 
Example 8
Project: PEAKachu   Author: tbischler   File: adaptive.py    ISC License 6 votes vote down vote up
def _filter_peaks_without_control(self, df):
        # calculate mad for original data frame
        median_abs_dev_from_zero = mad(df.loc[:, self._exp_lib_list].mean(
            axis=1), center=0.0)
        # minimum expression cutoff based on mean over experiment libraries
        print("Removing peaks based on mad cutoff from DataFrame "
              "with {} rows...".format(len(df)), flush=True)
        t_start = time()
        min_expr = (self._mad_multiplier * median_abs_dev_from_zero)
        print("Minimal peak expression based on mean over RIP/CLIP "
              "libraries:" "{} (MAD from zero: {})".format(
                  min_expr, median_abs_dev_from_zero), flush=True)
        df = df.loc[df.loc[:, self._exp_lib_list].mean(axis=1) >= min_expr, :]
        t_end = time()
        print("Removal took {} seconds. DataFrame contains now {} rows.".
              format((t_end-t_start), len(df)), flush=True)
        return df 
Example 9
Project: weibo-login   Author: littlepinecone   File: login.py    GNU General Public License v3.0 6 votes vote down vote up
def path_generate(a):
    pos = {'1': [32, 32],
           '2': [128, 32],
           '3': [32, 128],
           '4': [128, 128]}
    path = []
    t0 = (int(round(time.time() * 1000)))
    t00 = 0
    for j in range(0, 3):
        for i in range(0, 7):
            x = pos[a[j]][0] + i * (pos[a[j + 1]][0] - pos[a[j]][0]) / 6 + int(random.uniform(1, 3))

            y = pos[a[j]][1] + i * (pos[a[j + 1]][1] - pos[a[j]][1]) / 6 + int(random.uniform(2, 3))

            t = 30 * int(random.uniform(1, 2))
            t00 += t

            path0 = [x, y, t00]
            path.append(path0)
    path[0][2] = t0
    # print path
    return path 
Example 10
Project: incubator-spot   Author: apache   File: flow_oa.py    Apache License 2.0 6 votes vote down vote up
def start(self):       
        
        ####################
        start = time.time()
        ####################         

        self._create_folder_structure()
        self._clear_previous_executions()        
        self._add_ipynb()  
        self._get_flow_results()
        self._add_network_context()
        self._add_geo_localization()
        self._add_reputation()        
        self._create_flow_scores()
        self._get_oa_details()
        self._ingest_summary()

        ##################
        end = time.time()
        print(end - start)
        ################## 
Example 11
Project: incubator-spot   Author: apache   File: proxy_oa.py    Apache License 2.0 6 votes vote down vote up
def start(self):

        ####################
        start = time.time()
        ####################

        self._create_folder_structure()
        self._clear_previous_executions()   
        self._add_ipynb()
        self._get_proxy_results()
        self._add_reputation() 
        self._add_iana()
        self._add_network_context() 
        self._create_proxy_scores_csv()
        self._get_oa_details()
        self._ingest_summary()


        ##################
        end = time.time()
        print(end - start)
        ################## 
Example 12
Project: incubator-spot   Author: apache   File: dns_oa.py    Apache License 2.0 6 votes vote down vote up
def start(self):

        ####################
        start = time.time()
        ####################

        self._clear_previous_executions()
        self._create_folder_structure()
        self._add_ipynb()
        self._get_dns_results()
        self._add_tld_column()
        self._add_reputation()
        self._add_hh_column()
        self._add_iana()
        self._add_network_context()
        self._create_dns_scores()
        self._get_oa_details()
        self._ingest_summary()

        ##################
        end = time.time()
        print(end - start)
        ################## 
Example 13
Project: kicker-module   Author: EvanTheB   File: backend.py    GNU General Public License v3.0 6 votes vote down vote up
def test_concurrent():
    # run this in multiple processes,
    # the idea is that collisions will crash, but not munge data
    import random
    import time
    if not os.path.exists("tmp_test_con.log"):
        init_data_file("tmp_test_con.log")

    k = LadderData("tmp_test_con.log")
    # test concurrent log writes
    thread = str(random.randint(0, 100))
    print thread
    for x in range(100):
        time.sleep(0.01)
        k.add_player(thread + '_' + str(x))

    p, g = k.get_players_games()
    print len(p) 
Example 14
Project: webnull   Author: macrael   File: webnull.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def pretty_time(time, now=datetime.datetime.now()):
    tomorrow = now + datetime.timedelta(days=1)
    next_day = now + datetime.timedelta(days=2)
    next_week = now + datetime.timedelta(weeks=1)
    pretty_fmt = '%-I:%M %p'
    pretty_prefix = ''
    if tomorrow < time < next_day:
        pretty_prefix = 'tomorrow at '
    elif time > next_day and time < next_week:
        pretty_prefix = '%A at '
    elif time > next_week and time.month == now.month and time.year == now.year:
        pretty_prefix = '%A the %-d' + pretty_suffix(time.day) + ' at '
    elif time > next_week and time.year == now.year:
        pretty_prefix = '%B %-d' + pretty_suffix(time.day) + ' at '
    elif time > next_week:
        pretty_prefix = '%B %-d' + pretty_suffix(time.day) + ' %Y at '

    return time.strftime(pretty_prefix + pretty_fmt) 
Example 15
Project: webnull   Author: macrael   File: webnull.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def arg_parser():
    parser = argparse.ArgumentParser(description='A tool for putting websites into a black hole.')
    commands = parser.add_subparsers(title='commands', metavar='COMMAND')
    commands.required = True
    commands.dest = "commands"

    deny = commands.add_parser('deny', description='Add a site to the black hole. It will become unreachable.', help='Add a site to the black hole. It will become unreachable.')
    deny.add_argument('sitename', help='The website to be blackholed. A URL will be stripped down correctly.')
    deny.set_defaults(func=deny_site)

    allow = commands.add_parser('allow', description='Allow access to a blackholed site for a spell.', help='Allow access to a blackholed site for a spell.')
    time_or_tomorrow = allow.add_mutually_exclusive_group()
    time_or_tomorrow.add_argument('-t', '--time', help='sets the duration to enable a site for. Default is five minutes.', default=5, type=int)
    time_or_tomorrow.add_argument('-m', '--morning', help='allow all sites until tomorrow morning at 6am', action='store_true')
    all_or_one = allow.add_mutually_exclusive_group(required=True)
    all_or_one.add_argument('-a', '--all', action='store_true', help='All blackholed hostnames will be granted access instead of a matching sitename.')
    all_or_one.add_argument('sitename', help='All blackholed hostnames that contain this string will be temporarlly granted access.', nargs='?')
    allow.set_defaults(func=allow_site)

    return parser 
Example 16
Project: unicorn-hat-hd   Author: pimoroni   File: demo.py    MIT License 5 votes vote down vote up
def current_milli_time():
    return int(round(time.time() * 1000)) 
Example 17
Project: UrsaRobotics_SmartHome   Author: stav98   File: speech_recogn.py    GNU General Public License v3.0 5 votes vote down vote up
def wait_for_message(d):
    client.loop_start()
    time.sleep(d)
    client.loop_stop()

#Για πάντα 
Example 18
Project: s2g   Author: caesar0301   File: test.py    MIT License 5 votes vote down vote up
def test_subgraph_within_box(self):
        bounding_box = box(121.428387, 31.027371, 121.430863, 31.030227)
        a = time.time()
        subgraph = self.sg.subgraph_within_box(bounding_box)
        print(time.time() - a)
        if self.show_plots:
            plt.figure()
            nx.draw(subgraph, pos=self.sg.node_xy, node_size=50)
            plt.show() 
Example 19
Project: fs_image   Author: facebookincubator   File: demo_sendstreams.py    MIT License 5 votes vote down vote up
def _populate_sendstream_dict(d):
    d['build_start_time'] = _float_to_sec_nsec_tuple(time.time())
    yield d
    d['dump'] = subprocess.run(
        ['btrfs', 'receive', '--dump'],
        input=d['sendstream'], stdout=subprocess.PIPE, check=True,
        # split into lines to make the `pretty` output prettier
    ).stdout.rstrip(b'\n').split(b'\n')
    d['build_end_time'] = _float_to_sec_nsec_tuple(time.time())


# Takes `path_in_repo` because this is part of the library interface, and
# thus must work in @mode/opt, and thus we cannot use `__file__` here. 
Example 20
Project: fs_image   Author: facebookincubator   File: repo_objects.py    MIT License 5 votes vote down vote up
def new(cls, *, xml: bytes):  # NamedTuple.__new__ cannot be overridden
        repodatas = frozenset(_parse_repomd(xml))
        return cls.__new__(
            cls,
            xml=xml,
            fetch_timestamp=int(time.time()),
            build_timestamp=max(r.build_timestamp for r in repodatas),
            repodatas=repodatas,
            checksum=Checksum(
                algorithm=CANONICAL_HASH,
                hexdigest=hashlib.new(CANONICAL_HASH, xml).hexdigest(),
            ),
            size=len(xml),
        ) 
Example 21
Project: fs_image   Author: facebookincubator   File: test_common.py    MIT License 5 votes vote down vote up
def test_retry_fn(self):

        class Retriable:
            def __init__(self, attempts_to_fail=0):
                self.attempts = 0
                self.first_success_attempt = attempts_to_fail + 1

            def run(self):
                self.attempts += 1
                if self.attempts >= self.first_success_attempt:
                    return self.attempts
                raise RuntimeError(self.attempts)

        self.assertEqual(1, retry_fn(
            Retriable().run, delays=[], what='succeeds immediately'
        ))

        # Check log messages, and ensure that delays add up as expected
        start_time = time.time()
        with self.assertLogs(common_log) as log_ctx:
            self.assertEqual(4, retry_fn(
                Retriable(3).run, delays=[0, 0.1, 0.2], what='succeeds on try 4'
            ))
        self.assertTrue(any(
            '\n[Retry 3 of 3] succeeds on try 4 -- waiting 0.2 seconds.\n' in o
                for o in log_ctx.output
        ))
        self.assertGreater(time.time() - start_time, 0.3)

        # Check running out of retries
        with self.assertLogs(common_log) as log_ctx, \
                self.assertRaises(RuntimeError) as ex_ctx:
            retry_fn(Retriable(100).run, delays=[0] * 7, what='never succeeds')
        self.assertTrue(any(
            '\n[Retry 7 of 7] never succeeds -- waiting 0 seconds.\n' in o
                for o in log_ctx.output
        ))
        self.assertEqual((8,), ex_ctx.exception.args) 
Example 22
Project: SyNEThesia   Author: RunOrVeith   File: session_types.py    MIT License 5 votes vote down vote up
def time_diff(start_time):
    m, s = divmod(time.time() - start_time, 60)
    h, m = divmod(m, 60)
    d, h = divmod(h, 24)
    return "%d:%02d:%02d:%02d" % (d, h, m, s) 
Example 23
Project: SyNEThesia   Author: RunOrVeith   File: session_types.py    MIT License 5 votes vote down vote up
def _maybe_save(self, session_handler, step, start_time, save_every_n_steps):
        if step % save_every_n_steps == 0 and step > 0:
            session_handler.save(step=step)
            print(f"Step {step}, time: {time_diff(start_time)}: Saving in {session_handler.checkpoint_dir}") 
Example 24
Project: Caffe-Python-Data-Layer   Author: liuxianming   File: DataManager.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def load_all(self):
        """The function to load all data and labels

        Give:
        data: the list of raw data, needs to be decompressed
              (e.g., raw JPEG string)
        labels: numpy array, with each element is a string
        """
        start = time.time()
        print("Start Loading Data from BCF {}".format(
            'MEMORY' if self._bcf_mode == 'MEM' else 'FILE'))

        self._labels = np.loadtxt(self._label_fn).astype(str)

        if self._bcf.size() != self._labels.shape[0]:
            raise Exception("Number of samples in data"
                            "and labels are not equal")
        else:
            for idx in range(self._bcf.size()):
                datum_str = self._bcf.get(idx)
                self._data.append(datum_str)
        end = time.time()
        print("Loading {} samples Done: Time cost {} seconds".format(
            len(self._data), end - start))

        return self._data, self._labels 
Example 25
Project: Home_Surveillance_with_Python   Author: kalfasyan   File: base_camera.py    MIT License 5 votes vote down vote up
def wait(self):
        """Invoked from each client's thread to wait for the next frame."""
        ident = get_ident()
        if ident not in self.events:
            # this is a new client
            # add an entry for it in the self.events dict
            # each entry has two elements, a threading.Event() and a timestamp
            self.events[ident] = [threading.Event(), time.time()]
        return self.events[ident][0].wait() 
Example 26
Project: Home_Surveillance_with_Python   Author: kalfasyan   File: base_camera.py    MIT License 5 votes vote down vote up
def __init__(self):
        """Start the background camera thread if it isn't running yet."""
        if BaseCamera.thread is None:
            BaseCamera.last_access = time.time()

            # start background frame thread
            BaseCamera.thread = threading.Thread(target=self._thread)
            BaseCamera.thread.start()

            # wait until frames are available
            while self.get_frame() is None:
                time.sleep(0) 
Example 27
Project: Home_Surveillance_with_Python   Author: kalfasyan   File: base_camera.py    MIT License 5 votes vote down vote up
def get_frame(self):
        """Return the current camera frame."""
        BaseCamera.last_access = time.time()

        # wait for a signal from the camera thread
        BaseCamera.event.wait()
        BaseCamera.event.clear()

        return BaseCamera.frame 
Example 28
Project: PEAKachu   Author: tbischler   File: window.py    ISC License 5 votes vote down vote up
def _convert_to_data_frame(self):
        self._window_df = pd.DataFrame()
        for replicon in sorted(self._replicon_dict):
            for strand in ["+", "-"]:
                # add window positions to data frame
                row_number = len(self._replicon_dict[replicon]["window_list"])
                df = pd.concat([
                    pd.Series([replicon] * row_number),
                    pd.Series([strand] * row_number),
                    pd.Series([window[0]+1 for window in
                               self._replicon_dict[
                                   replicon]["window_list"]]),
                    pd.Series([window[1] for window in
                               self._replicon_dict[
                        replicon]["window_list"]])], axis=1)
                df.columns = ["replicon", "strand", "w_start", "w_end"]
                # add library counts to data frame
                for lib_name, lib in self._lib_dict.items():
                    df[lib_name] = (pd.Series(lib.replicon_dict[
                        replicon]["window_counts"].loc[:, strand]))
                self._window_df = self._window_df.append(df,
                                                         ignore_index=True)
            del self._replicon_dict[replicon]["window_list"]
        # remove windows without expression in any library
        print("Removing empty windows from DataFrame with {} rows...".format(
            len(self._window_df.index)), flush=True)
        t_start = time()
        self._window_df = self._window_df.loc[
            (self._window_df.loc[:, self._lib_names_list].sum(axis=1) > 0), :]
        t_end = time()
        print("Removal took {} seconds. DataFrame contains now {} rows.".
              format((t_end-t_start), len(self._window_df.index)), flush=True)
        if self._window_df.empty:
            print("**Dataframe empty**", flush=True)
            return
        if self._stat_test == "gtest":
            self._run_gtest_preprocessing()
        elif self._stat_test == "deseq":
            self._run_deseq_preprocessing() 
Example 29
Project: PEAKachu   Author: tbischler   File: window.py    ISC License 5 votes vote down vote up
def _run_gtest_preprocessing(self):
        # define size factors
        self._define_size_factors()
        # add pseudocounts
        self._window_df[self._lib_names_list] += 1.0
        # normalize counts
        self._window_df[self._lib_names_list] = self._window_df[
            self._lib_names_list].div(
                self._size_factors, axis='columns')
        t_end = time()
        # calculate base means for all windows
        print("Calculating base means and fold changes...", flush=True)
        t_start = time()
        self._window_df["base_means"] = self._window_df.loc[
            :, self._lib_names_list].mean(axis=1)
        # calculate fcs for all windows
        self._window_df["fold_change"] = (
            self._window_df.loc[:, self._exp_lib_list].sum(axis=1) /
            self._window_df.loc[:, self._ctr_lib_list].sum(axis=1))
        t_end = time()
        print("Calculation took {} seconds.".format(t_end-t_start), flush=True)
        # write raw windows to file
        print("Writing normalized windows to file...", flush=True)
        t_start = time()
        self._window_df.to_csv("{}/raw_windows.csv".format(
            self._output_folder), sep='\t', index=False, encoding='utf-8')
        t_end = time()
        print("Writing took {} seconds.".format(t_end-t_start), flush=True)
        # filter windows
        print("* Filtering windows...", flush=True)
        self._initial_window_df = self._window_df.copy()
        self._window_df = self._prefilter_windows_gtest(self._window_df) 
Example 30
Project: PEAKachu   Author: tbischler   File: window.py    ISC License 5 votes vote down vote up
def _prefilter_windows_deseq(self, df):
        print("Removing windows where not all experiment libs show "
              "expression from DataFrame with {} rows...".format(len(df)),
              flush=True)
        t_start = time()
        for exp_lib in self._exp_lib_list:
            exp_lib_zero_count = 0.0
            df = df.loc[(df.loc[:, exp_lib] > exp_lib_zero_count), :]
        t_end = time()
        print("Removal took {} seconds. DataFrame contains now {} rows.".
              format((t_end-t_start), len(df)), flush=True)
        if df.empty:
            return df
        initial_window_df = df.copy()
        # normalize counts on initial windows
        initial_window_df[self._lib_names_list] = initial_window_df[
            self._lib_names_list].div(self._size_factors, axis='columns')
        # minimum expression cutoff based on mean over experiment libraries
        print("Removing windows based on mad cutoff from DataFrame "
              "with {} rows...".format(len(df)), flush=True)
        t_start = time()
        median_abs_dev_from_zero = mad(initial_window_df.loc[
            :, self._exp_lib_list].mean(axis=1), center=0.0)
        min_expr = (self._mad_multiplier * median_abs_dev_from_zero)
        print("Minimal window expression based on mean over RIP/CLIP "
              "libraries: {} (MAD from zero: {})".format(
                  min_expr, median_abs_dev_from_zero), flush=True)
        df = df.loc[initial_window_df.loc[:, self._exp_lib_list].mean(
            axis=1) >= min_expr, :]
        t_end = time()
        print("Removal took {} seconds. DataFrame contains now {} rows.".
              format((t_end-t_start), len(df)), flush=True)
        return df 
Example 31
Project: PEAKachu   Author: tbischler   File: window.py    ISC License 5 votes vote down vote up
def _generate_peak_counts(self):
        print("* Peak read counting started for {} libraries...".format(len(
            self._lib_dict)), flush=True)
        t_start = time()
        for lib_name, lib in self._lib_dict.items():
            print(lib_name, flush=True)
            for replicon in self._replicon_dict:
                lib.replicon_dict[replicon]["peak_df"] = self._replicon_dict[
                    replicon]["peak_df"]
            lib.count_reads_for_peaks()
        t_end = time()
        print("Peak read counting finished in {} seconds.".format
              (t_end-t_start), flush=True) 
Example 32
Project: PEAKachu   Author: tbischler   File: adaptive.py    ISC License 5 votes vote down vote up
def _generate_peak_counts(self):
        print("** Peak read counting started for {} libraries...".format(
            len(self._lib_dict)), flush=True)
        t_start = time()
        for lib_name, lib in self._lib_dict.items():
            print(lib_name, flush=True)
            for replicon in self._replicon_dict:
                lib.replicon_dict[replicon]["peak_df"] = self._replicon_dict[
                    replicon]["peak_df"]
            lib.count_reads_for_peaks()
        t_end = time()
        print("Peak read counting finished in {} seconds.".format(
            t_end-t_start), flush=True) 
Example 33
Project: PEAKachu   Author: tbischler   File: adaptive.py    ISC License 5 votes vote down vote up
def _filter_peaks_without_replicates(self, df):
        # calculate mad for original data frame
        median_abs_dev_from_zero = mad(df.loc[:, self._exp_lib_list].mean(
            axis=1), center=0.0)
        # minimum expression cutoff based on mean over experiment libraries
        print("Removing peaks based on mad cutoff from DataFrame "
              "with {} rows...".format(len(df)), flush=True)
        t_start = time()
        min_expr = (self._mad_multiplier * median_abs_dev_from_zero)
        print("Minimal peak expression based on mean over RIP/CLIP "
              "libraries:" "{} (MAD from zero: {})".format(
                  min_expr, median_abs_dev_from_zero), flush=True)
        df = df.loc[df.loc[:, self._exp_lib_list].mean(axis=1) >= min_expr, :]
        t_end = time()
        print("Removal took {} seconds. DataFrame contains now {} rows.".
              format((t_end-t_start), len(df)), flush=True)
        if df.empty:
            return df
        # minimum fold change
        print("Removing windows based on minimum fold change from DataFrame "
              "with {} rows...".format(len(df)), flush=True)
        t_start = time()
        df = df.query('fold_change >= @self._fc_cutoff')
        t_end = time()
        print("Removal took {} seconds. DataFrame contains now {} rows.".
              format((t_end-t_start), len(df)), flush=True)
        return df 
Example 34
Project: PEAKachu   Author: tbischler   File: coverage.py    ISC License 5 votes vote down vote up
def generate_normalized_wiggle_files(project_folder, max_proc):
    parameter_dict = _read_parameters(project_folder)
    # create normalized coverage folder if it does not exist
    wiggle_folder = "{}/normalized_coverage".format(project_folder)
    if not exists(wiggle_folder):
        makedirs(wiggle_folder)
    # Generate coverage files in parallel
    print("** Generating normalized coverage files for {} libraries...".format(
          len(parameter_dict["libraries"])), flush=True)
    t_start = time()
    with futures.ProcessPoolExecutor(
            max_workers=max_proc) as executor:
        future_to_lib_name = {
            executor.submit(
                _generate_normalized_wiggle_file_for_lib, lib_name,
                lib["bam_file"], parameter_dict["paired_end"],
                parameter_dict["max_insert_size"], lib["size_factor"],
                wiggle_folder): lib_name for lib_name, lib
            in parameter_dict["libraries"].items()}
    for future in futures.as_completed(future_to_lib_name):
        lib_name = future_to_lib_name[future]
        print("* Coverage files for library {} generated.".format(lib_name),
              flush=True)
    t_end = time()
    print("Coverage file generation finished in {} seconds.".format(
        t_end-t_start), flush=True) 
Example 35
Project: PEAKachu   Author: tbischler   File: controller.py    ISC License 5 votes vote down vote up
def _init_replicons(self):
        print("** Initializing replicons and reading annotations from .gff "
              "files if present...", flush=True)
        t_start = time()
        replicons = Replicons(self._args.ctr_libs, self._args.exp_libs,
                              self._args.gff_folder, self._args.features,
                              self._args.sub_features)
        replicons.init_replicons()
        t_end = time()
        print("Finished replicon initialization in {} seconds.\n".format(
            t_end-t_start), flush=True)
        return replicons 
Example 36
Project: PEAKachu   Author: tbischler   File: controller.py    ISC License 5 votes vote down vote up
def _calc_sig_peaks_adaptive(self, adaptive, size_factors):
        # Run DESeq2 only if >1 libraries are available for both, experiment
        # and control libraries
        if len(self._args.exp_libs) > 1 and len(self._args.ctr_libs) > 1:
            print("** Calculating peak significance with DESeq2...",
                  flush=True)
            t_start = time()
            adaptive.run_deseq2_analysis(size_factors,
                                         self._args.pairwise_replicates)
            t_end = time()
            print("DESeq2 finished in {} seconds.\n".format(t_end-t_start),
                  flush=True)
        # If at least one control is available use fold change and MAD to
        # define significant peaks
        elif self._args.ctr_libs:
            print("** Calculating peak significance for insufficient "
                  "replicates based on fold change and MAD cutoff...",
                  flush=True)
            t_start = time()
            adaptive.run_analysis_without_replicates(size_factors)
            t_end = time()
            print("Peak calculation finished in {} seconds.\n".format(
                t_end-t_start), flush=True)
        # If no controls are available return initial peaks based on MAD cutoff
        else:
            print("** Calculating peaks without control based on MAD "
                  "cutoff...", flush=True)
            t_start = time()
            adaptive.run_analysis_without_control(size_factors)
            t_end = time()
            print("Peak calculation finished in {} seconds.\n".format(
                t_end-t_start), flush=True) 
Example 37
Project: 21tb_robot   Author: iloghyr   File: study_robot.py    MIT License 5 votes vote down vote up
def log(info):
    """simple log"""
    print time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()), info
    sys.stdout.flush() 
Example 38
Project: 21tb_robot   Author: iloghyr   File: study_robot.py    MIT License 5 votes vote down vote up
def study(self, course_id):
        """study one course"""
        time_step = 180
        log('start course:%s' % course_id)
        self.http.post(self.apis['enter_course'] % course_id, json_ret=False)
        course_show_api = self.apis['course_show'] % course_id
        log('url:%s' % course_show_api)
        self.http.post(course_show_api, json_ret=False)
        items_list = self.get_course_items(course_id)
        log('*' * 50)
        log('共有 %s 个子课程' % len(items_list))
        for index, i in enumerate(items_list):
            log('%s、%s ' % (index + 1, i['name']))
        log('*' * 50)
        log('begin to start...')
        for index, i in enumerate(items_list):
            sco_id = i['scoId']
            log('begin to study:%s-%s %s' % (index + 1, i['name'], sco_id))
            location = self.select_score_item(course_id, sco_id)
            cnt = 0
            while True:
                location = location + time_step * cnt
                cnt += 1
                log('location: %s' % location)
                self.do_heartbeat()
                self.update_timestep()
                ret = self.do_save(course_id, sco_id, location)
                if ret:
                    log('%s-%s %s' % (course_id, sco_id, 'done, start next'))
                    break
                log('*********** study %ss then go on *************' % time_step)
                time.sleep(time_step)
        info = '\033[92m\tDONE COURSE, url:%s\033[0m' % course_show_api
        log(info) 
Example 39
Project: 21tb_robot   Author: iloghyr   File: study_robot.py    MIT License 5 votes vote down vote up
def run(self):
        """入口"""
        s = time.time()
        course_list = []
        with open('study.list') as f:
            for course in f:
                course_list.append(course.strip())
        self.do_login()
        for course_id in course_list:
            try:
                self.study(course_id)
            except Exception as e:
                log("exception occured, study next..")
        cost = int(time.time() - s)
        log('main end, cost: %ss' % cost) 
Example 40
Project: incubator-spot   Author: apache   File: dns_oa.py    Apache License 2.0 5 votes vote down vote up
def _ingest_summary(self):
        # get date parameters.
        yr = self._date[:4]
        mn = self._date[4:6]
        dy = self._date[6:]

        self._logger.info("Getting ingest summary data for the day")
        
        ingest_summary_cols = ["date","total"]		
        result_rows = []        
        df_filtered =  pd.DataFrame()

        query_to_load = ("""
            SELECT frame_time, COUNT(*) as total FROM {0}.{1}
            WHERE y={2} AND m={3} AND d={4} AND unix_tstamp IS NOT NULL
            AND frame_time IS NOT NULL AND frame_len IS NOT NULL
            AND dns_qry_name IS NOT NULL AND ip_src IS NOT NULL
            AND (dns_qry_class IS NOT NULL AND dns_qry_type IS NOT NULL
            AND dns_qry_rcode IS NOT NULL ) GROUP BY frame_time;
        """).format(self._db,self._table_name, yr, mn, dy)

        results = impala.execute_query_as_list(query_to_load)
        df = pd.DataFrame(results)

        # Forms a new dataframe splitting the minutes from the time column
        df_new = pd.DataFrame([["{0}-{1}-{2} {3}:{4}".format(yr, mn, dy,\
            val['frame_time'].replace("  "," ").split(" ")[3].split(":")[0].zfill(2),\
            val['frame_time'].replace("  "," ").split(" ")[3].split(":")[1].zfill(2)),\
            int(val['total']) if not math.isnan(val['total']) else 0 ] for key,val in df.iterrows()],columns = ingest_summary_cols)

        #Groups the data by minute
        sf = df_new.groupby(by=['date'])['total'].sum()
        df_per_min = pd.DataFrame({'date':sf.index, 'total':sf.values})

        df_final = df_filtered.append(df_per_min, ignore_index=True).to_records(False,False)

        if len(df_final) > 0:
            query_to_insert=("""
                INSERT INTO {0}.dns_ingest_summary PARTITION (y={1}, m={2}, d={3}) VALUES {4};
            """).format(self._db, yr, mn, dy, tuple(df_final))
            impala.execute_query(query_to_insert) 
Example 41
Project: jumpserver-python-sdk   Author: jumpserver   File: utils.py    GNU General Public License v2.0 5 votes vote down vote up
def to_unixtime(time_string, format_string):
    with _STRPTIME_LOCK:
        return int(calendar.timegm(time.strptime(str(time_string), format_string))) 
Example 42
Project: jumpserver-python-sdk   Author: jumpserver   File: utils.py    GNU General Public License v2.0 5 votes vote down vote up
def make_signature(access_key_secret, date=None):
    if isinstance(date, bytes):
        date = bytes.decode(date)
    if isinstance(date, int):
        date_gmt = http_date(date)
    elif date is None:
        date_gmt = http_date(int(time.time()))
    else:
        date_gmt = date

    data = str(access_key_secret) + "\n" + date_gmt
    return content_md5(data) 
Example 43
Project: telegram-innovation-chatbot   Author: zaoldyeck   File: olami.py    MIT License 5 votes vote down vote up
def _gen_parameters(self, api, text, cusid):
        timestamp_ms = (int(time.time() * 1000))
        params = {'appkey': self.app_key,
                  'api': api,
                  'timestamp': timestamp_ms,
                  'sign': self._gen_sign(api, timestamp_ms),
                  'rq': self._gen_rq(text)}
        if cusid is not None:
            params.update(cusid=cusid)
        return params 
Example 44
Project: kicker-module   Author: EvanTheB   File: backend.py    GNU General Public License v3.0 5 votes vote down vote up
def add_player(self, name):
        with open(self.log_file, 'r+') as log:
            with LockFile(log):
                self._load_from_json(log)
                event = AddPlayerEvent(name, time.time())
                ret = event.process(self.players, self.games)
                self.events.append(event)
                log.seek(0)
                self._save_to_log(log)

        return ret 
Example 45
Project: kicker-module   Author: EvanTheB   File: backend.py    GNU General Public License v3.0 5 votes vote down vote up
def add_game(self, command_words):
        with open(self.log_file, 'r+') as log:
            with LockFile(log):
                self._load_from_json(log)
                event = AddGameEvent(command_words, time.time())
                ret = event.process(self.players, self.games)
                self.events.append(event)
                log.seek(0)
                self._save_to_log(log)

        return ret 
Example 46
Project: kicker-module   Author: EvanTheB   File: backend.py    GNU General Public License v3.0 5 votes vote down vote up
def to_json(self):
        ret = {}
        ret['type'] = 'AddPlayerEvent'
        ret['player'] = self.name
        ret['time'] = self.create_time
        return ret 
Example 47
Project: kicker-module   Author: EvanTheB   File: backend.py    GNU General Public License v3.0 5 votes vote down vote up
def from_json(the_json):
        assert the_json['type'] == 'AddPlayerEvent'
        return AddPlayerEvent(the_json['player'], the_json['time']) 
Example 48
Project: kicker-module   Author: EvanTheB   File: backend.py    GNU General Public License v3.0 5 votes vote down vote up
def to_json(self):
        ret = {}
        ret['type'] = 'AddGameEvent'
        ret['command'] = " ".join(self.command_words)
        ret['time'] = self.create_time
        return ret 
Example 49
Project: kicker-module   Author: EvanTheB   File: front.py    GNU General Public License v3.0 5 votes vote down vote up
def __init__(self):
        self.command = ["close_game",
                        "disrupt",
                        "class_warfare",
                        "sigma",
                        "time",
                        "variety",
                        "default",
                        "slow",
                        ] 
Example 50
Project: clikit   Author: sdispater   File: progress_bar.py    MIT License 5 votes vote down vote up
def __init__(self, io, max=0):  # type: (IO, int) -> None
        """
        Constructor.
        """

        self._io = io
        self._max = 0
        self._step_width = None
        self._set_max_steps(max)
        self._step = 0
        self._percent = 0.0
        self._format = None
        self._internal_format = None
        self._format_line_count = 0
        self._last_messages_length = 0
        self._should_overwrite = True

        if not self._io.error_output.supports_ansi():
            # Disable overwrite when output does not support ANSI codes.
            self._should_overwrite = False

            # Set a reasonable redraw frequency so output isn't flooded
            self.set_redraw_frequency(max / 10)

        self._messages = {}

        self._start_time = time.time()