Python utils.get_logger() Examples

def __init__(self, batch_size, epoch_num, hidden_dim, embeddings,
                 dropout_keep, optimizer, lr, clip_grad,
                 tag2label, vocab, shuffle,
                 model_path, summary_path, log_path, result_path,
                 CRF=True, update_embedding=True):
        self.batch_size = batch_size
        self.epoch_num = epoch_num
        self.hidden_dim = hidden_dim
        self.embeddings = embeddings
        self.dropout_keep_prob = dropout_keep
        self.optimizer = optimizer
        self.lr = lr
        self.clip_grad = clip_grad
        self.tag2label = tag2label
        self.num_tags = len(tag2label)
        self.vocab = vocab  # word2id
        self.shuffle = shuffle
        self.model_path = model_path
        self.summary_path = summary_path
        self.logger = get_logger(log_path)
        self.result_path = result_path
        self.CRF = CRF
        self.update_embedding = update_embedding 

def _initialize_config(self):
        # create folders and logger
        if not os.path.exists(self.cfg["checkpoint_path"]):
            os.makedirs(self.cfg["checkpoint_path"])
        if not os.path.exists(self.cfg["summary_path"]):
            os.makedirs(self.cfg["summary_path"])
        self.logger = get_logger(os.path.join(self.cfg["checkpoint_path"], "log.txt"))
        # load dictionary
        dict_data = load_dataset(self.cfg["vocab"])
        self.word_dict, self.char_dict = dict_data["word_dict"], dict_data["char_dict"]
        self.tag_dict = dict_data["tag_dict"]
        del dict_data
        self.word_vocab_size = len(self.word_dict)
        self.char_vocab_size = len(self.char_dict)
        self.tag_vocab_size = len(self.tag_dict)
        self.rev_word_dict = dict([(idx, word) for word, idx in self.word_dict.items()])
        self.rev_char_dict = dict([(idx, char) for char, idx in self.char_dict.items()])
        self.rev_tag_dict = dict([(idx, tag) for tag, idx in self.tag_dict.items()]) 

def evaluate_line():
    config = load_config(FLAGS.config_file)
    logger = get_logger(FLAGS.log_file)
    # limit GPU memory
    tf_config = tf.ConfigProto()
    tf_config.gpu_options.allow_growth = True
    with open(FLAGS.map_file, "rb") as f:
        char_to_id, id_to_char, tag_to_id, id_to_tag = pickle.load(f)
    with tf.Session(config=tf_config) as sess:
        model = create_model(sess, Model, FLAGS.ckpt_path, load_word2vec, config, id_to_char, logger, False)
        while True:
            # try:
            #     line = input("请输入测试句子:")
            #     result = ckpt.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag)
            #     print(result)
            # except Exception as e:
            #     logger.info(e)

            line = input("请输入测试句子:")
            result = model.evaluate_line(sess, input_from_line(line, char_to_id), id_to_tag)
            print(result) 

def make_path_preparations(args):
    seed = hash(str(args)) % 1000_000
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    random.seed(seed)

    # logger path
    args_hash = str(hash(str(args)))
    if not os.path.exists(args.logs_path):
        os.makedirs(args.logs_path)
    logger = get_logger(f"{args.logs_path}/l{args_hash}.log")
    print(f"{args.logs_path}/l{args_hash}.log")
    logger.info(f"args: {str(args)}")
    logger.info(f"args hash: {args_hash}")
    logger.info(f"random seed: {seed}")

    # model path
    args.model_dir = f"{args.model_dir}/m{args_hash}"
    if not os.path.exists(args.model_dir):
        os.makedirs(args.model_dir)
    logger.info(f"checkpoint's dir is: {args.model_dir}")

    # tensorboard path
    tensorboard_path = f"{args.tensorboard_path}/t{args_hash}"
    if not os.path.exists(tensorboard_path):
        os.makedirs(tensorboard_path)
    summary_writer = dict()
    summary_writer["train"] = SummaryWriter(log_dir=os.path.join(tensorboard_path, 'log' + args_hash, 'train'))
    summary_writer["valid"] = SummaryWriter(log_dir=os.path.join(tensorboard_path, 'log' + args_hash, 'valid'))

    return logger, summary_writer 

def make_path_preparations(args):
    seed = hash(str(args)) % 1000_000
    ListOpsBucketSampler.random_seed = seed
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    random.seed(seed)

    # logger path
    args_hash = str(hash(str(args)))
    if not os.path.exists(args.logs_path):
        os.makedirs(args.logs_path)
    logger = get_logger(f"{args.logs_path}/l{args_hash}.log")
    print(f"{args.logs_path}/l{args_hash}.log")
    logger.info(f"args: {str(args)}")
    logger.info(f"args hash: {args_hash}")
    logger.info(f"random seed: {seed}")

    # model path
    args.model_dir = f"{args.model_dir}/m{args_hash}"
    if not os.path.exists(args.model_dir):
        os.makedirs(args.model_dir)
    logger.info(f"checkpoint's dir is: {args.model_dir}")

    # tensorboard path
    tensorboard_path = f"{args.tensorboard_path}/t{args_hash}"
    if not os.path.exists(tensorboard_path):
        os.makedirs(tensorboard_path)
    summary_writer = dict()
    summary_writer["train"] = SummaryWriter(log_dir=os.path.join(tensorboard_path, 'log' + args_hash, 'train'))
    summary_writer["valid"] = SummaryWriter(log_dir=os.path.join(tensorboard_path, 'log' + args_hash, 'valid'))

    return logger, summary_writer 

def make_path_preparations(args):
    # TODO
    seed = 42
    # seed = hash(str(args)) % 1000_000
    ListOpsBucketSampler.random_seed = seed
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    random.seed(seed)

    # logger path
    args_hash = str(hash(str(args)))
    if not os.path.exists(args.logs_path):
        os.makedirs(args.logs_path)
    logger = get_logger(f"{args.logs_path}/l{args_hash}.log")
    print(f"{args.logs_path}/l{args_hash}.log")
    logger.info(f"args: {str(args)}")
    logger.info(f"args hash: {args_hash}")
    logger.info(f"random seed: {seed}")

    # model path
    args.model_dir = f"{args.model_dir}/m{args_hash}"
    if not os.path.exists(args.model_dir):
        os.makedirs(args.model_dir)
    logger.info(f"checkpoint's dir is: {args.model_dir}")

    # tensorboard path
    tensorboard_path = f"{args.tensorboard_path}/t{args_hash}"
    if not os.path.exists(tensorboard_path):
        os.makedirs(tensorboard_path)
    summary_writer = dict()
    summary_writer["train"] = SummaryWriter(log_dir=os.path.join(tensorboard_path, 'log' + args_hash, 'train'))
    summary_writer["valid"] = SummaryWriter(log_dir=os.path.join(tensorboard_path, 'log' + args_hash, 'valid'))

    return logger, summary_writer 

def make_path_preparations(args):
    seed = hash(str(args)) % 1000_000
    ListOpsBucketSampler.random_seed = seed
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    random.seed(seed)

    # logger path
    args_hash = str(hash(str(args)))
    if not os.path.exists(args.logs_path):
        os.makedirs(args.logs_path)
    logger = get_logger(f"{args.logs_path}/l{args_hash}.log")
    print(f"{args.logs_path}/l{args_hash}.log")
    logger.info(f"args: {str(args)}")
    logger.info(f"args hash: {args_hash}")
    logger.info(f"random seed: {seed}")

    # model path
    args.model_dir = f"{args.model_dir}/m{args_hash}"
    if not os.path.exists(args.model_dir):
        os.makedirs(args.model_dir)
    logger.info(f"checkpoint's dir is: {args.model_dir}")

    # tensorboard path
    tensorboard_path = f"{args.tensorboard_path}/t{args_hash}"
    if not os.path.exists(tensorboard_path):
        os.makedirs(tensorboard_path)
    summary_writer = dict()
    summary_writer["train"] = SummaryWriter(log_dir=os.path.join(tensorboard_path, 'log' + args_hash, 'train'))
    summary_writer["valid"] = SummaryWriter(log_dir=os.path.join(tensorboard_path, 'log' + args_hash, 'valid'))

    return logger, summary_writer 

def make_path_preparations(args):
    seed = hash(str(args)) % 1000_000
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    random.seed(seed)

    # logger path
    args_hash = str(hash(str(args)))
    if not os.path.exists(args.logs_path):
        os.makedirs(args.logs_path)
    logger = get_logger(f"{args.logs_path}/l{args_hash}.log")
    print(f"{args.logs_path}/l{args_hash}.log")
    logger.info(f"args: {str(args)}")
    logger.info(f"args hash: {args_hash}")
    logger.info(f"random seed: {seed}")

    # model path
    args.model_dir = f"{args.model_dir}/m{args_hash}"
    if not os.path.exists(args.model_dir):
        os.makedirs(args.model_dir)
    logger.info(f"checkpoint's dir is: {args.model_dir}")

    # tensorboard path
    tensorboard_path = f"{args.tensorboard_path}/t{args_hash}"
    if not os.path.exists(tensorboard_path):
        os.makedirs(tensorboard_path)
    summary_writer = dict()
    summary_writer["train"] = SummaryWriter(log_dir=os.path.join(tensorboard_path, 'log' + args_hash, 'train'))
    summary_writer["valid"] = SummaryWriter(log_dir=os.path.join(tensorboard_path, 'log' + args_hash, 'valid'))

    return logger, summary_writer 

def __init__(self, config, resume_training=True, model_name='dense_bi_lstm'):
        # set configurations
        self.cfg, self.model_name, self.resume_training, self.start_epoch = config, model_name, resume_training, 1
        self.logger = get_logger(os.path.join(self.cfg.ckpt_path, 'log.txt'))
        # build model
        self._add_placeholder()
        self._add_embedding_lookup()
        self._build_model()
        self._add_loss_op()
        self._add_accuracy_op()
        self._add_train_op()
        print('params number: {}'.format(np.sum([np.prod(v.get_shape().as_list()) for v in tf.trainable_variables()])))
        # initialize model
        self.sess, self.saver = None, None
        self.initialize_session() 

def __init__(self):
        self.settings = get_channel_reaper_settings()
        self.logger = get_logger('channel_reaper', './audit.log') 

def print_stat_message(update, context):
    chat_id = update.effective_chat.id
    utils.get_logger().info(f'[{chat_id}] Displaying statistics', extra={'user': chat_id})

    msg = get_msg(update)
    all_jobs = [x for x in job_storage.get_jobs() if 'chat_id' in x.kwargs]
    average_interval = sum([x.trigger.interval.seconds for x in all_jobs]) / 60 / len(all_jobs)

    termin_list = [x.kwargs['termin'] for x in all_jobs]
    most_popular_termin = max(set(termin_list), key=lambda x: termin_list.count(x))

    msg.reply_text(f'ℹ️ Some piece of statistics:\n\n'
                   f'{len(all_jobs)} active subscription(s)\n'
                   f'{average_interval} min average interval\n'
                   f'{most_popular_termin} is the most popular termin') 

def clear_jobs():
    logger = utils.get_logger()
    logger.info("Cleaning jobs...")

    # remove jobs scheduled more than a week ago
    for job in scheduler.get_jobs():
        if 'created_at' in job.kwargs and (datetime.datetime.now() - job.kwargs['created_at']).days >= 7:
            logger.info("Removing job %s" % job.kwargs['chat_id'], extra={'user': job.kwargs['chat_id']})
            remove_subscription(job.kwargs['chat_id'], automatic=True) 

def init_scheduler():
    scheduler.start()
    if not scheduler.get_job('cleanup'):
        scheduler.add_job(clear_jobs, "interval", minutes=30, id="cleanup")
    else:
        # Just to make sure interval is always correct here
        utils.get_logger().info("Rescheduling cleanup job...")
        scheduler.reschedule_job('cleanup', trigger='interval', minutes=30) 

def add_subscription(update, context, interval):
    logger = utils.get_logger()
    metric_collector = MetricCollector.get_collector()

    buro = context.user_data['buro']
    termin = context.user_data['termin_type']
    chat_id = str(update.effective_chat.id)
    kwargs = {'chat_id': chat_id, 'buro': buro.get_id(), 'termin': termin, 'created_at': datetime.datetime.now()}
    scheduler.add_job(printers.notify_about_termins, 'interval', kwargs=kwargs, minutes=int(interval),
                      id=chat_id)

    logger.info(f'[{chat_id}] Subscription for %s-{termin} created with interval {interval}' % buro.get_name(), extra={'user': chat_id})
    metric_collector.log_subscription(buro=buro, appointment=termin, interval=interval, user=int(chat_id)) 

def remove_subscription(chat_id, automatic=False):
    if not scheduler.get_job(chat_id):
        return
    scheduler.remove_job(chat_id)
    if automatic:
        utils.get_logger().info(f'[{chat_id}] Subscription removed since it\'s expired', extra={'user': chat_id})
        utils.get_bot().send_message(chat_id=chat_id,
                                     text='Subscription was removed since it was created more than a week ago')
    else:
        utils.get_logger().info(f'[{chat_id}] Subscription removed by request', extra={'user': chat_id})
        utils.get_bot().send_message(chat_id=chat_id, text='You were unsubscribed successfully')