Python logging.FileHandler() Examples

def get_logger(cls,logger_name,create_file=False):

        # create logger for prd_ci
        log = logging.getLogger(logger_name)
        log.setLevel(level=logging.INFO)

        # create formatter and add it to the handlers
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

        if create_file:
                # create file handler for logger.
                fh = logging.FileHandler('SPOT.log')
                fh.setLevel(level=logging.DEBUG)
                fh.setFormatter(formatter)
        # reate console handler for logger.
        ch = logging.StreamHandler()
        ch.setLevel(level=logging.DEBUG)
        ch.setFormatter(formatter)

        # add handlers to logger.
        if create_file:
            log.addHandler(fh)

        log.addHandler(ch)
        return  log 

def log_config(log_dir=None, log_file=None, prefix=None, rank=0):
    reload(logging)
    head = '%(asctime)-15s Node[' + str(rank) + '] %(message)s'
    if log_dir:
        logging.basicConfig(level=logging.DEBUG, format=head)
        if not os.path.exists(log_dir):
            os.makedirs(log_dir)
        if not log_file:
            log_file = (prefix if prefix else '') + datetime.now().strftime('_%Y_%m_%d-%H_%M.log')
            log_file = log_file.replace('/', '-')
        else:
            log_file = log_file
        log_file_full_name = os.path.join(log_dir, log_file)
        handler = logging.FileHandler(log_file_full_name, mode='w')
        formatter = logging.Formatter(head)
        handler.setFormatter(formatter)
        logging.getLogger().addHandler(handler)
        logging.info('start with arguments %s', args)
    else:
        logging.basicConfig(level=logging.DEBUG, format=head)
        logging.info('start with arguments %s', args) 

def getLogger(fname=None):
    format='%(asctime)s:%(process)d:%(threadName)s:%(levelname)s:%(name)s:%(pathname)s %(lineno)d:%(message)s'
    formatter = MyFormatter(format)

    logger = logging.getLogger('sdx')

    if fname:
        fh = logging.FileHandler(fname)
        fh.setFormatter(formatter)
        logger.addHandler(fh)

    ch = logging.StreamHandler(stream=sys.stdout)
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    return logger 

def get_logger(cls, logger_name, create_file=False):

        # create logger for prd_ci
        log = logging.getLogger(logger_name)
        log.setLevel(level=logging.INFO)

        # create formatter and add it to the handlers
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')

        if create_file:
            # create file handler for logger.
            fh = logging.FileHandler('oa.log')
            fh.setLevel(level=logging.DEBUG)
            fh.setFormatter(formatter)
        # reate console handler for logger.
        ch = logging.StreamHandler()
        ch.setLevel(level=logging.DEBUG)
        ch.setFormatter(formatter)

        # add handlers to logger.
        if create_file:
            log.addHandler(fh)

        log.addHandler(ch)
        return log 

def create_file_logger(root, name, file_name="log.txt", timestamp_format="", debug=False):
    file_api = FileAPI(root)
    timestamp = ""
    if timestamp_format:
        timestamp = datetime.now().strftime(timestamp_format)
    else:
        timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")

    folder = name + "_" + timestamp
    # prepare folder and file
    with file_api.open_with_mkdir(folder + "/" + file_name) as f:
        f.write("".encode("utf-8"))

    log_root = os.path.join(root, folder)
    logger = create_logger(name, debug)
    fh = FileHandler(os.path.join(log_root, file_name), encoding="utf-8")
    fh.setLevel(_bool_2_level(debug))
    logger.addHandler(fh)

    # add close method to release resource
    logger.close = types.MethodType(__close, logger)
    return logger, log_root 

def setup_logging(log_loc):
    if os.path.exists(log_loc):
        shutil.move(log_loc, log_loc + "_" + str(int(os.path.getctime(log_loc))))
    os.makedirs(log_loc)

    log_file = '{}/benchmark.log'.format(log_loc)
    LOGGER = logging.getLogger('benchmark')
    LOGGER.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s %(levelname)s:%(name)s %(message)s')
    file_handler = logging.FileHandler(log_file)
    console_handler = logging.StreamHandler()
    file_handler.setFormatter(formatter)
    console_handler.setFormatter(formatter)

    LOGGER.addHandler(file_handler)
    LOGGER.addHandler(console_handler)
    return LOGGER 

def create_logger():
    logger = logging.getLogger('sgd_svrg')
    logger.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(message)s')
    fh = logging.FileHandler('experiments.log')
    fh.setFormatter(formatter)
    logger.addHandler(fh)
    return logger


################################################################################
# Functions below are for benchmark purpose to calcuate expectation, variance of
# gradients per epoch for each parameter. These calculations will be helpful when
# benchmarking SVRG optimization with other optimization techniques, such as SGD.
# Currently it only calculates the expectation, variance for single context but
# can be extended to multi-context in later iterations.
################################################################################ 

def logging_config(name=None, level=logging.DEBUG, console_level=logging.DEBUG):
    if name is None:
        name = inspect.stack()[1][1].split('.')[0]
    folder = os.path.join(os.getcwd(), name)
    if not os.path.exists(folder):
        os.makedirs(folder)
    logpath = os.path.join(folder, name + ".log")
    print("All Logs will be saved to %s"  %logpath)
    logging.root.setLevel(level)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    logfile = logging.FileHandler(logpath)
    logfile.setLevel(level)
    logfile.setFormatter(formatter)
    logging.root.addHandler(logfile)
    #TODO Update logging patterns in other files
    logconsole = logging.StreamHandler()
    logconsole.setLevel(console_level)
    logconsole.setFormatter(formatter)
    logging.root.addHandler(logconsole)
    return folder 

def setup_logging(filepath, verbose):
    logFormatter = logging.Formatter(
        '%(levelname)s %(asctime)-20s:\t %(message)s')
    rootLogger = logging.getLogger()
    if verbose:
        rootLogger.setLevel(logging.DEBUG)
    else:
        rootLogger.setLevel(logging.INFO)
    logging.getLogger('PIL').setLevel(logging.WARNING)

    # Setup the logger to write into file
    fileHandler = logging.FileHandler(filepath)
    fileHandler.setFormatter(logFormatter)
    rootLogger.addHandler(fileHandler)

    # Setup the logger to write into stdout
    consoleHandler = logging.StreamHandler(sys.stdout)
    consoleHandler.setFormatter(logFormatter)
    rootLogger.addHandler(consoleHandler) 

def train(session, model, curr_dir, data_train, data_val):
    curr_dir = os.path.join(curr_dir, model.algorithm)
    bestmodel_dir = os.path.join(curr_dir, 'best_checkpoint')
    
    if not os.path.exists(curr_dir):
        os.makedirs(curr_dir)
    
    file_handler = logging.FileHandler(os.path.join(curr_dir, 'log.txt'))
    logging.getLogger().addHandler(file_handler)
    
    with open(os.path.join(curr_dir, FLAGS['save_name'] + '.json'), 'w') as f:
        json.dump(FLAGS, f)
    
    if not os.path.exists(bestmodel_dir):
        os.makedirs(bestmodel_dir)
    
    initialize_model(session, model, curr_dir, expect_exists=False)
    model.train(session, curr_dir, bestmodel_dir, data_train, data_val) 

def get_logger(exp_dir):
    """
    creates logger instance. writing out info to file and to terminal.
    :param exp_dir: experiment directory, where exec.log file is stored.
    :return: logger instance.
    """

    logger = logging.getLogger('medicaldetectiontoolkit')
    logger.setLevel(logging.DEBUG)
    log_file = exp_dir + '/exec.log'
    hdlr = logging.FileHandler(log_file)
    print('Logging to {}'.format(log_file))
    logger.addHandler(hdlr)
    logger.addHandler(ColorHandler())
    logger.propagate = False
    return logger 

def get_logger(name=None, args=None, level=INFO, append=False):
    """Get customized logger.

    Args:
        name: Name of the logger.
        level: Level to log.

    Returns:
        A logger.
    """
    logger = logging.getLogger(name)
    if name and not getattr(logger, '_init_done', None):
        logger._init_done = True
        logger.addHandler(_handler)
        if args is not None:
            fileHandler = None
            if append:
                fileHandler = logging.FileHandler(args.log, mode='a')
            else:
                fileHandler = logging.FileHandler(args.log, mode='w')
            fileHandler.setFormatter(_Formatter())
            logger.addHandler(fileHandler)
        logger.setLevel(level)
    return logger 

def getLogger(name=None):
		#configure logging
		LoggerManager._loggers[name] = logging.getLogger(name)
		LoggerManager._loggers[name].setLevel(logging.INFO)
		
		if logging_dest == 'mysql':
			db = {'host':mysql_hostname, 'port':3306, 'dbuser':mysql_username, 'dbpassword':mysql_password, 'dbname':mysql_database}
		
			sqlh = mySQLHandler.mySQLHandler(db)
			LoggerManager._loggers[name].addHandler(sqlh)
		else:
			fileh = logging.FileHandler('actions.log')
			fileh.setFormatter(logging.Formatter('%(asctime)s - %(module)s - %(message)s'))
			LoggerManager._loggers[name].addHandler(fileh)
		
		requests_log = logging.getLogger("requests")
		requests_log.setLevel(logging.WARNING)
		
		return LoggerManager._loggers[name] 

def get_logger(logpath, filepath, package_files=[], displaying=True, saving=True, debug=False):
    logger = logging.getLogger()
    if debug:
        level = logging.DEBUG
    else:
        level = logging.INFO
    logger.setLevel(level)
    if saving:
        info_file_handler = logging.FileHandler(logpath, mode="a")
        info_file_handler.setLevel(level)
        logger.addHandler(info_file_handler)
    if displaying:
        console_handler = logging.StreamHandler()
        console_handler.setLevel(level)
        logger.addHandler(console_handler)
    logger.info(filepath)
    with open(filepath, "r") as f:
        logger.info(f.read())

    for f in package_files:
        logger.info(f)
        with open(f, "r") as package_f:
            logger.info(package_f.read())

    return logger 

def modify_logger(logger, log_file):
    # refer: https://docs.python.org/3.5/library/logging.html#logrecord-attributes
    formatter = logging.Formatter(
        fmt='\n'.join([
            '[%(name)s] %(asctime)s.%(msecs)d',
            '\t%(pathname)s [line: %(lineno)d]',
            '\t%(processName)s[%(process)d] => %(threadName)s[%(thread)d] => %(module)s.%(filename)s:%(funcName)s()',
            '\t%(levelname)s: %(message)s\n'
        ]),
        datefmt='%Y-%m-%d %H:%M:%S'
    )
    # stream_handler = logging.StreamHandler()
    # stream_handler.setFormatter(formatter)
    # logger.addHandler(stream_handler)

    file_handler = logging.FileHandler(log_file, mode='a', encoding='utf-8')
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)

    logger.setLevel(logging.DEBUG)

    return logger 

def get_logger(logger_name='main'):
    logger = logging.getLogger(logger_name)
    if logger.handlers:
        return logger
    log_file = os.path.join(app_dir.user_log_dir, '{}.log'.format(logger_name))
    if not os.path.exists(app_dir.user_log_dir):
        os.makedirs(app_dir.user_log_dir)
    formatter = logging.Formatter(config.get('LOG', 'format',
                                             fallback="%(asctime)s %(name)s [%(levelname)s] %(message)s"))
    file_handler = logging.FileHandler(log_file, encoding='utf-8')
    file_handler.setFormatter(formatter)
    console_handler = logging.StreamHandler()
    console_handler.setFormatter(formatter)
    logger.setLevel(config.get('LOG', 'level', fallback='ERROR'))
    logger.addHandler(file_handler)
    logger.addHandler(console_handler)
    return logger 

def prepare_dirs_loggers(config, script=""):
    logFormatter = logging.Formatter("%(message)s")
    rootLogger = logging.getLogger()
    rootLogger.setLevel(logging.DEBUG)

    consoleHandler = logging.StreamHandler(sys.stdout)
    consoleHandler.setLevel(logging.DEBUG)
    consoleHandler.setFormatter(logFormatter)
    rootLogger.addHandler(consoleHandler)

    if hasattr(config, 'forward_only') and config.forward_only:
        return

    fileHandler = logging.FileHandler(os.path.join(config.saved_path,'session.log'))
    fileHandler.setLevel(logging.DEBUG)
    fileHandler.setFormatter(logFormatter)
    rootLogger.addHandler(fileHandler) 

def setup_logs(save_dir, run_name):
    # initialize logger
    logger = logging.getLogger("anti-spoofing")
    logger.setLevel(logging.INFO)

    # create the logging file handler
    log_file = os.path.join(save_dir, run_name + ".log")
    fh = logging.FileHandler(log_file)

    # create the logging console handler
    ch = logging.StreamHandler()

    # format
    formatter = logging.Formatter("%(asctime)s - %(message)s")
    fh.setFormatter(formatter)

    # add handlers to logger object
    logger.addHandler(fh)
    logger.addHandler(ch)

    return logger 

def setup_logger(name, save_dir, distributed_rank, filename="log.txt"):
    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)
    # don't log results for the non-master process
    if distributed_rank > 0:
        return logger
    ch = logging.StreamHandler(stream=sys.stdout)
    ch.setLevel(logging.DEBUG)
    formatter = logging.Formatter("%(asctime)s %(name)s %(levelname)s: %(message)s")
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    if save_dir:
        fh = logging.FileHandler(os.path.join(save_dir, filename))
        fh.setLevel(logging.DEBUG)
        fh.setFormatter(formatter)
        logger.addHandler(fh)

    return logger 

def get_logger(self) -> logging.Logger:
        if self.__logger is None:
            self.__logger = logging.getLogger('code2vec')
            self.__logger.setLevel(logging.INFO)
            self.__logger.handlers = []
            self.__logger.propagate = 0

            formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')

            if self.VERBOSE_MODE >= 1:
                ch = logging.StreamHandler(sys.stdout)
                ch.setLevel(logging.INFO)
                ch.setFormatter(formatter)
                self.__logger.addHandler(ch)

            if self.LOGS_PATH:
                fh = logging.FileHandler(self.LOGS_PATH)
                fh.setLevel(logging.INFO)
                fh.setFormatter(formatter)
                self.__logger.addHandler(fh)

        return self.__logger 

def reopen_files(self):
        """Close and reopen all file handlers."""
        for log in (self.error_log, self.access_log):
            for h in log.handlers:
                if isinstance(h, logging.FileHandler):
                    h.acquire()
                    h.stream.close()
                    h.stream = open(h.baseFilename, h.mode)
                    h.release() 

def _add_builtin_file_handler(self, log, fname):
        h = logging.FileHandler(fname)
        h.setFormatter(logfmt)
        h._cpbuiltin = 'file'
        log.addHandler(h) 

def error_file(self):
        """The filename for self.error_log.

        If you set this to a string, it'll add the appropriate FileHandler for
        you. If you set it to ``None`` or ``''``, it will remove the handler.
        """
        h = self._get_builtin_handler(self.error_log, 'file')
        if h:
            return h.baseFilename
        return '' 

def access_file(self):
        """The filename for self.access_log.

        If you set this to a string, it'll add the appropriate FileHandler for
        you. If you set it to ``None`` or ``''``, it will remove the handler.
        """
        h = self._get_builtin_handler(self.access_log, 'file')
        if h:
            return h.baseFilename
        return '' 

def __init__(self, ckpt_path, name='train'):
    self.logger = logging.getLogger()
    self.logger.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s %(message)s',
                                  datefmt=blue('[%Y-%m-%d,%H:%M:%S]'))

    fh = logging.FileHandler(os.path.join(ckpt_path, '{}.log'.format(name)), 'w')
    fh.setLevel(logging.INFO)
    fh.setFormatter(formatter)
    self.logger.addHandler(fh)

    ch = logging.StreamHandler(sys.stdout)
    ch.setLevel(logging.INFO)
    ch.setFormatter(formatter)
    self.logger.addHandler(ch) 

def set_log_file(self, log_file, mode='a', level=logging.INFO):
		"""
		Add a stream where messages are outputted to.

		@attention: file stream will only be closed if a file path is given!

		@param log_file: file stream or file path of logfile
		@type log_file: file | FileIO | StringIO | basestring
		@param mode: opening mode for logfile, if a file path is given
		@type mode: basestring
		@param level: minimum level of messages to be logged
		@type level: int or long

		@return: None
		@rtype: None
		"""
		assert isinstance(log_file, basestring) or self.is_stream(log_file)
		assert level in self._levelNames

		if LoggingWrapper._map_logfile_handler[self._label] is not None:
			self._logger.removeHandler(LoggingWrapper._map_logfile_handler[self._label])
			LoggingWrapper._map_logfile_handler[self._label].close()
			LoggingWrapper._map_logfile_handler[self._label] = None

		if self.is_stream(log_file):
			self.add_log_stream(stream=log_file, level=level)
			return

		try:
			err_handler_file = logging.FileHandler(log_file, mode)
			err_handler_file.setFormatter(self.message_formatter)
			err_handler_file.setLevel(level)
			self._logger.addHandler(err_handler_file)
			LoggingWrapper._map_logfile_handler[self._label] = err_handler_file
		except Exception:
			sys.stderr.write("[LoggingWrapper] Could not open '{}' for logging\n".format(log_file))
			return 

def set_log_file(self, log_file, mode='a', level=logging.INFO):
        """
        Add a stream where messages are outputted to.

        @attention: file stream will only be closed if a file path is given!

        @param log_file: file stream or file path of logfile
        @type log_file: file | FileIO | StringIO | str
        @param mode: opening mode for logfile, if a file path is given
        @type mode: str
        @param level: minimum level of messages to be logged
        @type level: int or long

        @return: None
        @rtype: None
        """
        assert isinstance(log_file, str) or self.is_stream(log_file)
        assert level in self._levelNames

        if LoggingWrapper._map_logfile_handler[self._label] is not None:
            self._logger.removeHandler(LoggingWrapper._map_logfile_handler[self._label])
            LoggingWrapper._map_logfile_handler[self._label].close()
            LoggingWrapper._map_logfile_handler[self._label] = None

        if self.is_stream(log_file):
            self.add_log_stream(stream=log_file, level=level)
            return

        try:
            err_handler_file = logging.FileHandler(log_file, mode)
            err_handler_file.setFormatter(self.message_formatter)
            err_handler_file.setLevel(level)
            self._logger.addHandler(err_handler_file)
            LoggingWrapper._map_logfile_handler[self._label] = err_handler_file
        except Exception:
            sys.stderr.write("[LoggingWrapper] Could not open '{}' for logging\n".format(log_file))
            return 

def _get_logger(self, name, log_path, lvl=logging.INFO, display_name=None):
        if isinstance(lvl, str):
            lvl = lvl.lower().strip()
            if lvl == 'debug':
                lvl = logging.DEBUG
            elif lvl == 'info':
                lvl = logging.INFO
            elif lvl == 'warn' or lvl == 'warning':
                lvl = logging.WARN
            elif lvl == 'error':
                lvl = logging.ERROR
            elif lvl == 'fatal' or lvl == 'critical':
                lvl = logging.CRITICAL
            else:
                raise ValueError('unknown logging level')

        file_handler = logging.FileHandler(log_path)
        file_handler.setLevel(logging.DEBUG)
        file_handler.setFormatter(LoggerClass._normal_formatter)
        console_handler = logging.StreamHandler()
        console_handler.setLevel(lvl)
        console_handler.setFormatter(LoggerClass._color_formatter)
        if display_name is None:
            display_name = name
        logger = logging.getLogger(display_name)
        logger.setLevel(logging.DEBUG)
        logger.addHandler(console_handler)
        logger.addHandler(file_handler)

        return logger

    ###############
    ### Logging ###
    ############### 

def __init__(self, cfg="/etc/insightfinder.ini", lvl="INFO", attempts=1, flush_kb=3000):
        """
        Implements an interface that allows metrics to be persisted to InsightFinder.
        Raises a :class:`ValueError` on bad arguments or `Exception` on missing
        configuration section.
        :Parameters:
                - `cfg` (optional) : INI configuration file.
                - `lvl` (optional) : logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
                - `attempts` (optional) : The number of re-connect retries before failing.
                - `flush_kb` (optional) : The size of metric data to send in KB(The actual metrics will have a few statsite metrics too)
        """
        if attempts < 1:
            raise ValueError("Must have at least 1 attempt!")

        attempts = int(attempts)

        #Set up logging
        self.logger = logging.getLogger(__name__)
        # create a file handler
        handler = logging.handlers.RotatingFileHandler('insightfinder.log', mode='w', maxBytes=5*1024*1024,
                                 backupCount=2, encoding=None, delay=0)
        # handler = logging.FileHandler('insightfinder.log')
        handler.setLevel(lvl)
        # create a logging format
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(process)d - %(threadName)s - %(levelname)s - %(message)s')
        handler.setFormatter(formatter)
        # add the handlers to the logger
        self.logger.addHandler(handler)
        self.logger.setLevel(lvl)

        self.attempts = attempts
        self.metrics_map = {}
        self.to_send_metrics = []
        self.cfg = cfg
        self.load(cfg)
        self.temp_group_id = 10000
        self.flush_kb = int(flush_kb)
        self._load_grouping() 

def initialize(path_to_log_file):
        logging.basicConfig(level=logging.INFO,
                            format='%(asctime)s %(levelname)-8s %(message)s',
                            datefmt='%Y-%m-%d %H:%M:%S',
                            handlers=[logging.FileHandler(path_to_log_file),
                                      logging.StreamHandler()])
        Logger.Initialized = True