Python logging.CRITICAL Examples

def __init__(self, use_Dingtalk=False):
        chrome_options = Options()
        self.__exit_flag = threading.Event()
        self.__exit_flag.clear()
        self.use_Dingtalk = use_Dingtalk
        if config['mute']:
            chrome_options.add_argument('--mute-audio')  # 关闭声音

        if os.path.exists('driver/chrome.exe'):
            chrome_options.binary_location = 'driver/chrome.exe'
        # chrome_options.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错
        # chrome_options.add_argument('window-size=800x600') #指定浏览器分辨率
        # chrome_options.add_argument('--disable-gpu') #谷歌文档提到需要加上这个属性来规避bug
        # chrome_options.add_argument('--hide-scrollbars') #隐藏滚动条, 应对一些特殊页面
        # chrome_options.add_argument('blink-settings=imagesEnabled=false') #不加载图片, 提升速度
        if config['background_process'] and self.use_Dingtalk:
            chrome_options.add_argument('--headless')  # 浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败
        self.driver = webdriver.Chrome('driver/chromedriver.exe', options=chrome_options)
        LOGGER.setLevel(logging.CRITICAL) 

def decompile_dictionary(infile, outfile, kind="token",
                         oenc="UTF-8",
                         log_level=logging.CRITICAL, log_file=None):
    if log_file is not None:
        decompile_dictionary_logger.addHandler(file_handler(log_file))
    decompile_dictionary_logger.setLevel(log_level)
    
    if kind not in _choices:
        raise RuntimeError("Invalid kind: {0}".format(kind))
    
    compile_dictionary_logger.info(u'compiling {0} dictionary from "{1}" to "{2}"'.format(kind, infile, outfile))
    
    resource = pickle.load(open(infile))
    entry    = _entry[kind]
    with codecs.open(outfile, "w", oenc) as O:
        tokens = []
        for token in resource:
            tokens.append(token[:])
        for token in sorted(tokens):
            O.write(entry(token) + u"\n")
    
    compile_dictionary_logger.info(u"done") 

def __init__(self, appname, dllname=None, logtype="Application"):
        logging.Handler.__init__(self)
        try:
            import win32evtlogutil, win32evtlog
            self.appname = appname
            self._welu = win32evtlogutil
            if not dllname:
                dllname = os.path.split(self._welu.__file__)
                dllname = os.path.split(dllname[0])
                dllname = os.path.join(dllname[0], r'win32service.pyd')
            self.dllname = dllname
            self.logtype = logtype
            self._welu.AddSourceToRegistry(appname, dllname, logtype)
            self.deftype = win32evtlog.EVENTLOG_ERROR_TYPE
            self.typemap = {
                logging.DEBUG   : win32evtlog.EVENTLOG_INFORMATION_TYPE,
                logging.INFO    : win32evtlog.EVENTLOG_INFORMATION_TYPE,
                logging.WARNING : win32evtlog.EVENTLOG_WARNING_TYPE,
                logging.ERROR   : win32evtlog.EVENTLOG_ERROR_TYPE,
                logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE,
         }
        except ImportError:
            print("The Python Win32 extensions for NT (service, event "\
                        "logging) appear not to be available.")
            self._welu = None 

def __init__(self, appname, dllname=None, logtype="Application"):
        logging.Handler.__init__(self)
        try:
            import win32evtlogutil, win32evtlog
            self.appname = appname
            self._welu = win32evtlogutil
            if not dllname:
                dllname = os.path.split(self._welu.__file__)
                dllname = os.path.split(dllname[0])
                dllname = os.path.join(dllname[0], r'win32service.pyd')
            self.dllname = dllname
            self.logtype = logtype
            self._welu.AddSourceToRegistry(appname, dllname, logtype)
            self.deftype = win32evtlog.EVENTLOG_ERROR_TYPE
            self.typemap = {
                logging.DEBUG   : win32evtlog.EVENTLOG_INFORMATION_TYPE,
                logging.INFO    : win32evtlog.EVENTLOG_INFORMATION_TYPE,
                logging.WARNING : win32evtlog.EVENTLOG_WARNING_TYPE,
                logging.ERROR   : win32evtlog.EVENTLOG_ERROR_TYPE,
                logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE,
         }
        except ImportError:
            print("The Python Win32 extensions for NT (service, event "\
                        "logging) appear not to be available.")
            self._welu = None 

def format(self, record):
        date = colored('[%(asctime)s @%(filename)s:%(lineno)d]', 'green')
        msg = '%(message)s'
        if record.levelno == logging.WARNING:
            fmt = date + ' ' + colored('WRN', 'red', attrs=['blink']) + ' ' + msg
        elif record.levelno == logging.ERROR or record.levelno == logging.CRITICAL:
            fmt = date + ' ' + colored('ERR', 'red', attrs=['blink', 'underline']) + ' ' + msg
        elif record.levelno == logging.DEBUG:
            fmt = date + ' ' + colored('DBG', 'yellow', attrs=['blink']) + ' ' + msg
        else:
            fmt = date + ' ' + msg
        if hasattr(self, '_style'):
            # Python3 compatibility
            self._style._fmt = fmt
        self._fmt = fmt
        return super(_MyFormatter, self).format(record) 

def set_level(self, level):
		"""
		Set the minimum level of messages to be logged.

		Level of Log Messages
		CRITICAL	50
		ERROR	40
		WARNING	30
		INFO	20
		DEBUG	10
		NOTSET	0

		@param level: minimum level of messages to be logged
		@type level: int or long

		@return: None
		@rtype: None
		"""
		assert level in self._levelNames

		list_of_handlers = self._logger.handlers
		for handler in list_of_handlers:
			handler.setLevel(level) 

def set_level(self, level):
        """
        Set the minimum level of messages to be logged.

        Level of Log Messages
        CRITICAL    50
        ERROR    40
        WARNING    30
        INFO    20
        DEBUG    10
        NOTSET    0

        @param level: minimum level of messages to be logged
        @type level: int or long

        @return: None
        @rtype: None
        """
        assert level in self._levelNames

        list_of_handlers = self._logger.handlers
        for handler in list_of_handlers:
            handler.setLevel(level) 

def set_log(level, filename='spider.log'):
    """
    return a log file object
    根据提示设置log打印
    """
    if not os.path.isdir(LOG_DIR):
    	os.mkdir(LOG_DIR)
    log_file = os.path.join(LOG_DIR, filename)
    if not os.path.isfile(log_file):
        os.mknod(log_file)
        os.chmod(log_file, 0777)
    log_level_total = {'debug': logging.DEBUG, 'info': logging.INFO, 'warning': logging.WARN, 'error': logging.ERROR,
                       'critical': logging.CRITICAL}
    logger_f = logging.getLogger('spider')
    logger_f.setLevel(logging.DEBUG)
    fh = logging.FileHandler(log_file,'a')
    fh.setLevel(log_level_total.get(level, logging.DEBUG))
    formatter = logging.Formatter('%(asctime)s  %(filename)s  [line:%(lineno)d] %(levelname)s  %(message)s')
    fh.setFormatter(formatter)
    logger_f.addHandler(fh)
    keep_fds = [fh.stream.fileno()]
    return logger_f,keep_fds 

def configure_logging():
    """Configure logging."""
    logging.basicConfig(stream=sys.stderr, format="%(message)s", level=logging.CRITICAL)
    logging.getLogger("greynoise").setLevel(logging.WARNING)
    structlog.configure(
        processors=[
            structlog.stdlib.add_logger_name,
            structlog.stdlib.add_log_level,
            structlog.stdlib.PositionalArgumentsFormatter(),
            structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M.%S"),
            structlog.processors.StackInfoRenderer(),
            structlog.processors.format_exc_info,
            structlog.dev.ConsoleRenderer(),
        ],
        context_class=dict,
        logger_factory=structlog.stdlib.LoggerFactory(),
        wrapper_class=structlog.stdlib.BoundLogger,
        cache_logger_on_first_use=True,
    ) 

def test_launch_instance_exception_on_flavors(self):
        trove_exception = self.exceptions.nova
        self.mock_flavor_list.side_effect = trove_exception

        toSuppress = ["trove_dashboard.content.databases."
                      "workflows.create_instance",
                      "horizon.workflows.base"]

        # Suppress expected log messages in the test output
        loggers = []
        for cls in toSuppress:
            logger = logging.getLogger(cls)
            loggers.append((logger, logger.getEffectiveLevel()))
            logger.setLevel(logging.CRITICAL)

        try:
            with self.assertRaises(exceptions.Http302):
                self.client.get(LAUNCH_URL)
                self.mock_datastore_flavors.assert_called_once_with(
                    test.IsHttpRequest(), mock.ANY, mock.ANY)

        finally:
            # Restore the previous log levels
            for (log, level) in loggers:
                log.setLevel(level) 

def get_color_wrapper(cls, level):
        if not cls.COLOR_MAP:
            import colorama

            def _color_wrapper(color_marker):
                def wrap_msg_with_color(msg):
                    return '{}{}{}'.format(color_marker, msg, colorama.Style.RESET_ALL)
                return wrap_msg_with_color

            cls.COLOR_MAP = {
                logging.CRITICAL: _color_wrapper(colorama.Fore.LIGHTRED_EX),
                logging.ERROR: _color_wrapper(colorama.Fore.LIGHTRED_EX),
                logging.WARNING: _color_wrapper(colorama.Fore.YELLOW),
                logging.INFO: _color_wrapper(colorama.Fore.GREEN),
                logging.DEBUG: _color_wrapper(colorama.Fore.CYAN)
            }

        return cls.COLOR_MAP.get(level, None) 

def sendLogMessage(self, level, msg, *args):
        """Sends a log message asynchronously.

        The method could be used safely from a non-GUI thread.

        level => integer, one of those found in logging:
                          logging.CRITICAL
                          logging.ERROR
                          logging.WARNING
                          logging.INFO
                          logging.DEBUG
        msg => message
        args => message arguments to be substituted (mgs % args)
        """
        try:
            self.__parent.pluginLogMessage.emit(level, msg % args)
        except Exception as exc:
            self.__parent.pluginLogMessage.emit(
                logging.ERROR,
                "Error sending a plugin log message. Error: " + str(exc)) 

def test_basic_loading(caplog):
    caplog.set_level(logging.CRITICAL)
    tokenizer = Tokenizer.load(
        pretrained_model_name_or_path="bert-base-cased",
        do_lower_case=True
        )
    assert type(tokenizer) == BertTokenizer
    assert tokenizer.basic_tokenizer.do_lower_case == True

    tokenizer = Tokenizer.load(
        pretrained_model_name_or_path="xlnet-base-cased",
        do_lower_case=True
        )
    assert type(tokenizer) == XLNetTokenizer
    assert tokenizer.do_lower_case == True

    tokenizer = Tokenizer.load(
        pretrained_model_name_or_path="roberta-base"
        )
    assert type(tokenizer) == RobertaTokenizer 

def test_bert_tokenizer_all_meta(caplog):
    caplog.set_level(logging.CRITICAL)

    lang_model = "bert-base-cased"

    tokenizer = Tokenizer.load(
        pretrained_model_name_or_path=lang_model,
        do_lower_case=False
        )

    basic_text = "Some Text with neverseentokens plus !215?#. and a combined-token_with/chars"

    # original tokenizer from transformer repo
    tokenized = tokenizer.tokenize(basic_text)
    assert tokenized == ['Some', 'Text', 'with', 'never', '##see', '##nto', '##ken', '##s', 'plus', '!', '215', '?', '#', '.', 'and', 'a', 'combined', '-', 'token', '_', 'with', '/', 'ch', '##ars']

    # ours with metadata
    tokenized_meta = tokenize_with_metadata(text=basic_text, tokenizer=tokenizer)
    assert tokenized_meta["tokens"] == tokenized
    assert tokenized_meta["offsets"] == [0, 5, 10, 15, 20, 23, 26, 29, 31, 36, 37, 40, 41, 42, 44, 48, 50, 58, 59, 64, 65, 69, 70, 72]
    assert tokenized_meta["start_of_word"] == [True, True, True, True, False, False, False, False, True, True, False, False, False, False, True, True, True, False, False, False, False, False, False, False] 

def test_truncate_sequences(caplog):
    caplog.set_level(logging.CRITICAL)

    lang_names = ["bert-base-cased", "roberta-base", "xlnet-base-cased"]
    tokenizers = []
    for lang_name in lang_names:
        t = Tokenizer.load(lang_name, lower_case=False)
        tokenizers.append(t)

    # artificial sequences (could be tokens, offsets, or anything else)
    seq_a = list(range(10))
    seq_b = list(range(15))
    max_seq_len = 20
    for tokenizer in tokenizers:
        for strategy in ["longest_first", "only_first","only_second"]:
            trunc_a, trunc_b, overflow = truncate_sequences(seq_a=seq_a,seq_b=seq_b,tokenizer=tokenizer,
                                                        max_seq_len=max_seq_len, truncation_strategy=strategy)

            assert len(trunc_a) + len(trunc_b) + tokenizer.num_special_tokens_to_add(pair=True) == max_seq_len 

def __init__(self, appname, dllname=None, logtype="Application"):
        logging.Handler.__init__(self)
        try:
            import win32evtlogutil, win32evtlog
            self.appname = appname
            self._welu = win32evtlogutil
            if not dllname:
                dllname = os.path.split(self._welu.__file__)
                dllname = os.path.split(dllname[0])
                dllname = os.path.join(dllname[0], r'win32service.pyd')
            self.dllname = dllname
            self.logtype = logtype
            self._welu.AddSourceToRegistry(appname, dllname, logtype)
            self.deftype = win32evtlog.EVENTLOG_ERROR_TYPE
            self.typemap = {
                logging.DEBUG   : win32evtlog.EVENTLOG_INFORMATION_TYPE,
                logging.INFO    : win32evtlog.EVENTLOG_INFORMATION_TYPE,
                logging.WARNING : win32evtlog.EVENTLOG_WARNING_TYPE,
                logging.ERROR   : win32evtlog.EVENTLOG_ERROR_TYPE,
                logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE,
         }
        except ImportError:
            print("The Python Win32 extensions for NT (service, event "\
                        "logging) appear not to be available.")
            self._welu = None 

def test_bert_custom_vocab(caplog):
    caplog.set_level(logging.CRITICAL)

    lang_model = "bert-base-cased"

    tokenizer = Tokenizer.load(
        pretrained_model_name_or_path=lang_model,
        do_lower_case=False
        )

    #deprecated: tokenizer.add_custom_vocab("samples/tokenizer/custom_vocab.txt")
    tokenizer.add_tokens(new_tokens=["neverseentokens"])

    basic_text = "Some Text with neverseentokens plus !215?#. and a combined-token_with/chars"

    # original tokenizer from transformer repo
    tokenized = tokenizer.tokenize(basic_text)
    assert tokenized == ['Some', 'Text', 'with', 'neverseentokens', 'plus', '!', '215', '?', '#', '.', 'and', 'a', 'combined', '-', 'token', '_', 'with', '/', 'ch', '##ars']

    # ours with metadata
    tokenized_meta = tokenize_with_metadata(text=basic_text, tokenizer=tokenizer)
    assert tokenized_meta["tokens"] == tokenized
    assert tokenized_meta["offsets"] == [0, 5, 10, 15, 31, 36, 37, 40, 41, 42, 44, 48, 50, 58, 59, 64, 65, 69, 70, 72]
    assert tokenized_meta["start_of_word"] == [True, True, True, True, True, True, False, False, False, False, True, True, True, False, False, False, False, False, False, False] 

def test_sample_to_features_qa(caplog):
    if caplog:
        caplog.set_level(logging.CRITICAL)

    sample_types = ["span", "no_answer"]

    for sample_type in sample_types:
        clear_text = json.load(open(f"samples/qa/{sample_type}/clear_text.json"))
        tokenized = json.load(open(f"samples/qa/{sample_type}/tokenized.json"))
        features_gold = json.load(open(f"samples/qa/{sample_type}/features.json"))
        max_seq_len = len(features_gold["input_ids"])

        tokenizer = Tokenizer.load(pretrained_model_name_or_path=MODEL, do_lower_case=False)
        curr_id = "-".join([str(x) for x in features_gold["id"]])

        s = Sample(id=curr_id, clear_text=clear_text, tokenized=tokenized)
        features = sample_to_features_qa(s, tokenizer, max_seq_len, SP_TOKENS_START, SP_TOKENS_MID)[0]
        features = to_list(features)

        keys = features_gold.keys()
        for k in keys:
            value_gold = features_gold[k]
            value = to_list(features[k])
            assert value == value_gold, f"Mismatch between the {k} features in the {sample_type} test sample." 

def activity_logger(logpipes, logfilename, terminate):
    handler = logging.handlers.RotatingFileHandler(
        logfilename,
        maxBytes=20971520,
        backupCount=10,
    )
    handler.setFormatter(logging.Formatter(fmt=None))
    logger = logging.getLogger('auditor-logger-raw')
    logger.addHandler(handler)
    logger.setLevel(logging.CRITICAL)  # The level of this logger is irrelevant

    while not terminate.is_set():
        ready, _, _ = select.select(logpipes, tuple(), tuple(), 30)
        if ready:
            for logpipe in ready:
                logger.critical(logpipe.recv()) 

def compile_dictionary(infile, outfile, kind="token",
                       ienc="UTF-8",
                       log_level=logging.CRITICAL, log_file=None):
    if log_file is not None:
        compile_dictionary_logger.addHandler(file_handler(log_file))
    compile_dictionary_logger.setLevel(log_level)
    
    if kind not in _choices:
        raise RuntimeError("Invalid kind: {0}".format(kind))
    
    compile_dictionary_logger.info(u'compiling {0} dictionary from "{1}" to "{2}"'.format(kind, infile, outfile))
    
    try:
        dictionary_compile = _compile[kind]
    except KeyError: # invalid kind asked
        compile_dictionary_logger.exception("Invalid kind: {0}. Should be in: {1}".format(kind, u", ".join(_compile.keys())))
        raise
    
    pickle.dump(dictionary_compile(infile, ienc), open(outfile, "w"))
    
    compile_dictionary_logger.info(u"done") 

def  set_logging(self, level):
        level_list= ["CRITICAL", "ERROR", "WARNING", "INFO", "DEBUG", "NOTSET"]
        if level in level_list:
            if(level == "CRITICAL"):
                logging.basicConfig(level=logging.CRITICAL)
            if (level == "ERROR"):
                logging.basicConfig(level=logging.ERROR)
            if (level == "WARNING"):
                logging.basicConfig(level=logging.WARNING)
            if (level == "INFO"):
                logging.basicConfig(level=logging.INFO)
            if (level == "DEBUG"):
                logging.basicConfig(level=logging.DEBUG)
            if (level == "NOTSET"):
                logging.basicConfig(level=logging.NOTSET)
        else:
            print ("set logging level failed ,the level is invalid.") 

def formatMessage(self, record: logging.LogRecord) -> str:
        """Convert the already filled log record to a string."""
        level_color = "0"
        text_color = "0"
        fmt = ""
        if record.levelno <= logging.DEBUG:
            fmt = "\033[0;37m" + logging.BASIC_FORMAT + "\033[0m"
        elif record.levelno <= logging.INFO:
            level_color = "1;36"
            lmsg = record.message.lower()
            if self.GREEN_RE.search(lmsg):
                text_color = "1;32"
        elif record.levelno <= logging.WARNING:
            level_color = "1;33"
        elif record.levelno <= logging.CRITICAL:
            level_color = "1;31"
        if not fmt:
            fmt = "\033[" + level_color + \
                  "m%(levelname)s\033[0m:%(rthread)s:%(name)s:\033[" + text_color + \
                  "m%(message)s\033[0m"
        fmt = _fest + fmt
        record.rthread = reduce_thread_id(record.thread)
        return fmt % record.__dict__ 

def getEventType(self, record):
        """
        Return the event type for the record.

        Override this if you want to specify your own types. This version does
        a mapping using the handler's typemap attribute, which is set up in
        __init__() to a dictionary which contains mappings for DEBUG, INFO,
        WARNING, ERROR and CRITICAL. If you are using your own levels you will
        either need to override this method or place a suitable dictionary in
        the handler's typemap attribute.
        """
        return self.typemap.get(record.levelno, self.deftype) 

def critical(self, msg, *args, **kwargs):
        """Delegate a critical call to the underlying logger."""

        self.log(logging.CRITICAL, msg, *args, **kwargs) 

def test_qa_onnx_inference(caplog=None):
    if caplog:
        caplog.set_level(logging.CRITICAL)


    QA_input = [
        {
            "questions": ["Who counted the game among the best ever made?"],
            "text": "Twilight Princess was released to universal critical acclaim and commercial success. It received perfect scores from major publications such as 1UP.com, Computer and Video Games, Electronic Gaming Monthly, Game Informer, GamesRadar, and GameSpy. On the review aggregators GameRankings and Metacritic, Twilight Princess has average scores of 95% and 95 for the Wii version and scores of 95% and 96 for the GameCube version. GameTrailers in their review called it one of the greatest games ever created."
        }]

    base_LM_model = "deepset/bert-base-cased-squad2"

    # Pytorch
    inferencer = Inferencer.load(base_LM_model, batch_size=2, gpu=False, task_type="question_answering", num_processes=0)
    result = inferencer.inference_from_dicts(dicts=QA_input)[0]

    # ONNX
    onnx_model_export_path = Path("testsave/onnx-export")
    inferencer.model.convert_to_onnx(onnx_model_export_path)
    inferencer = Inferencer.load(model_name_or_path=onnx_model_export_path, task_type="question_answering", num_processes=0)

    result_onnx = inferencer.inference_from_dicts(QA_input)[0]

    for (onnx, regular) in zip(result_onnx["predictions"][0]["answers"][0].items(), result["predictions"][0]["answers"][0].items()):
        # keys
        assert onnx[0] == regular[0]
        # values
        if type(onnx[1]) == float:
            np.testing.assert_almost_equal(onnx[1], regular[1], decimal=4)  # score
        else:
            assert onnx[1] == regular[1] 

def critical(cls, *args, **kwargs):
    return Log(logging.CRITICAL, *args, **kwargs) 

def test_conversion_adaptive_model(caplog):
    if caplog:
        caplog.set_level(logging.CRITICAL)

    model = AdaptiveModel.convert_from_transformers("deepset/bert-base-cased-squad2", device="cpu", task_type="question_answering")
    transformer_model = model.convert_to_transformers()
    transformer_model2 = AutoModelForQuestionAnswering.from_pretrained("deepset/bert-base-cased-squad2")
    # compare weights
    for p1, p2 in zip(transformer_model.parameters(), transformer_model2.parameters()):
        assert(p1.data.ne(p2.data).sum() == 0) 

def getEventType(self, record):
        """
        Return the event type for the record.

        Override this if you want to specify your own types. This version does
        a mapping using the handler's typemap attribute, which is set up in
        __init__() to a dictionary which contains mappings for DEBUG, INFO,
        WARNING, ERROR and CRITICAL. If you are using your own levels you will
        either need to override this method or place a suitable dictionary in
        the handler's typemap attribute.
        """
        return self.typemap.get(record.levelno, self.deftype) 

def _get_label(self, level):
        if level == logging.CRITICAL:
            return 'C'
        elif level == logging.ERROR:
            return 'E'
        elif level == logging.WARNING:
            return 'W'
        elif level == logging.INFO:
            return 'I'
        elif level == logging.DEBUG:
            return 'D'
        else:
            return 'U' 

def _get_console_log_configs():
        return [
            # --only-show-critical [RESERVED]
            {
                CLI_LOGGER_NAME: logging.CRITICAL,
                'root': logging.CRITICAL
            },
            # --only-show-errors
            {
                CLI_LOGGER_NAME: logging.ERROR,
                'root': logging.CRITICAL
            },
            # (default)
            {
                CLI_LOGGER_NAME: logging.WARNING,
                'root': logging.CRITICAL,
            },
            # --verbose
            {
                CLI_LOGGER_NAME: logging.INFO,
                'root': logging.CRITICAL,
            },
            # --debug
            {
                CLI_LOGGER_NAME: logging.DEBUG,
                'root': logging.DEBUG,
            }]