Python logging.fatal() Examples

def get_args_for_config(config_name):
  configs = config_name.split('.')
  type = configs[0]
  config_name = '.'.join(configs[1:])
  if type == 'cmp':
    args = config_cmp.get_args_for_config(config_name)
    args.setup_to_run = cmp.setup_to_run
    args.setup_train_step_kwargs = cmp.setup_train_step_kwargs

  elif type == 'bl':
    args = config_vision_baseline.get_args_for_config(config_name)
    args.setup_to_run = vision_baseline_lstm.setup_to_run
    args.setup_train_step_kwargs = vision_baseline_lstm.setup_train_step_kwargs

  else:
    logging.fatal('Unknown type: {:s}'.format(type))
  return args 

def set_verbosity(v):
  """Sets the logging verbosity.

  Causes all messages of level <= v to be logged,
  and all messages of level > v to be silently discarded.

  Args:
    v: int|str, the verbosity level as an integer or string. Legal string values
        are those that can be coerced to an integer as well as case-insensitive
        'debug', 'info', 'warning', 'error', and 'fatal'.
  """
  try:
    new_level = int(v)
  except ValueError:
    new_level = converter.ABSL_NAMES[v.upper()]
  FLAGS.verbosity = new_level 

def __init__(self, config, timer, db):
        self.config             = config
        self.timer              = timer
        self.db                 = db
        self.balancer_wait_secs = self.config.sharding.balancer.wait_secs
        self.balancer_sleep     = self.config.sharding.balancer.ping_secs

        self.timer_name            = self.__class__.__name__
        self.config_server         = None
        self.config_db             = None
        self.mongos_db             = None
        self._balancer_state_start = None
        self.restored              = False

        # Get a DB connection
        try:
            if isinstance(self.db, DB):
                self.connection = self.db.connection()
                if not self.db.is_mongos() and not self.db.is_configsvr():
                    raise DBOperationError('MongoDB connection is not to a mongos or configsvr!')
            else:
                raise Error("'db' field is not an instance of class: 'DB'!")
        except Exception, e:
            logging.fatal("Could not get DB connection! Error: %s" % e)
            raise DBOperationError(e) 

def InsortBrowser(cls, browsers, browser):
        """Insert a browser, in-place, into a sorted list of browsers.

        Args:
            browsers: a list of strings (e.g. ['iPhone 3.1', 'Safari 4.1'])
            browser: a list of strings
        """
        browser_key = cls.BrowserKey(browser)
        low, high = 0, len(browsers)
        while low < high:
            mid = (low + high) / 2
            if browser_key < cls.BrowserKey(browsers[mid]):
                high = mid
            else:
                low = mid + 1
        if not hasattr(browsers, 'insert'):
            logging.fatal('Unexpected browsers list: %s', browsers)
        browsers.insert(low, browser) 

def get_args_for_config(config_name):
  configs = config_name.split('.')
  type = configs[0]
  config_name = '.'.join(configs[1:])
  if type == 'cmp':
    args = config_cmp.get_args_for_config(config_name)
    args.setup_to_run = cmp.setup_to_run
    args.setup_train_step_kwargs = cmp.setup_train_step_kwargs

  elif type == 'bl':
    args = config_vision_baseline.get_args_for_config(config_name)
    args.setup_to_run = vision_baseline_lstm.setup_to_run
    args.setup_train_step_kwargs = vision_baseline_lstm.setup_train_step_kwargs

  else:
    logging.fatal('Unknown type: {:s}'.format(type))
  return args 

def __init__(self, robot, env, task_params, category_list=None,
               building_name=None, flip=False, logdir=None,
               building_loader=None, r_obj=None):
    tt = utils.Timer()
    tt.tic()
    Building.__init__(self, building_name, robot, env, category_list,
                      small=task_params.toy_problem, flip=flip, logdir=logdir,
                      building_loader=building_loader)

    self.set_r_obj(r_obj)
    self.task_params = task_params
    self.task = None
    self.episode = None
    self._preprocess_for_task(self.task_params.building_seed)
    if hasattr(self.task_params, 'map_scales'):
      self.task.scaled_maps = resize_maps(
          self.traversible.astype(np.float32)*1, self.task_params.map_scales,
          self.task_params.map_resize_method)
    else:
      logging.fatal('VisualNavigationEnv does not support scale_f anymore.')
    self.task.readout_maps_scaled = resize_maps(
      self.traversible.astype(np.float32)*1,
      self.task_params.readout_maps_scales,
      self.task_params.map_resize_method)
    tt.toc(log_at=1, log_str='VisualNavigationEnv __init__: ') 

def process_arch_str(args, arch_str):
  # This function modifies args.
  args.arch, args.mapper_arch = get_default_cmp_args()

  arch_vars = get_arch_vars(arch_str)

  args.navtask.task_params.outputs.ego_maps = True
  args.navtask.task_params.outputs.ego_goal_imgs = True
  args.navtask.task_params.outputs.egomotion = True
  args.navtask.task_params.toy_problem = False

  if arch_vars.var1 == 'lmap':
    args = process_arch_learned_map(args, arch_vars)

  elif arch_vars.var1 == 'pmap':
    args = process_arch_projected_map(args, arch_vars)

  else:
    logging.fatal('arch_vars.var1 should be lmap or pmap, but is %s', arch_vars.var1)
    assert(False)

  return args 

def combine_setup(name, combine_type, embed_img, embed_goal, num_img_neuorons=None,
                  num_goal_neurons=None):
  with tf.name_scope(name + '_' + combine_type):
    if combine_type == 'add':
      # Simple concat features from goal and image
      out = embed_img + embed_goal

    elif combine_type == 'multiply':
      # Multiply things together
      re_embed_img = tf.reshape(
          embed_img, shape=[-1, num_img_neuorons / num_goal_neurons,
                            num_goal_neurons])
      re_embed_goal = tf.reshape(embed_goal, shape=[-1, num_goal_neurons, 1])
      x = tf.matmul(re_embed_img, re_embed_goal, transpose_a=False, transpose_b=False)
      out = slim.flatten(x)
    elif combine_type == 'none' or combine_type == 'imgonly':
      out = embed_img
    elif combine_type == 'goalonly':
      out = embed_goal
    else:
      logging.fatal('Undefined combine_type: %s', combine_type)
  return out 

def Send(self, path, params, method='POST', json_response=True):
    # Drop parameters with value=None. Otherwise, the string 'None' gets sent.
    rpc_params = dict((str(k), v) for k, v in params.items() if v is not None)
    logging.info(
        'http://%s%s%s', self.host, path, rpc_params and '?%s' % '&'.join(
        ['%s=%s' % (k, v) for k, v in sorted(rpc_params.items())]) or '')
    # "payload=None" would a GET instead a POST.
    if method == 'GET':
      response_data = self.rpc_server.Send(path, payload=None, **rpc_params)
    else:
      response_data = self.rpc_server.Send(
          path, payload=urllib.urlencode(rpc_params))
    if response_data.startswith('bailing'):
      logging.fatal(response_data)
      raise RuntimeError
    elif json_response:
      return simplejson.loads(response_data)
    else:
      return response_data 

def stop_balancer(self):
        logging.info("Stopping the balancer and waiting a max of %i sec" % self.balancer_wait_secs)
        wait_cnt = 0
        self.timer.start(self.timer_name)
        self.set_balancer(False)
        while wait_cnt < self.balancer_wait_secs:
            if self.check_balancer_running():
                wait_cnt += self.balancer_sleep
                logging.info("Balancer is still running, sleeping for %i sec(s)" % self.balancer_sleep)
                sleep(self.balancer_sleep)
            else:
                self.timer.stop(self.timer_name)
                logging.info("Balancer stopped after %.2f seconds" % self.timer.duration(self.timer_name))
                return
        logging.fatal("Could not stop balancer %s!" % self.db.uri)
        raise DBOperationError("Could not stop balancer %s" % self.db.uri) 

def get_configdb_hosts(self):
        try:
            cmdlineopts = self.db.admin_command("getCmdLineOpts")
            config_string = None
            if cmdlineopts.get('parsed').get('configdb'):
                config_string = cmdlineopts.get('parsed').get('configdb')
            elif cmdlineopts.get('parsed').get('sharding').get('configDB'):
                config_string = cmdlineopts.get('parsed').get('sharding').get('configDB')

            if config_string:
                return MongoUri(config_string, 27019)
            elif self.db.is_configsvr():
                return self.db.uri
            else:
                logging.fatal("Unable to locate config servers for %s!" % self.db.uri)
                raise OperationError("Unable to locate config servers for %s!" % self.db.uri)
        except Exception, e:
            raise OperationError(e) 

def get_config_server(self, force=False):
        if force or not self.config_server:
            configdb_uri = self.get_configdb_hosts()
            try:
                logging.info("Found sharding config server: %s" % configdb_uri)
                if self.db.uri.hosts() == configdb_uri.hosts():
                    self.config_db = self.db
                    logging.debug("Re-using seed connection to config server(s)")
                else:
                    self.config_db = DB(configdb_uri, self.config, True)
                if not self.config_db.is_replset():
                    raise OperationError("configsvrs must have replication enabled")
                self.config_server = Replset(self.config, self.config_db)
            except Exception, e:
                logging.fatal("Unable to locate config servers using %s: %s!" % (self.db.uri, e))
                raise OperationError(e) 

def __init__(self, manager, config, timer, base_dir, backup_dir, tailed_oplogs, backup_oplogs):
        super(Resolver, self).__init__(self.__class__.__name__, manager, config, timer, base_dir, backup_dir)
        self.tailed_oplogs = tailed_oplogs
        self.backup_oplogs = backup_oplogs

        self.compression_supported = ['none', 'gzip']
        self.resolver_summary      = {}
        self.resolver_state        = {}

        self.running   = False
        self.stopped   = False
        self.completed = False
        self._pool     = None
        self._pooled   = []
        self._results  = {}

        self.threads(self.config.oplog.resolver.threads)

        try:
            self._pool = Pool(processes=self.threads())
        except Exception, e:
            logging.fatal("Could not start oplog resolver pool! Error: %s" % e)
            raise Error(e) 

def combine_setup(name, combine_type, embed_img, embed_goal, num_img_neuorons=None,
                  num_goal_neurons=None):
  with tf.name_scope(name + '_' + combine_type):
    if combine_type == 'add':
      # Simple concat features from goal and image
      out = embed_img + embed_goal

    elif combine_type == 'multiply':
      # Multiply things together
      re_embed_img = tf.reshape(
          embed_img, shape=[-1, num_img_neuorons / num_goal_neurons,
                            num_goal_neurons])
      re_embed_goal = tf.reshape(embed_goal, shape=[-1, num_goal_neurons, 1])
      x = tf.matmul(re_embed_img, re_embed_goal, transpose_a=False, transpose_b=False)
      out = slim.flatten(x)
    elif combine_type == 'none' or combine_type == 'imgonly':
      out = embed_img
    elif combine_type == 'goalonly':
      out = embed_goal
    else:
      logging.fatal('Undefined combine_type: %s', combine_type)
  return out 

def init_display(self, width, height, fov, z_near, z_far, rgb_shader,
                   d_shader):
    self.init_renderer_egl(width, height)
    dir_path = os.path.dirname(os.path.realpath(__file__))
    if d_shader is not None and rgb_shader is not None:
      logging.fatal('Does not support setting both rgb_shader and d_shader.')
    
    if d_shader is not None:
      assert rgb_shader is None
      shader = d_shader
      self.modality = 'depth'
    
    if rgb_shader is not None:
      assert d_shader is None
      shader = rgb_shader
      self.modality = 'rgb'
    
    self.create_shaders(os.path.join(dir_path, shader+'.vp'),
                        os.path.join(dir_path, shader + '.fp'))
    aspect = width*1./(height*1.)
    self.set_camera(fov, z_near, z_far, aspect) 

def process_arch_str(args, arch_str):
  # This function modifies args.
  args.arch, args.mapper_arch = get_default_cmp_args()

  arch_vars = get_arch_vars(arch_str)

  args.navtask.task_params.outputs.ego_maps = True
  args.navtask.task_params.outputs.ego_goal_imgs = True
  args.navtask.task_params.outputs.egomotion = True
  args.navtask.task_params.toy_problem = False

  if arch_vars.var1 == 'lmap':
    args = process_arch_learned_map(args, arch_vars)

  elif arch_vars.var1 == 'pmap':
    args = process_arch_projected_map(args, arch_vars)

  else:
    logging.fatal('arch_vars.var1 should be lmap or pmap, but is %s', arch_vars.var1)
    assert(False)

  return args 

def __init__(self, robot, env, task_params, category_list=None,
               building_name=None, flip=False, logdir=None,
               building_loader=None, r_obj=None):
    tt = utils.Timer()
    tt.tic()
    Building.__init__(self, building_name, robot, env, category_list,
                      small=task_params.toy_problem, flip=flip, logdir=logdir,
                      building_loader=building_loader)

    self.set_r_obj(r_obj)
    self.task_params = task_params
    self.task = None
    self.episode = None
    self._preprocess_for_task(self.task_params.building_seed)
    if hasattr(self.task_params, 'map_scales'):
      self.task.scaled_maps = resize_maps(
          self.traversible.astype(np.float32)*1, self.task_params.map_scales,
          self.task_params.map_resize_method)
    else:
      logging.fatal('VisualNavigationEnv does not support scale_f anymore.')
    self.task.readout_maps_scaled = resize_maps(
      self.traversible.astype(np.float32)*1,
      self.task_params.readout_maps_scales,
      self.task_params.map_resize_method)
    tt.toc(log_at=1, log_str='VisualNavigationEnv __init__: ') 

def combine_setup(name, combine_type, embed_img, embed_goal, num_img_neuorons=None,
                  num_goal_neurons=None):
  with tf.name_scope(name + '_' + combine_type):
    if combine_type == 'add':
      # Simple concat features from goal and image
      out = embed_img + embed_goal

    elif combine_type == 'multiply':
      # Multiply things together
      re_embed_img = tf.reshape(
          embed_img, shape=[-1, num_img_neuorons / num_goal_neurons,
                            num_goal_neurons])
      re_embed_goal = tf.reshape(embed_goal, shape=[-1, num_goal_neurons, 1])
      x = tf.matmul(re_embed_img, re_embed_goal, transpose_a=False, transpose_b=False)
      out = slim.flatten(x)
    elif combine_type == 'none' or combine_type == 'imgonly':
      out = embed_img
    elif combine_type == 'goalonly':
      out = embed_goal
    else:
      logging.fatal('Undefined combine_type: %s', combine_type)
  return out 

def init_display(self, width, height, fov, z_near, z_far, rgb_shader,
                   d_shader):
    self.init_renderer_egl(width, height)
    dir_path = os.path.dirname(os.path.realpath(__file__))
    if d_shader is not None and rgb_shader is not None:
      logging.fatal('Does not support setting both rgb_shader and d_shader.')
    
    if d_shader is not None:
      assert rgb_shader is None
      shader = d_shader
      self.modality = 'depth'
    
    if rgb_shader is not None:
      assert d_shader is None
      shader = rgb_shader
      self.modality = 'rgb'
    
    self.create_shaders(os.path.join(dir_path, shader+'.vp'),
                        os.path.join(dir_path, shader + '.fp'))
    aspect = width*1./(height*1.)
    self.set_camera(fov, z_near, z_far, aspect) 

def process_arch_str(args, arch_str):
  # This function modifies args.
  args.arch, args.mapper_arch = get_default_cmp_args()

  arch_vars = get_arch_vars(arch_str)

  args.navtask.task_params.outputs.ego_maps = True
  args.navtask.task_params.outputs.ego_goal_imgs = True
  args.navtask.task_params.outputs.egomotion = True
  args.navtask.task_params.toy_problem = False

  if arch_vars.var1 == 'lmap':
    args = process_arch_learned_map(args, arch_vars)

  elif arch_vars.var1 == 'pmap':
    args = process_arch_projected_map(args, arch_vars)

  else:
    logging.fatal('arch_vars.var1 should be lmap or pmap, but is %s', arch_vars.var1)
    assert(False)

  return args 

def __init__(self, robot, env, task_params, category_list=None,
               building_name=None, flip=False, logdir=None,
               building_loader=None, r_obj=None):
    tt = utils.Timer()
    tt.tic()
    Building.__init__(self, building_name, robot, env, category_list,
                      small=task_params.toy_problem, flip=flip, logdir=logdir,
                      building_loader=building_loader)

    self.set_r_obj(r_obj)
    self.task_params = task_params
    self.task = None
    self.episode = None
    self._preprocess_for_task(self.task_params.building_seed)
    if hasattr(self.task_params, 'map_scales'):
      self.task.scaled_maps = resize_maps(
          self.traversible.astype(np.float32)*1, self.task_params.map_scales,
          self.task_params.map_resize_method)
    else:
      logging.fatal('VisualNavigationEnv does not support scale_f anymore.')
    self.task.readout_maps_scaled = resize_maps(
      self.traversible.astype(np.float32)*1,
      self.task_params.readout_maps_scales,
      self.task_params.map_resize_method)
    tt.toc(log_at=1, log_str='VisualNavigationEnv __init__: ') 

def get_args_for_config(config_name):
  configs = config_name.split('.')
  type = configs[0]
  config_name = '.'.join(configs[1:])
  if type == 'cmp':
    args = config_cmp.get_args_for_config(config_name)
    args.setup_to_run = cmp.setup_to_run
    args.setup_train_step_kwargs = cmp.setup_train_step_kwargs

  elif type == 'bl':
    args = config_vision_baseline.get_args_for_config(config_name)
    args.setup_to_run = vision_baseline_lstm.setup_to_run
    args.setup_train_step_kwargs = vision_baseline_lstm.setup_train_step_kwargs

  else:
    logging.fatal('Unknown type: {:s}'.format(type))
  return args 

def create_forward_rnn(self):
        """
        Create the forward-only RNN

        Parameters
        -------
        :return: the logits
        """
        if self.rnn_created:
            logging.fatal("Trying to create the language RNN but it is already.")

        # Set placeholders for input
        self.inputs_ph = tf.placeholder(tf.float32, shape=[self.max_input_seq_length, None, self.input_dim],
                                        name="inputs_ph")

        self.input_seq_lengths_ph = tf.placeholder(tf.int32, shape=[None], name="input_seq_lengths_ph")

        # Build the RNN
        self.global_step, logits, self.prediction, self.rnn_keep_state_op, self.rnn_state_zero_op, \
            _, _, self.rnn_tuple_state = self._build_base_rnn(self.inputs_ph, self.input_seq_lengths_ph, True)

        # Add the saving and restore operation
        self.saver_op = self._add_saving_op()

        return logits 

def create_forward_rnn(self):
        """
        Create the forward-only RNN

        Parameters
        -------
        :return: the logits
        """
        if self.rnn_created:
            logging.fatal("Trying to create the acoustic RNN but it is already.")

        # Set placeholders for input
        self.inputs_ph = tf.placeholder(tf.float32, shape=[self.max_input_seq_length, None, self.input_dim],
                                        name="inputs_ph")

        self.input_seq_lengths_ph = tf.placeholder(tf.int32, shape=[None], name="input_seq_lengths_ph")

        # Build the RNN
        self.global_step, logits, self.prediction, self.rnn_keep_state_op, self.rnn_state_zero_op,\
            _, _, self.rnn_tuple_state = self._build_base_rnn(self.inputs_ph, self.input_seq_lengths_ph, True)

        # Add the saving and restore operation
        self.saver_op = self._add_saving_op()

        return logits 

def redis(self):
        """
        Lazy-loaded redis connection
        """
        from redis import StrictRedis
        import redis.exceptions

        try:
            url = os.environ.get('REDIS_CONNECTION_URL',
                                 'redis://localhost:6379/0')
            conn = StrictRedis.from_url(url)
            conn.ping()
        except redis.exceptions.ConnectionError:
            logging.fatal("Redis server is not running")
            raise
        return conn 

def check_config():
    if not (
            cfg['login_token'] and
            cfg['domain'] and
            cfg['sub_domain']):
        logging.fatal('config error: need login info')
        exit()
    try:
        if not(int(cfg["interval"])):
            logging.fatal('interval error')
            exit()
        if not(int(cfg["ip_count"])):
            logging.fatal('ip_count error')            
            exit()
    except:
        logging.fatal('config error')
        exit()
    logging.info('config checked') 

def value(self, v):
    if v in _CPP_LEVEL_TO_NAMES:
      # --stderrthreshold also accepts numberic strings whose values are
      # Abseil C++ log levels.
      cpp_value = int(v)
      v = _CPP_LEVEL_TO_NAMES[v]  # Normalize to strings.
    elif v.lower() in _CPP_NAME_TO_LEVELS:
      v = v.lower()
      if v == 'warn':
        v = 'warning'  # Use 'warning' as the canonical name.
      cpp_value = int(_CPP_NAME_TO_LEVELS[v])
    else:
      raise ValueError(
          '--stderrthreshold must be one of (case-insensitive) '
          "'debug', 'info', 'warning', 'error', 'fatal', "
          "or '0', '1', '2', '3', not '%s'" % v)

    self._value = v 

def init_display(self, width, height, fov, z_near, z_far, rgb_shader,
                   d_shader):
    self.init_renderer_egl(width, height)
    dir_path = os.path.dirname(os.path.realpath(__file__))
    if d_shader is not None and rgb_shader is not None:
      logging.fatal('Does not support setting both rgb_shader and d_shader.')
    
    if d_shader is not None:
      assert rgb_shader is None
      shader = d_shader
      self.modality = 'depth'
    
    if rgb_shader is not None:
      assert d_shader is None
      shader = rgb_shader
      self.modality = 'rgb'
    
    self.create_shaders(os.path.join(dir_path, shader+'.vp'),
                        os.path.join(dir_path, shader + '.fp'))
    aspect = width*1./(height*1.)
    self.set_camera(fov, z_near, z_far, aspect) 

def __init__(self, bucket_name, region, access_key, secret_key, file_name, key_name, byte_count, target_bandwidth, multipart_id=None,
                 multipart_num=None, multipart_parts=None, multipart_offset=None, retries=5, secure=True, retry_sleep_secs=1):
        self.bucket_name      = bucket_name
        self.region           = region
        self.access_key       = access_key
        self.secret_key       = secret_key
        self.file_name        = file_name
        self.key_name         = key_name
        self.byte_count       = byte_count
        self.target_bandwidth = target_bandwidth
        self.multipart_id     = multipart_id
        self.multipart_num    = multipart_num
        self.multipart_parts  = multipart_parts
        self.multipart_offset = multipart_offset
        self.retries          = retries
        self.secure           = secure
        self.retry_sleep_secs = retry_sleep_secs
        self.do_stop          = False

        if self.target_bandwidth is not None:
            logging.debug("Target bandwidth: %.2f" % self.target_bandwidth)
        progress_key_name = self.short_key_name(self.key_name)
        if self.multipart_num and self.multipart_parts:
            progress_key_name = "%s %d/%d" % (self.short_key_name(self.key_name), self.multipart_num, self.multipart_parts)
        self._progress    = S3ProgressBar(progress_key_name, max=float(self.byte_count / 1024.00 / 1024.00))
        self._last_bytes  = None
        self._last_status_ts = None

        try:
            self.s3_conn = S3Session(self.region, self.access_key, self.secret_key, self.bucket_name, self.secure, self.retries)
            self.bucket  = self.s3_conn.get_bucket(self.bucket_name)
        except Exception, e:
            logging.fatal("Could not get AWS S3 connection to bucket %s! Error: %s" % (self.bucket_name, e))
            raise OperationError("Could not get AWS S3 connection to bucket") 

def evaluate(hyper_params):
    if hyper_params["test_dataset_dirs"] is None:
        logging.fatal("Setting test_dataset_dirs in config file is mandatory for evaluation mode")
        return

    # Load the test set data
    data_processor = dataprocessor.DataProcessor(hyper_params["test_dataset_dirs"])
    test_set = data_processor.get_dataset()

    logging.info("Using %d size of test set", len(test_set))

    if len(test_set) == 0:
        logging.fatal("No files in test set during an evaluation mode")
        return

    with tf.Session() as sess:
        # create model
        model = AcousticModel(hyper_params["num_layers"], hyper_params["hidden_size"], hyper_params["batch_size"],
                              hyper_params["max_input_seq_length"], hyper_params["max_target_seq_length"],
                              hyper_params["input_dim"], hyper_params["batch_normalization"],
                              hyper_params["char_map_length"])

        model.create_forward_rnn()
        model.initialize(sess)
        model.restore(sess, hyper_params["checkpoint_dir"] + "/acoustic/")

        wer, cer = model.evaluate_full(sess, test_set, hyper_params["max_input_seq_length"],
                                       hyper_params["signal_processing"], hyper_params["char_map"])
        print("Resulting WER : {0:.3g} %".format(wer))
        print("Resulting CER : {0:.3g} %".format(cer))
        return