Python logging.exception() Examples

def send_json_msg(self, rk, body, **kwargs):
		if self.connection.is_open:
			try:
				logging.debug("Sending message to manager")
				properties = pika.BasicProperties(content_type='application/json')
				self.channel.basic_publish(exchange=utils.EXCHANGE, routing_key=rk, body=json.dumps(body), properties=properties, **kwargs)
				return True
			except Exception as e:
				logging.exception("Error while sending data to queue:\n%s" % e)
				return False
		else:
			logging.error("Can't send message to manager")
			message = {"rk":rk, "body": body, "kwargs": kwargs, "json": True}
			if message not in self.message_queue: # could happen if we have another disconnect when we try to clear the message queue
				self.message_queue.append(message)
				logging.info("Added message to message queue")
			else:
				logging.debug("Message already in queue")

			return False
	
	# Try to resend the messages which couldn't be sent before 

def pool(self):
        result = self.parse_html(self.host)
        try:
            with concurrent.futures.ThreadPoolExecutor(max_workers=30) as executor:
                futures = [executor.submit(self.parse_html, i) for i in result]
                for future in concurrent.futures.as_completed(futures, timeout=3):
                    future.result()
        except (EOFError, concurrent.futures._base.TimeoutError):
            pass
        except Exception as e:
            logging.exception(e)
        
        jslink = JsLeaks().pool(self.js)
        
        self.result.extend(jslink)
        self.result = list(set(self.result))
        
        for i in self.result:
            console('Crawl', self.host, i + '\n')
        
        Sqldb(self.dbname).get_crawl(self.domain, self.result) 

def handle_incoming(modules, db, event):
    """Handle all incoming messages"""
    logging.debug("Incoming message!")
    message = utils.censor(event.message)
    blacklist_chats = db.get(__name__, "blacklist_chats", [])
    whitelist_chats = db.get(__name__, "whitelist_chats", [])
    whitelist_modules = db.get(__name__, "whitelist_modules", [])
    if utils.get_chat_id(message) in blacklist_chats or (whitelist_chats and utils.get_chat_id(message) not in
                                                         whitelist_chats) or message.from_id is None:
        logging.debug("Message is blacklisted")
        return
    for func in modules.watchers:
        if str(utils.get_chat_id(message)) + "." + func.__self__.__module__ in blacklist_chats:
            logging.debug("Command is blacklisted in chat")
            return
        if whitelist_modules and not (str(utils.get_chat_id(message)) + "."
                                      + func.__self__.__module__ in whitelist_modules):
            logging.debug("Command is not whitelisted in chat")
            return
        try:
            await func(message)
        except Exception:
            logging.exception("Error running watcher") 

def send_msg(self, rk, body, **kwargs):
		if self.connection.is_open:
			try:
				logging.debug("Sending message to manager")
				self.channel.basic_publish(exchange=utils.EXCHANGE, routing_key=rk, body=body, **kwargs)
				return True
			except Exception as e:
				logging.exception("Error while sending data to queue:\n%s" % e)
				return False
		else:
			logging.error("Can't send message to manager")
			message = {"rk":rk, "body": body, "kwargs": kwargs, "json": False}
			if message not in self.message_queue: # could happen if we have another disconnect when we try to clear the message queue
				self.message_queue.append(message)
				logging.info("Added message to message queue")
			else:
				logging.debug("Message already in queue")

			return False
	
	# sends a message to the manager 

def got_init_config(self, ch, method, properties, body):
		logging.info("Received intitial config %r" % (body))
		if self.corr_id == properties.correlation_id: #we got the right config
			try: #TODO: add check if response is empty...
				new_conf = json.loads(body)
				new_conf["rabbitmq"] = config.get("rabbitmq")
			except Exception as e:
				logging.exception("Wasn't able to read JSON config from manager:\n%s" % e)
				time.sleep(60) #sleep for X seconds and then ask again
				self.fetch_init_config()
				return
		
			logging.info("Trying to apply config and reconnect")
			self.apply_config(new_conf)
			self.connection_cleanup()
			self.connect() #hope this is the right spot
			logging.info("Initial config activated")
			self.start()
		else:
			logging.info("This config isn't meant for us")
	
	# Create a zip of all the files which were collected while actions were executed 

def custom_exception_handler(exc, context):
    # give more context on the error since DRF masks it as Not Found
    if isinstance(exc, Http404):
        set_rollback()
        return Response(str(exc), status=status.HTTP_404_NOT_FOUND)

    # Call REST framework's default exception handler after specific 404 handling,
    # to get the standard error response.
    response = exception_handler(exc, context)

    # No response means DRF couldn't handle it
    # Output a generic 500 in a JSON format
    if response is None:
        logging.exception('Uncaught Exception', exc_info=exc)
        set_rollback()
        return Response({'detail': 'Server Error'}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

    # log a few different types of exception instead of using APIException
    if isinstance(exc, (DeisException, ServiceUnavailable, HealthcheckException)):
        logging.exception(exc.__cause__, exc_info=exc)

    return response 

def do_execute(self):

        for step in self.steps:
            try:
                passed = step.do_execute()
                if passed:
                    self.status = self.success_status
                    logging.debug('Verification step %s passed', self.__class__.__name__)
                else:
                    self.status = StepStatus.failed
                    logging.error('Verification step %s failed!', self.__class__.__name__)
                    break
            except Exception:
                logging.exception('caught exception executing step %s', self.__class__.__name__)
                self.status = StepStatus.failed
                break
        return self.status == StepStatus.done or self.status == StepStatus.passed 

def create_urls(self):
        try:
            cursor = self.conn.cursor()
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS urls (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                time varchar(255),
                domain varchar(255) DEFAULT '',
                title varchar(255) DEFAULT '',
                url varchar(255) DEFAULT '',
                contype varchar(255) DEFAULT '',
                rsp_len varchar(255) DEFAULT '',
                rsp_code varchar(255) DEFAULT '',
                md5 varchar(40) UNIQUE
                )
                """)
        except Exception as e:
            logging.exception(e) 

def scan(self, url):
        url = verify(url)
        try:
            r = self.session.get(url,
                                 timeout=self.timeout,
                                 headers=self.headers,
                                 verify=False,
                                 stream=True,
                                 allow_redirects=False)
            return r

        except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, requests.exceptions.Timeout,
                requests.exceptions.SSLError, requests.exceptions.ConnectionError, ssl.SSLError, AttributeError,
                ConnectionRefusedError, socket.timeout, urllib3.exceptions.ReadTimeoutError, OpenSSL.SSL.WantReadError,
                urllib3.exceptions.DecodeError, requests.exceptions.ContentDecodingError):
            pass
        except Exception as e:
            logging.exception(e) 

def request(self, url, method, data=None, headers=None):
        url = verify(url)
        try:
            if method == 'get':
                r = self.session.get(url, timeout=self.timeout, headers=headers, verify=False, allow_redirects=True)
                return r
            else:
                r = self.session.post(url,
                                      data=data,
                                      timeout=self.timeout,
                                      headers=headers,
                                      verify=False,
                                      allow_redirects=False)
                return r
        except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, requests.exceptions.Timeout,
                requests.exceptions.SSLError, requests.exceptions.ConnectionError, ssl.SSLError, AttributeError,
                ConnectionRefusedError, socket.timeout, urllib3.exceptions.ReadTimeoutError, OpenSSL.SSL.WantReadError,
                urllib3.exceptions.DecodeError, requests.exceptions.ContentDecodingError):
            pass
        except Exception as e:
            logging.exception(e) 

def run(self):
        scripts = []
        try:
            for _ in glob.glob('script/*.py'):
                script_name = os.path.basename(_).replace('.py', '')
                vuln = importlib.import_module('script.%s' % script_name)
                scripts.append(vuln)
            # 随机打乱脚本加载顺序
            random.shuffle(scripts)
            with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
                vulns = {executor.submit(self.vuln, script): script for script in scripts}
                for future in concurrent.futures.as_completed(vulns, timeout=3):
                    future.result()
            self.out = list(filter(None, self.out))
            for i in self.out:
                console('Vuln', self.ip, i + '\n')

            Sqldb(self.dbname).get_vuln(self.ip, self.out)
        except (EOFError, concurrent.futures._base.TimeoutError):
            pass
        except Exception as e:
            logging.exception(e) 

def parse_ip(host):
    host = parse_host(host)
    # 根据domain得到ip 例如www.xxx.com 得到 x.x.x.x
    try:
        resolver = dns.resolver.Resolver()
        resolver.nameservers = ['1.1.1.1', '8.8.8.8']
        a = resolver.query(host, 'A')
        for i in a.response.answer:
            for j in i.items:
                if hasattr(j, 'address'):
                    if not re.search(r'1\.1\.1\.1|8\.8\.8\.8|127\.0\.0\.1|114\.114\.114\.114|0\.0\.0\.0', j.address):
                        return j.address
    except dns.resolver.NoAnswer:
        pass
    except Exception as e:
        logging.exception(e)
    return host 

def osdetect(ip):
    # sys.stdout.write(Bcolors.RED + "\nOS：\n" + Bcolors.ENDC)
    nm = nmap.PortScanner()
    try:
        result = nm.scan(hosts=ip, arguments='-sS -O -vv -n -T4 -p 80,22,443')
        for k, v in result.get('scan').items():
            if v.get('osmatch'):
                for i in v.get('osmatch'):
                    console('OSdetect', ip, i.get('name') + '\n')
                    return i.get('name')
            else:
                break
    except (xml.etree.ElementTree.ParseError, nmap.nmap.PortScannerError):
        pass
    except Exception as e:
        console('OSdetect', ip, 'None\n')
        logging.exception(e) 

def checkwaf(url):
    result = 'NoWAF'
    host = parse_host(url)

    if not iscdn(host):
        return 'CDN IP'

    try:
        req = Requests()
        r = req.get(url)
        result = verify(r.headers, r.text)
        if result == 'NoWAF':
            for i in payload:
                r = req.get(url + i)
                result = verify(r.headers, r.text)
                if result != 'NoWAF':
                    return result
        else:
            return result
    except (UnboundLocalError, AttributeError):
        pass
    except Exception as e:
        logging.exception(e) 

def jsparse(self, r):
        try:
            html = etree.HTML(r.text)
            result = html.xpath('//script/@src')
            for i in result:
                if not re.search(
                    r'jquery|bootstrap|adsbygoogle|angular|javascript|#|vue|react|51.la/=|map\.baidu\.com|canvas|cnzz\.com|slick\.js|autofill-event\.js|tld\.js|clipboard|Chart\.js',
                    i):
                    if '://' not in i:
                        i = re.sub(r'^/|^\.\./', '', i)
                        i = self.host + '/' + i
                    self.js.append(i)
        except (AttributeError, AttributeError, ValueError):
            pass
        except Exception as e:
            logging.exception(e) 

def geoip(ipaddr):
    # 获取IP地理位置
    try:
        reader = geoip2.database.Reader('data/GeoLite2-City.mmdb')
        response = reader.city(ipaddr)
        country = response.country.names["zh-CN"]
        site = response.subdivisions.most_specific.names.get("zh-CN")
        if not site:
            site = ''
        city = response.city.names.get("zh-CN")
        if not city:
            city = ''
        address = '{} {} {}'.format(country, site, city)
    except FileNotFoundError:
        address = 'Geoip File Not Found'
    except (KeyError, geoip2.errors.AddressNotFoundError):
        address = 'Address Not In Databases'
    except Exception as e:
        logging.exception(e)
        address = 'None'
    console('GeoIP', ipaddr, 'Address: {}\n'.format(address))
    console('GeoIP', ipaddr, 'Ipaddr: {}\n'.format(ipaddr))
    return address 

def run_task(task, topic):
    if task.startswith(topic_prefix):
        task = task[len(topic_prefix):]

    topic = Topic(topic if topic.startswith(topic_prefix) else topic_prefix + topic)
    try:
        payload = get_value(task)
        is_seq = isinstance(payload, list) or isinstance(payload, dict)
        if is_seq and not topic.is_multitopic():
            raise Exception("Result of task '" + task + "' has several values but topic doesn't contain '*' char")
        if isinstance(payload, list):
            for i, v in enumerate(payload):
                subtopic = topic.get_subtopic(str(i))
                mqttc.publish(subtopic, payload_as_string(v), qos=qos, retain=retain)
        elif isinstance(payload, dict):
            for key in payload:
                subtopic = topic.get_subtopic(str(key))
                v = payload[key]
                mqttc.publish(subtopic, payload_as_string(v), qos=qos, retain=retain)
        else:
            mqttc.publish(topic.get_topic(), payload_as_string(payload), qos=qos, retain=retain)
    except Exception as ex:
        mqttc.publish(topic.get_error_topic(), str(ex), qos=qos, retain=retain)
        logging.exception(task + ": " + str(ex)) 

def event_proc(self, event):
        etype = event["type"]
        switch = {
            "m.room.member": self.parse_membership,
            "m.room.message": self.parse_msg
        }
        try:
            switch[etype](event)
        except KeyError:
            try:
                for p in self.plugins:
                    self.plugins[p].on_event(event, etype)
            except Exception as e:
                log.exception(e)
        except Exception as e:
            log.error("Couldn't process event: %s", e) 

def do_POST(self, service=""):
        log.debug("NebHookServer: Plugin=%s : Incoming request from %s",
                  service, request.remote_addr)
        if service.split("/")[0] not in self.plugin_mappings:
            return ("", 404, {})

        plugin = self.plugin_mappings[service.split("/")[0]]

        try:
            # tuple (body, status_code, headers)
            response = plugin.on_receive_webhook(
                request.url,
                request.get_data(),
                request.remote_addr,
                request.headers
            )
            if response:
                return response
            return ("", 200, {})
        except Exception as e:
            log.exception(e)
            return ("", 500, {}) 

def connect(self):
        """ Establish netconf session """

        if self.handle is not None:
            return True

        try:
            # timeout is configurable as environment variable
            timeout = int(os.getenv("NCCLIENT_TIMEOUT", 90))
            self.handle = manager.connect(host=self.host,
                                          port=self.port,
                                          username=self.username,
                                          password=self.password,
                                          device_params=self.params,
                                          unknown_host_cb=self._unknown_host_cb,
                                          look_for_keys=False,
                                          timeout=timeout)
        except:
            logging.exception("Failed to create netconf session:")
            self.handle = None
            return False

        logging.debug("Connected: %s" % self.__str__())
        return True 

def call_method(method, data):
  data = json.dumps(data)
  try:
    result = urlfetch.fetch(
        BASE_URL.format(token=TOKEN, method=method),
        payload=data,
        method=urlfetch.POST,
        deadline=10,
        headers={'Content-Type': 'application/json'})
  except DeadlineExceededError as e:
    logging.exception(e)
    return None
  if result.status_code == 200:
    return json.loads(result.content)
  else:
    logging.error(result.content)
    return None 

def call_method(method, data):
  data.update({'key': TOKEN})
  data = json.dumps(data)
  try:
    result = urlfetch.fetch(
        BASE_URL.format(method=method, api_key=TOKEN),
        payload=data,
        method=urlfetch.POST,
        deadline=10,
        headers={'Content-Type': 'application/json'})
  except DeadlineExceededError as e:
    logging.exception(e)
    return None
  if result.status_code == 200:
    return json.loads(result.content)
  else:
    logging.error(result.content)
    return None 

def call_method(method, data):
  data.update({'access_token': TOKEN})
  data = urllib.urlencode(data)
  try:
    result = urlfetch.fetch(
        BASE_URL.format(method=method, qs=data),
        method=urlfetch.GET,
        deadline=10)
  except DeadlineExceededError as e:
    logging.exception(e)
    return None
  if result.status_code == 200:
    return json.loads(result.content).get('data')
  else:
    logging.error(result.content)
    return None 

def insert_ticks(ticks):
	c = db.cursor()
	for tick in ticks:
		c.execute(insert_tick_statement, {
			"date": datetime.now(),
			"token": tick["instrument_token"],
			"price": tick["last_price"]})

	logging.info("Inserting ticks to db : {}".format(json.dumps(ticks)))

	try:
		db.commit()
	except Exception:
		db.rollback()
		logging.exception("Couldn't write ticks to db: ") 

def client_ready(self, client):
        """Signal all mods that client_ready()"""
        self.clients += [client]
        for mod in self.created:
            try:
                await mod.client_ready(client)
            except BaseException:
                logging.exception("Failed to send client_ready to compat layer " + repr(mod)) 

def register_all(self, babelfish):
        """Load all modules in the module directory"""
        if self._compat_layer is None:
            from .compat import uniborg
            from . import compat  # Avoid circular import
            self._compat_layer = compat.activate([])
        logging.debug(os.listdir(os.path.join(os.path.dirname(os.path.abspath(__file__)), MODULES_NAME)))
        mods = filter(lambda x: (len(x) > 3 and x[-3:] == ".py" and x[0] != "_"),
                      os.listdir(os.path.join(utils.get_base_dir(), MODULES_NAME)))
        logging.debug(mods)
        for mod in mods:
            try:
                module_name = __package__ + "." + MODULES_NAME + "." + mod[:-3]  # FQN
                logging.debug(module_name)
                logging.debug(os.path.join(utils.get_base_dir(), MODULES_NAME, mod))
                spec = importlib.util.spec_from_file_location(module_name,
                                                              os.path.join(utils.get_base_dir(), MODULES_NAME, mod))
                module = importlib.util.module_from_spec(spec)
                sys.modules[module_name] = module  # Do this early for the benefit of RaphielGang compat layer
                module.borg = uniborg.UniborgClient(module_name)
                spec.loader.exec_module(module)
                module._ = babelfish.gettext
                try:
                    module.register(self.register_module, module_name)
                except TypeError:  # Too many arguments
                    module.register(self.register_module)
            except BaseException:
                logging.exception("Failed to load module %s due to:", mod) 

def send_config_one(self, mod, db, babel=None, skip_hook=False):  # pylint: disable=R0201
        """Send config to single instance"""
        if hasattr(mod, "config"):
            modcfg = db.get(mod.__module__, "__config__", {})
            logging.debug(modcfg)
            for conf in mod.config.keys():
                logging.debug(conf)
                if conf in modcfg.keys():
                    mod.config[conf] = modcfg[conf]
                else:
                    try:
                        mod.config[conf] = os.environ[mod.__module__ + "." + conf]
                        logging.debug("Loaded config key %s from environment", conf)
                    except KeyError:
                        logging.debug("No config value for %s", conf)
                        mod.config[conf] = mod.config.getdef(conf)
            logging.debug(mod.config)
        if hasattr(mod, "strings") and babel is not None:
            mod.strings = mod.strings.copy()  # For users with many accounts with diff. translations
            for key, value in mod.strings.items():
                new = babel.getkey(mod.__module__ + "." + key)
                if new is not False:
                    mod.strings[key] = new
        if skip_hook:
            return
        mod.babel = babel
        try:
            mod.config_complete()
        except Exception:
            logging.exception("Failed to send mod config complete signal") 

def send_ready_one(self, mod, client, db, allclients, core=False):
        mod.allclients = allclients
        try:
            await mod.client_ready(client, db)
        except Exception:
            logging.exception("Failed to send mod init complete signal for %r", mod)
        if not hasattr(mod, "commands"):
            mod.commands = get_commands(mod)

        self.register_commands(mod)
        self.register_watcher(mod) 

def _build_save_container(platform, registry, load_cache) -> Optional[str]:
    """
    Build image for passed platform and upload the cache to the specified S3 bucket
    :param platform: Platform
    :param registry: Docker registry name
    :param load_cache: Load cache before building
    :return: Platform if failed, None otherwise
    """
    docker_tag = build_util.get_docker_tag(platform=platform, registry=registry)

    # Preload cache
    if load_cache:
        load_docker_cache(registry=registry, docker_tag=docker_tag)

    # Start building
    logging.debug('Building %s as %s', platform, docker_tag)
    try:
        # Increase the number of retries for building the cache.
        image_id = build_util.build_docker(docker_binary='docker', platform=platform, registry=registry, num_retries=10, use_cache=True)
        logging.info('Built %s as %s', docker_tag, image_id)

        # Push cache to registry
        _upload_image(registry=registry, docker_tag=docker_tag, image_id=image_id)
        return None
    except Exception:
        logging.exception('Unexpected exception during build of %s', docker_tag)
        return platform
        # Error handling is done by returning the errorous platform name. This is necessary due to
        # Parallel being unable to handle exceptions 

def _get_dockerhub_credentials():  # pragma: no cover
    import boto3
    import botocore
    secret_name = os.environ['DOCKERHUB_SECRET_NAME']
    endpoint_url = os.environ['DOCKERHUB_SECRET_ENDPOINT_URL']
    region_name = os.environ['DOCKERHUB_SECRET_ENDPOINT_REGION']

    session = boto3.Session()
    client = session.client(
        service_name='secretsmanager',
        region_name=region_name,
        endpoint_url=endpoint_url
    )
    try:
        get_secret_value_response = client.get_secret_value(
            SecretId=secret_name
        )
    except botocore.exceptions.ClientError as client_error:
        if client_error.response['Error']['Code'] == 'ResourceNotFoundException':
            logging.exception("The requested secret %s was not found", secret_name)
        elif client_error.response['Error']['Code'] == 'InvalidRequestException':
            logging.exception("The request was invalid due to:")
        elif client_error.response['Error']['Code'] == 'InvalidParameterException':
            logging.exception("The request had invalid params:")
        else:
            raise
    else:
        secret = get_secret_value_response['SecretString']
        secret_dict = json.loads(secret)
        return secret_dict