Python logging.exception() Examples

def pool(self):
        result = self.parse_html(self.host)
        try:
            with concurrent.futures.ThreadPoolExecutor(max_workers=30) as executor:
                futures = [executor.submit(self.parse_html, i) for i in result]
                for future in concurrent.futures.as_completed(futures, timeout=3):
                    future.result()
        except (EOFError, concurrent.futures._base.TimeoutError):
            pass
        except Exception as e:
            logging.exception(e)
        
        jslink = JsLeaks().pool(self.js)
        
        self.result.extend(jslink)
        self.result = list(set(self.result))
        
        for i in self.result:
            console('Crawl', self.host, i + '\n')
        
        Sqldb(self.dbname).get_crawl(self.domain, self.result) 

def send_json_msg(self, rk, body, **kwargs):
		if self.connection.is_open:
			try:
				logging.debug("Sending message to manager")
				properties = pika.BasicProperties(content_type='application/json')
				self.channel.basic_publish(exchange=utils.EXCHANGE, routing_key=rk, body=json.dumps(body), properties=properties, **kwargs)
				return True
			except Exception as e:
				logging.exception("Error while sending data to queue:\n%s" % e)
				return False
		else:
			logging.error("Can't send message to manager")
			message = {"rk":rk, "body": body, "kwargs": kwargs, "json": True}
			if message not in self.message_queue: # could happen if we have another disconnect when we try to clear the message queue
				self.message_queue.append(message)
				logging.info("Added message to message queue")
			else:
				logging.debug("Message already in queue")

			return False
	
	# Try to resend the messages which couldn't be sent before 

def scan(self, url):
        url = verify(url)
        try:
            r = self.session.get(url,
                                 timeout=self.timeout,
                                 headers=self.headers,
                                 verify=False,
                                 stream=True,
                                 allow_redirects=False)
            return r

        except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, requests.exceptions.Timeout,
                requests.exceptions.SSLError, requests.exceptions.ConnectionError, ssl.SSLError, AttributeError,
                ConnectionRefusedError, socket.timeout, urllib3.exceptions.ReadTimeoutError, OpenSSL.SSL.WantReadError,
                urllib3.exceptions.DecodeError, requests.exceptions.ContentDecodingError):
            pass
        except Exception as e:
            logging.exception(e) 

def call_method(method, data):
  data = json.dumps(data)
  try:
    result = urlfetch.fetch(
        BASE_URL.format(token=TOKEN, method=method),
        payload=data,
        method=urlfetch.POST,
        deadline=10,
        headers={'Content-Type': 'application/json'})
  except DeadlineExceededError as e:
    logging.exception(e)
    return None
  if result.status_code == 200:
    return json.loads(result.content)
  else:
    logging.error(result.content)
    return None 

def connect(self):
        """ Establish netconf session """

        if self.handle is not None:
            return True

        try:
            # timeout is configurable as environment variable
            timeout = int(os.getenv("NCCLIENT_TIMEOUT", 90))
            self.handle = manager.connect(host=self.host,
                                          port=self.port,
                                          username=self.username,
                                          password=self.password,
                                          device_params=self.params,
                                          unknown_host_cb=self._unknown_host_cb,
                                          look_for_keys=False,
                                          timeout=timeout)
        except:
            logging.exception("Failed to create netconf session:")
            self.handle = None
            return False

        logging.debug("Connected: %s" % self.__str__())
        return True 

def do_POST(self, service=""):
        log.debug("NebHookServer: Plugin=%s : Incoming request from %s",
                  service, request.remote_addr)
        if service.split("/")[0] not in self.plugin_mappings:
            return ("", 404, {})

        plugin = self.plugin_mappings[service.split("/")[0]]

        try:
            # tuple (body, status_code, headers)
            response = plugin.on_receive_webhook(
                request.url,
                request.get_data(),
                request.remote_addr,
                request.headers
            )
            if response:
                return response
            return ("", 200, {})
        except Exception as e:
            log.exception(e)
            return ("", 500, {}) 

def call_method(method, data):
  data.update({'key': TOKEN})
  data = json.dumps(data)
  try:
    result = urlfetch.fetch(
        BASE_URL.format(method=method, api_key=TOKEN),
        payload=data,
        method=urlfetch.POST,
        deadline=10,
        headers={'Content-Type': 'application/json'})
  except DeadlineExceededError as e:
    logging.exception(e)
    return None
  if result.status_code == 200:
    return json.loads(result.content)
  else:
    logging.error(result.content)
    return None 

def handle_incoming(modules, db, event):
    """Handle all incoming messages"""
    logging.debug("Incoming message!")
    message = utils.censor(event.message)
    blacklist_chats = db.get(__name__, "blacklist_chats", [])
    whitelist_chats = db.get(__name__, "whitelist_chats", [])
    whitelist_modules = db.get(__name__, "whitelist_modules", [])
    if utils.get_chat_id(message) in blacklist_chats or (whitelist_chats and utils.get_chat_id(message) not in
                                                         whitelist_chats) or message.from_id is None:
        logging.debug("Message is blacklisted")
        return
    for func in modules.watchers:
        if str(utils.get_chat_id(message)) + "." + func.__self__.__module__ in blacklist_chats:
            logging.debug("Command is blacklisted in chat")
            return
        if whitelist_modules and not (str(utils.get_chat_id(message)) + "."
                                      + func.__self__.__module__ in whitelist_modules):
            logging.debug("Command is not whitelisted in chat")
            return
        try:
            await func(message)
        except Exception:
            logging.exception("Error running watcher") 

def call_method(method, data):
  data.update({'access_token': TOKEN})
  data = urllib.urlencode(data)
  try:
    result = urlfetch.fetch(
        BASE_URL.format(method=method, qs=data),
        method=urlfetch.GET,
        deadline=10)
  except DeadlineExceededError as e:
    logging.exception(e)
    return None
  if result.status_code == 200:
    return json.loads(result.content).get('data')
  else:
    logging.error(result.content)
    return None 

def send_msg(self, rk, body, **kwargs):
		if self.connection.is_open:
			try:
				logging.debug("Sending message to manager")
				self.channel.basic_publish(exchange=utils.EXCHANGE, routing_key=rk, body=body, **kwargs)
				return True
			except Exception as e:
				logging.exception("Error while sending data to queue:\n%s" % e)
				return False
		else:
			logging.error("Can't send message to manager")
			message = {"rk":rk, "body": body, "kwargs": kwargs, "json": False}
			if message not in self.message_queue: # could happen if we have another disconnect when we try to clear the message queue
				self.message_queue.append(message)
				logging.info("Added message to message queue")
			else:
				logging.debug("Message already in queue")

			return False
	
	# sends a message to the manager 

def got_init_config(self, ch, method, properties, body):
		logging.info("Received intitial config %r" % (body))
		if self.corr_id == properties.correlation_id: #we got the right config
			try: #TODO: add check if response is empty...
				new_conf = json.loads(body)
				new_conf["rabbitmq"] = config.get("rabbitmq")
			except Exception as e:
				logging.exception("Wasn't able to read JSON config from manager:\n%s" % e)
				time.sleep(60) #sleep for X seconds and then ask again
				self.fetch_init_config()
				return
		
			logging.info("Trying to apply config and reconnect")
			self.apply_config(new_conf)
			self.connection_cleanup()
			self.connect() #hope this is the right spot
			logging.info("Initial config activated")
			self.start()
		else:
			logging.info("This config isn't meant for us")
	
	# Create a zip of all the files which were collected while actions were executed 

def run_task(task, topic):
    if task.startswith(topic_prefix):
        task = task[len(topic_prefix):]

    topic = Topic(topic if topic.startswith(topic_prefix) else topic_prefix + topic)
    try:
        payload = get_value(task)
        is_seq = isinstance(payload, list) or isinstance(payload, dict)
        if is_seq and not topic.is_multitopic():
            raise Exception("Result of task '" + task + "' has several values but topic doesn't contain '*' char")
        if isinstance(payload, list):
            for i, v in enumerate(payload):
                subtopic = topic.get_subtopic(str(i))
                mqttc.publish(subtopic, payload_as_string(v), qos=qos, retain=retain)
        elif isinstance(payload, dict):
            for key in payload:
                subtopic = topic.get_subtopic(str(key))
                v = payload[key]
                mqttc.publish(subtopic, payload_as_string(v), qos=qos, retain=retain)
        else:
            mqttc.publish(topic.get_topic(), payload_as_string(payload), qos=qos, retain=retain)
    except Exception as ex:
        mqttc.publish(topic.get_error_topic(), str(ex), qos=qos, retain=retain)
        logging.exception(task + ": " + str(ex)) 

def geoip(ipaddr):
    # 获取IP地理位置
    try:
        reader = geoip2.database.Reader('data/GeoLite2-City.mmdb')
        response = reader.city(ipaddr)
        country = response.country.names["zh-CN"]
        site = response.subdivisions.most_specific.names.get("zh-CN")
        if not site:
            site = ''
        city = response.city.names.get("zh-CN")
        if not city:
            city = ''
        address = '{} {} {}'.format(country, site, city)
    except FileNotFoundError:
        address = 'Geoip File Not Found'
    except (KeyError, geoip2.errors.AddressNotFoundError):
        address = 'Address Not In Databases'
    except Exception as e:
        logging.exception(e)
        address = 'None'
    console('GeoIP', ipaddr, 'Address: {}\n'.format(address))
    console('GeoIP', ipaddr, 'Ipaddr: {}\n'.format(ipaddr))
    return address 

def event_proc(self, event):
        etype = event["type"]
        switch = {
            "m.room.member": self.parse_membership,
            "m.room.message": self.parse_msg
        }
        try:
            switch[etype](event)
        except KeyError:
            try:
                for p in self.plugins:
                    self.plugins[p].on_event(event, etype)
            except Exception as e:
                log.exception(e)
        except Exception as e:
            log.error("Couldn't process event: %s", e) 

def request(self, url, method, data=None, headers=None):
        url = verify(url)
        try:
            if method == 'get':
                r = self.session.get(url, timeout=self.timeout, headers=headers, verify=False, allow_redirects=True)
                return r
            else:
                r = self.session.post(url,
                                      data=data,
                                      timeout=self.timeout,
                                      headers=headers,
                                      verify=False,
                                      allow_redirects=False)
                return r
        except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, requests.exceptions.Timeout,
                requests.exceptions.SSLError, requests.exceptions.ConnectionError, ssl.SSLError, AttributeError,
                ConnectionRefusedError, socket.timeout, urllib3.exceptions.ReadTimeoutError, OpenSSL.SSL.WantReadError,
                urllib3.exceptions.DecodeError, requests.exceptions.ContentDecodingError):
            pass
        except Exception as e:
            logging.exception(e) 

def run(self):
        scripts = []
        try:
            for _ in glob.glob('script/*.py'):
                script_name = os.path.basename(_).replace('.py', '')
                vuln = importlib.import_module('script.%s' % script_name)
                scripts.append(vuln)
            # 随机打乱脚本加载顺序
            random.shuffle(scripts)
            with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
                vulns = {executor.submit(self.vuln, script): script for script in scripts}
                for future in concurrent.futures.as_completed(vulns, timeout=3):
                    future.result()
            self.out = list(filter(None, self.out))
            for i in self.out:
                console('Vuln', self.ip, i + '\n')

            Sqldb(self.dbname).get_vuln(self.ip, self.out)
        except (EOFError, concurrent.futures._base.TimeoutError):
            pass
        except Exception as e:
            logging.exception(e) 

def parse_ip(host):
    host = parse_host(host)
    # 根据domain得到ip 例如www.xxx.com 得到 x.x.x.x
    try:
        resolver = dns.resolver.Resolver()
        resolver.nameservers = ['1.1.1.1', '8.8.8.8']
        a = resolver.query(host, 'A')
        for i in a.response.answer:
            for j in i.items:
                if hasattr(j, 'address'):
                    if not re.search(r'1\.1\.1\.1|8\.8\.8\.8|127\.0\.0\.1|114\.114\.114\.114|0\.0\.0\.0', j.address):
                        return j.address
    except dns.resolver.NoAnswer:
        pass
    except Exception as e:
        logging.exception(e)
    return host 

def osdetect(ip):
    # sys.stdout.write(Bcolors.RED + "\nOS：\n" + Bcolors.ENDC)
    nm = nmap.PortScanner()
    try:
        result = nm.scan(hosts=ip, arguments='-sS -O -vv -n -T4 -p 80,22,443')
        for k, v in result.get('scan').items():
            if v.get('osmatch'):
                for i in v.get('osmatch'):
                    console('OSdetect', ip, i.get('name') + '\n')
                    return i.get('name')
            else:
                break
    except (xml.etree.ElementTree.ParseError, nmap.nmap.PortScannerError):
        pass
    except Exception as e:
        console('OSdetect', ip, 'None\n')
        logging.exception(e) 

def checkwaf(url):
    result = 'NoWAF'
    host = parse_host(url)

    if not iscdn(host):
        return 'CDN IP'

    try:
        req = Requests()
        r = req.get(url)
        result = verify(r.headers, r.text)
        if result == 'NoWAF':
            for i in payload:
                r = req.get(url + i)
                result = verify(r.headers, r.text)
                if result != 'NoWAF':
                    return result
        else:
            return result
    except (UnboundLocalError, AttributeError):
        pass
    except Exception as e:
        logging.exception(e) 

def create_urls(self):
        try:
            cursor = self.conn.cursor()
            cursor.execute("""
                CREATE TABLE IF NOT EXISTS urls (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                time varchar(255),
                domain varchar(255) DEFAULT '',
                title varchar(255) DEFAULT '',
                url varchar(255) DEFAULT '',
                contype varchar(255) DEFAULT '',
                rsp_len varchar(255) DEFAULT '',
                rsp_code varchar(255) DEFAULT '',
                md5 varchar(40) UNIQUE
                )
                """)
        except Exception as e:
            logging.exception(e) 

def jsparse(self, r):
        try:
            html = etree.HTML(r.text)
            result = html.xpath('//script/@src')
            for i in result:
                if not re.search(
                    r'jquery|bootstrap|adsbygoogle|angular|javascript|#|vue|react|51.la/=|map\.baidu\.com|canvas|cnzz\.com|slick\.js|autofill-event\.js|tld\.js|clipboard|Chart\.js',
                    i):
                    if '://' not in i:
                        i = re.sub(r'^/|^\.\./', '', i)
                        i = self.host + '/' + i
                    self.js.append(i)
        except (AttributeError, AttributeError, ValueError):
            pass
        except Exception as e:
            logging.exception(e) 

def do_execute(self):

        for step in self.steps:
            try:
                passed = step.do_execute()
                if passed:
                    self.status = self.success_status
                    logging.debug('Verification step %s passed', self.__class__.__name__)
                else:
                    self.status = StepStatus.failed
                    logging.error('Verification step %s failed!', self.__class__.__name__)
                    break
            except Exception:
                logging.exception('caught exception executing step %s', self.__class__.__name__)
                self.status = StepStatus.failed
                break
        return self.status == StepStatus.done or self.status == StepStatus.passed 

def custom_exception_handler(exc, context):
    # give more context on the error since DRF masks it as Not Found
    if isinstance(exc, Http404):
        set_rollback()
        return Response(str(exc), status=status.HTTP_404_NOT_FOUND)

    # Call REST framework's default exception handler after specific 404 handling,
    # to get the standard error response.
    response = exception_handler(exc, context)

    # No response means DRF couldn't handle it
    # Output a generic 500 in a JSON format
    if response is None:
        logging.exception('Uncaught Exception', exc_info=exc)
        set_rollback()
        return Response({'detail': 'Server Error'}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)

    # log a few different types of exception instead of using APIException
    if isinstance(exc, (DeisException, ServiceUnavailable, HealthcheckException)):
        logging.exception(exc.__cause__, exc_info=exc)

    return response 

def process_all(self):
        """Get data from all sensors."""
        next_list = self.config.sensors
        timeout = 1  # initial timeout in seconds
        max_retry = 6  # number of retries
        retry_count = 0

        while retry_count < max_retry and next_list:
            # if this is not the first try: wait some time before trying again
            if retry_count > 0:
                logging.info('try %d of %d: could not process sensor(s) %s. Waiting %d sec for next try',
                             retry_count, max_retry, SensorConfig.get_name_string(next_list), timeout)
                time.sleep(timeout)
                timeout *= 2  # exponential backoff-time

            current_list = next_list
            retry_count += 1
            next_list = []
            for sensor in current_list:
                try:
                    self.process_mac(sensor)
                # pylint: disable=bare-except, broad-except
                except Exception as exception:
                    next_list.append(sensor)  # if it failed, we'll try again in the next round
                    msg = "could not read data from {} ({}) with reason: {}".format(
                        sensor.mac, sensor.alias, str(exception))
                    if sensor.fail_silent:
                        logging.error(msg)
                        logging.warning('fail_silent is set for sensor %s, so not raising an exception.', sensor.alias)
                    else:
                        logging.exception(msg)
                        print(msg)

        # return sensors that could not be processed after max_retry
        return next_list 

def map_with_retries(self, requests, responses_for_requests):
        """Provides session-based retry functionality

        :param requests: A collection of Request objects.
        :param responses_for_requests: Dictionary mapping of requests to responses
        :param max_retries: The maximum number of retries to perform per session
        :param args: Additional arguments to pass into a retry mapping call


        """
        retries = []
        response_futures = [preq.callable() for preq in requests]

        for request, response_future in zip(requests, response_futures):
            try:
                response = response_future.result()
                if response is not None and response.status_code == 403:
                    logging.warning('Request to {} caused a 403 response status code.'.format(request.url))
                    raise InvalidRequestError('Access forbidden')
                if response is not None:
                    responses_for_requests[request] = response
            except RequestException as re:
                logging.error('An exception was raised for {}: {}'.format(request.url, re))
                if self.total_retries > 0:
                    self.total_retries -= 1
                    retries.append(request)

        # Recursively retry failed requests with the modified total retry count
        if retries:
            self.map_with_retries(retries, responses_for_requests) 

def _handle_file_download(self, response):
        name = None
        data = None
        try:
            name = re.findall('filename=(.+)', response.headers['content-disposition'])[0]
            data = urlsafe_b64encode(response.text.encode('utf-8')).decode('utf-8')
        except Exception:
            logging.exception('Unable to extract download data for {} '.format(response.request.url))
        return {'data': {'id': name, 'text': data}} 

def web_info(url):
    host = parse_host(url)
    ipaddr = parse_ip(host)
    url = url.strip('/')
    address = geoip(ipaddr)
    wafresult = checkwaf(url)
    req = Requests()
    # noinspection PyBroadException
    try:
        r = req.get(url)
        coding = chardet.detect(r.content).get('encoding')
        r.encoding = coding
        webinfo = WebPage(r.url, r.text, r.headers).info()
    except Exception as e:
        logging.exception(e)
        webinfo = {}
    if webinfo:
        console('Webinfo', host, 'title: {}\n'.format(webinfo.get('title')))
        console('Webinfo', host, 'Fingerprint: {}\n'.format(webinfo.get('apps')))
        console('Webinfo', host, 'Server: {}\n'.format(webinfo.get('server')))
        console('Webinfo', host, 'WAF: {}\n'.format(wafresult))
    else:
        webinfo = {}
        wafresult = 'None'
    if iscdn(host):
        osname = osdetect(host)
    else:
        osname = None
    
    data = {
        host: {
            'WAF': wafresult,
            'Ipaddr': ipaddr,
            'Address': address,
            'Webinfo': webinfo,
            'OS': osname,
        }
    }
    
    return data, webinfo.get('apps'), webinfo.get('title') 

def __call__(self, f):
    @functools.wraps(f)
    def wrapper(*args, **kwargs):
      retry_delay = self.delay
      for i in xrange(self.retries):
        try:
          return f(*args, **kwargs)
        except Exception as e:
          if (i+1) >= self.retries or not self.condition(e):
            raise
          logging.exception('Exception encountered, retrying in %s',
                            retry_delay)
          time.sleep(retry_delay.total_seconds())
          retry_delay *= 2
    return wrapper 

def __init__(self, retries=None, delay=None, condition=None):
    """Creates a new exponential retry decorator.

    Args:
      retries (int): Maximum number of retries before giving up.
      delay (datetime.timedelta): Amount of time to wait before retrying. This
          will double every retry attempt (exponential).
      condition (func): If not None, a function that will be passed the
          exception as its one argument. Retries will only happen if this
          function returns True. If None, retries will always happen.
    """
    self.retries = retries or 5
    self.delay = delay or datetime.timedelta(seconds=1)
    self.condition = condition or (lambda e: True) 

def hide_me(tb, g=globals()):
    """Hide stack traceback of given stack"""
    base_tb = tb
    try:
        while tb and tb.tb_frame.f_globals is not g:
            tb = tb.tb_next
        while tb and tb.tb_frame.f_globals is g:
            tb = tb.tb_next
    except Exception as e:
        logging.exception(e)
        tb = base_tb
    if not tb:
        tb = base_tb
    return tb