# -*- coding: utf-8 -*-
"""
ESD
~~~
Implements enumeration sub domains
:author: Feei <feei@feei.cn>
:homepage: https://github.com/FeeiCN/ESD
:license: GPL, see LICENSE for more details.
:copyright: Copyright (c) 2018 Feei. All rights reserved
"""
import os
import re
import time
import ssl
import math
import string
import random
import traceback
import itertools
import datetime
import colorlog
import asyncio
import uvloop
import aiodns
import aiohttp
import logging
import requests
import backoff
import socket
import async_timeout
import dns.query
import dns.zone
import dns.resolver
import multiprocessing
import threading
import tldextract
import json
import configparser
import base64
from tqdm import *
from colorama import Fore
from shodan import Shodan
import censys.certificates
from shodan.cli.helpers import get_api_key
from optparse import OptionParser
import urllib.parse as urlparse
from collections import Counter
from aiohttp.resolver import AsyncResolver
from itertools import islice
from difflib import SequenceMatcher
__version__ = '0.0.24'
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
handler = colorlog.StreamHandler()
formatter = colorlog.ColoredFormatter(
'%(log_color)s%(asctime)s [%(name)s] [%(levelname)s] %(message)s%(reset)s',
datefmt=None,
reset=True,
log_colors={
'DEBUG': 'cyan',
'INFO': 'green',
'WARNING': 'yellow',
'ERROR': 'red',
'CRITICAL': 'red,bg_white',
},
secondary_log_colors={},
style='%'
)
handler.setFormatter(formatter)
logger = colorlog.getLogger('ESD')
logger.addHandler(handler)
logger.setLevel(logging.INFO)
ssl.match_hostname = lambda cert, hostname: True
# 只采用了递归，速度非常慢，在优化完成前不建议开启
# TODO:优化DNS查询，递归太慢了
class DNSQuery(object):
def __init__(self, root_domain, subs, suffix):
# root domain
self.suffix = suffix
self.sub_domains = []
if root_domain:
self.sub_domains.append(root_domain)
for sub in subs:
sub = ''.join(sub.rsplit(suffix, 1)).rstrip('.')
self.sub_domains.append('{sub}.{domain}'.format(sub=sub, domain=suffix))
def dns_query(self):
"""
soa,txt,mx,aaaa
:param sub:
:return:
"""
final_list = []
for subdomain in self.sub_domains:
try:
soa = []
q_soa = dns.resolver.query(subdomain, 'SOA')
for a in q_soa:
soa.append(str(a.rname).strip('.'))
soa.append(str(a.mname).strip('.'))
except Exception as e:
logger.warning('Query failed. {e}'.format(e=str(e)))
try:
aaaa = []
q_aaaa = dns.resolver.query(subdomain, 'AAAA')
aaaa = [str(a.address).strip('.') for a in q_aaaa]
except Exception as e:
logger.warning('Query failed. {e}'.format(e=str(e)))
try:
txt = []
q_txt = dns.resolver.query(subdomain, 'TXT')
txt = [t.strings[0].decode('utf-8').strip('.') for t in q_txt]
except Exception as e:
logger.warning('Query failed. {e}'.format(e=str(e)))
try:
mx = []
q_mx = dns.resolver.query(subdomain, 'MX')
mx = [str(m.exchange).strip('.') for m in q_mx]
except Exception as e:
logger.warning('Query failed. {e}'.format(e=str(e)))
domain_set = soa + aaaa + txt + mx
domain_list = [i for i in domain_set]
for p in domain_set:
re_domain = re.findall(r'^(([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}\.?)$', p)
if len(re_domain) > 0 and subdomain in re_domain[0][0] and tldextract.extract(p).suffix != '':
continue
else:
domain_list.remove(p)
final_list = domain_list + final_list
# 递归调用，在子域名的dns记录中查找新的子域名
recursive = []
# print("before: {0}".format(final_list))
# print("self.sub_domain: {0}".format(self.sub_domains))
final_list = list(set(final_list).difference(set(self.sub_domains)))
# print("after: {0}".format(final_list))
if final_list:
d = DNSQuery('', final_list, self.suffix)
recursive = d.dns_query()
return final_list + recursive
class DNSTransfer(object):
def __init__(self, domain):
self.domain = domain
def transfer_info(self):
ret_zones = list()
try:
nss = dns.resolver.query(self.domain, 'NS')
nameservers = [str(ns) for ns in nss]
ns_addr = dns.resolver.query(nameservers[0], 'A')
# dnspython 的 bug，需要设置 lifetime 参数
zones = dns.zone.from_xfr(dns.query.xfr(ns_addr, self.domain, relativize=False, timeout=2, lifetime=2), check_origin=False)
names = zones.nodes.keys()
for n in names:
subdomain = ''
for t in range(0, len(n) - 1):
if subdomain != '':
subdomain += '.'
subdomain += str(n[t].decode())
if subdomain != self.domain:
ret_zones.append(subdomain)
return ret_zones
except BaseException:
return []
class CAInfo(object):
def __init__(self, domain):
self.domain = domain
def dns_resolve(self):
padding_domain = 'www.' + self.domain
# loop = asyncio.get_event_loop()
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
resolver = aiodns.DNSResolver(loop=loop)
f = resolver.query(padding_domain, 'A')
result = loop.run_until_complete(f)
return result[0].host
def get_cert_info_by_ip(self, ip):
s = socket.socket()
s.settimeout(2)
base_dir = os.path.dirname(os.path.abspath(__file__))
cert_path = base_dir + '/cacert.pem'
connect = ssl.wrap_socket(s, cert_reqs=ssl.CERT_REQUIRED, ca_certs=cert_path)
connect.settimeout(2)
connect.connect((ip, 443))
cert_data = connect.getpeercert().get('subjectAltName')
return cert_data
def get_ca_domain_info(self):
domain_list = list()
try:
ip = self.dns_resolve()
cert_data = self.get_cert_info_by_ip(ip)
except Exception as e:
return domain_list
for domain_info in cert_data:
hostname = domain_info[1]
if not hostname.startswith('*') and hostname.endswith(self.domain):
domain_list.append(hostname)
return domain_list
def get_subdomains(self):
subs = list()
subdomain_list = self.get_ca_domain_info()
for sub in subdomain_list:
subs.append(sub[:len(sub) - len(self.domain) - 1])
return subs
# 使用shodan接口进行枚举，但经测试并不能增加多少成果
class ShodanEngine(object):
def __init__(self, skey, conf, domain):
self.domain = domain
self.conf = conf
self.skey = skey
self.api = None
# 初始化shodan的api
def initialize(self, base_dir):
if self.skey:
logger.info('Initializing the shodan api.')
result = os.system('shodan init {skey}'.format(skey=self.skey))
if result:
logger.warning('Initializ failed, please check your key.')
return False
self.conf.set("shodan", "shodan_key", self.skey)
self.conf.write(open(base_dir + "/key.ini", "w"))
self.api = Shodan(get_api_key())
else:
from click.exceptions import ClickException
try:
key = None if get_api_key() == '' else get_api_key()
if key:
self.api = Shodan(key)
else:
return False
except ClickException as e:
logger.warning('The shodan api is empty so you can not use shodan api.')
return False
return True
def search(self):
subs = list()
result = self.api.search('hostname:{domain}'.format(domain=self.domain))
for service in result['matches']:
domain = service['hostnames'][0]
subs.append(domain.rsplit(self.domain, 1)[0].strip('.'))
return set(subs)
# fofa的sdk不支持python3，就只能调用restful api了，但是挖掘成果比shodan多
class FofaEngine(object):
def __init__(self, fofa_struct, conf, domain):
self.base_url = "https://fofa.so/api/v1/search/all?email={email}&key={key}&qbase64={domain}"
self.email = fofa_struct['femail']
self.fkey = fofa_struct['fkey']
self.domain = base64.b64encode(domain.encode('utf-8')).decode('utf-8')
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.8',
'Accept-Encoding': 'gzip',
}
self.timeout = 30
self.conf = conf
def initialize(self, base_dir):
if self.fkey is not None and self.email is not None:
self.conf.set("fofa", "fofa_key", self.fkey)
self.conf.set("fofa", "fofa_email", self.email)
self.conf.write(open(base_dir + "/key.ini", "w"))
return True
else:
self.fkey = self.conf.items("fofa")[0][1]
self.email = self.conf.items("fofa")[1][1]
if self.fkey and self.email:
return True
return False
def search(self):
result = list()
url = self.base_url.format(email=self.email, key=self.fkey, domain=self.domain)
try:
resp = requests.Session().get(url, headers=self.headers, timeout=self.timeout)
json_resp = json.loads(resp.text)
for res in json_resp['results']:
domain = urlparse.urlparse(res[0]).netloc
result.append(domain.rsplit(self.domain, 1)[0].strip('.'))
except Exception as e:
result = []
return result
# Zoomeye的效果还可以，但是比fofa还贵
class ZoomeyeEngine(object):
def __init__(self, domain, zoomeye_struct, conf):
self.headers = {
"Authorization": "JWT {token}"
}
self.url = 'https://api.zoomeye.org/web/search?query=site:{domain}&page={num}'
self.domain = domain
self.zoomeye_struct = zoomeye_struct
self.conf = conf
def initialize(self, base_dir):
username = self.zoomeye_struct['username']
password = self.zoomeye_struct['password']
if username != '' and password != '':
resp = requests.Session().post(url='https://api.zoomeye.org/user/login', data=json.dumps(self.zoomeye_struct))
resp_json = json.loads(resp.text)
else:
username = self.conf.items("zoomeye")[0][1]
password = self.conf.items("zoomeye")[1][1]
if username != '' and password != '':
self.zoomeye_struct['username'] = username
self.zoomeye_struct['password'] = password
resp = requests.Session().post(url='https://api.zoomeye.org/user/login', data=json.dumps(self.zoomeye_struct))
resp_json = json.loads(resp.text)
else:
return False
if 'error' in resp_json.keys():
# logger.warning('In Zoomeye' + resp_json['message'])
return False
self.conf.set("zoomeye", "zoomeye_username", username)
self.conf.set("zoomeye", "zoomeye_password", password)
self.conf.write(open(base_dir + "/key.ini", "w"))
self.headers['Authorization'] = "JWT {token}".format(token=resp_json['access_token'])
return True
def search(self, num):
url = self.url.format(domain=self.domain, num=num)
resp = requests.Session().get(url=url, headers=self.headers)
try:
# zoomeye对于频繁的api调用会做限制，但是降低频率又会影响效率
response = json.loads(resp.text)
except Exception:
response = None
return response
def enumerate(self):
flag = True
num = 1
result = list()
while flag:
response = self.search(num)
if response is None or 'error' in response.keys():
# print(response)
flag = False
else:
match_list = response["matches"]
for block in match_list:
domain = block['site']
result.append(domain.rsplit(self.domain, 1)[0].strip('.'))
num = num + 1
return result
# censys的接口有点不稳定，经常出现timeout的情况
class CensysEngine(object):
def __init__(self, domain, censys_struct, conf):
self.domain = domain
self.conf = conf
self.censys_struct = censys_struct
self.certificates = None
self.fields = ['parsed.subject_dn']
def initialize(self, base_dir):
uid = self.censys_struct['uid']
secret = self.censys_struct['secret']
try:
if uid is not None and secret is not None:
self.certificates = censys.certificates.CensysCertificates(uid, secret)
else:
uid = self.conf.items("censys")[0][1]
secret = self.conf.items("censys")[1][1]
if uid != '' and secret != '':
self.certificates = censys.certificates.CensysCertificates(uid, secret)
else:
return False
self.conf.set("censys", "UID", uid)
self.conf.set("censys", "SECRET", secret)
self.conf.write(open(base_dir + "/key.ini", "w"))
except Exception as e:
return False
return True
def search(self):
result = list()
try:
for c in self.certificates.search(self.domain, fields=self.fields):
subject = c['parsed.subject_dn'].strip()
reg_domain = self.domain.replace('.', '[.]')
reg_text = r'(([-a-zA-Z0-9]+[.])*{reg_domain}$)'.format(reg_domain=reg_domain)
match_list = re.findall(reg_text, subject)
if match_list:
domain = match_list[0][0]
result.append(domain.rsplit(self.domain, 1)[0].strip('.'))
except Exception as e:
logger.warning(str(e))
return result
else:
return result
class EngineBase(multiprocessing.Process):
def __init__(self, base_url, domain, q, verbose, proxy):
multiprocessing.Process.__init__(self)
self.lock = threading.Lock()
self.q = q
self.subdomains = []
self.base_url = base_url
self.domain = domain
self.session = requests.Session()
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.8',
'Accept-Encoding': 'gzip',
}
self.timeout = 30
self.verbose = verbose
self.proxy = proxy
def get_page(self, num):
return num + 10
# 应当在子类里重写
def check_response_errors(self, resp):
return True
def should_sleep(self):
time.sleep(random.randint(2, 5))
return
def get_response(self, response):
if response is None:
return 0
return response.text if hasattr(response, "text") else response.content
def check_max_pages(self, num):
if self.MAX_PAGES == 0:
return False
return num >= self.MAX_PAGES
def send_req(self, page_no=1):
url = self.base_url.format(domain=self.domain, page_no=page_no)
try:
resp = self.session.get(
url, headers=self.headers, timeout=self.timeout)
except Exception:
resp = None
return self.get_response(resp)
def enumerate(self):
flag = True
page_no = 0
prev_links = []
retries = 0
while flag:
if self.check_max_pages(page_no):
return self.subdomains
resp = self.send_req(page_no)
if not self.check_response_errors(resp):
return self.subdomains
links = self.extract_domains(resp)
if links == prev_links:
retries += 1
page_no = self.get_page(page_no)
if retries >= 3:
return self.subdomains
prev_links = links
self.should_sleep()
return self.subdomains
def run(self):
domain_list = self.enumerate()
for domain in domain_list:
self.q.append(domain.rsplit(self.domain, 1)[0].strip('.'))
class Google(EngineBase):
def __init__(self, domain, q, verbose, proxy):
base_url = "https://www.google.com/search?q=site:{domain}+-www.{domain}&start={page_no}"
super(Google, self).__init__(base_url, domain, q, verbose, proxy)
self.MAX_DOMAINS = 11
self.MAX_PAGES = 200
self.engine_name = 'Google'
def extract_domains(self, resp):
links_list = list()
link_regx = re.compile(r'<cite.*?>(.*?)<\/cite>')
try:
links_list = link_regx.findall(resp)
for link in links_list:
link = re.sub('<span.*>', '', link)
if not link.startswith('http'):
link = "http://" + link
subdomain = urlparse.urlparse(link).netloc
if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
logger.info('{engine_name}: {subdomain}'.format(engine_name=self.engine_name, subdomain=subdomain))
self.subdomains.append(subdomain.strip())
except Exception:
pass
return links_list
def check_response_errors(self, resp):
if isinstance(resp, int):
logger.warning("Please use proxy to access Google!")
logger.warning("Finished now the Google Enumeration ...")
return False
return True
def send_req(self, page_no=1):
url = self.base_url.format(domain=self.domain, page_no=page_no)
try:
resp = self.session.get(url, proxies=self.proxy, headers=self.headers, timeout=self.timeout)
except Exception as e:
resp = None
return self.get_response(resp)
class Bing(EngineBase):
def __init__(self, domain, q, verbose, proxy):
base_url = 'https://www.bing.com/search?q=domain%3A{domain}%20-www.{domain}&go=Submit&first={page_no}'
super(Bing, self).__init__(base_url, domain, q, verbose, proxy)
self.MAX_PAGES = 30
self.engine_name = 'Bing'
def extract_domains(self, resp):
links_list = list()
link_regx = re.compile('<li class="b_algo"><div class="b_title"><h2><a target="_blank" href="(.*?)"')
link_regx2 = re.compile('<li class="b_algo"><h2><a target="_blank" href="(.*?)"')
try:
links1 = link_regx.findall(resp)
links2 = link_regx2.findall(resp)
links_list = links1 + links2
for link in links_list:
link = re.sub(r'<(\/)?strong>|<span.*?>|<|>', '', link)
if not (link.startswith('http') or link.startswith('https')):
link = "http://" + link
subdomain = urlparse.urlparse(link).netloc
if subdomain not in self.subdomains and subdomain != self.domain:
logger.info('{engine_name}: {subdomain}'.format(engine_name=self.engine_name, subdomain=subdomain))
self.subdomains.append(subdomain.strip())
except Exception:
pass
return links_list
class Yahoo(EngineBase):
def __init__(self, domain, q, verbose, proxy):
base_url = "https://search.yahoo.com/search?p=site%3A{domain}%20-domain%3Awww.{domain}&b={page_no}"
super(Yahoo, self).__init__(base_url, domain, q, verbose, proxy)
self.engine_name = "Yahoo"
self.MAX_DOMAINS = 10
self.MAX_PAGES = 0
def extract_domains(self, resp):
link_regx2 = re.compile('<span class=" fz-.*? fw-m fc-12th wr-bw.*?">(.*?)</span>')
link_regx = re.compile('<span class="txt"><span class=" cite fw-xl fz-15px">(.*?)</span>')
links_list = []
try:
links = link_regx.findall(resp)
links2 = link_regx2.findall(resp)
links_list = links + links2
for link in links_list:
link = re.sub(r"<(\/)?b>", "", link)
if not link.startswith('http'):
link = "http://" + link
subdomain = urlparse.urlparse(link).netloc
if not subdomain.endswith(self.domain):
continue
if subdomain and subdomain not in self.subdomains and subdomain != self.domain:
logger.info('{engine_name}: {subdomain}'.format(engine_name=self.engine_name, subdomain=subdomain))
self.subdomains.append(subdomain.strip())
except Exception:
pass
return links_list
def check_response_errors(self, resp):
if isinstance(resp, int):
logger.warning("Please use proxy to access Yahoo!")
logger.warning("Finished now the Yahoo Enumeration ...")
return False
return True
def send_req(self, page_no=1):
url = self.base_url.format(domain=self.domain, page_no=page_no)
try:
resp = self.session.get(url, proxies=self.proxy, headers=self.headers, timeout=self.timeout)
except Exception as e:
resp = None
return self.get_response(resp)
class Baidu(EngineBase):
def __init__(self, domain, q, verbose, proxy):
base_url = "https://www.baidu.com/s?ie=UTF-8&wd=site%3A{domain}%20-site%3Awww.{domain}&pn={page_no}"
super(Baidu, self).__init__(base_url, domain, q, verbose, proxy)
self.MAX_PAGES = 30
self.engine_name = 'Baidu'
def extract_domains(self, resp):
links = list()
found_newdomain = False
subdomain_list = []
link_regx = re.compile('<a.*?class="c-showurl".*?>(.*?)</a>')
try:
links = link_regx.findall(resp)
for link in links:
link = re.sub('<.*?>|>|<| ', '', link)
if not (link.startswith('http') or link.startswith('https')):
link = "http://" + link
subdomain = urlparse.urlparse(link).netloc
if subdomain.endswith(self.domain):
subdomain_list.append(subdomain)
if subdomain not in self.subdomains and subdomain != self.domain:
found_newdomain = True
logger.info('{engine_name}: {subdomain}'.format(engine_name=self.engine_name, subdomain=subdomain))
self.subdomains.append(subdomain.strip())
except Exception:
pass
if not found_newdomain and subdomain_list:
self.querydomain = self.findsubs(subdomain_list)
return links
def findsubs(self, subdomains):
count = Counter(subdomains)
subdomain1 = max(count, key=count.get)
count.pop(subdomain1, "None")
subdomain2 = max(count, key=count.get) if count else ''
return (subdomain1, subdomain2)
class EnumSubDomain(object):
def __init__(self, domain, response_filter=None, dns_servers=None, skip_rsc=False, debug=False,
split=None, engines=[Baidu, Google, Bing, Yahoo], proxy={}, multiresolve=False,
shodan_key=None, fofa={'fkey': None, 'femail': None}, zoomeye={'username': None, 'password': None}, censys={'uid': None, 'secret': None}):
self.project_directory = os.path.abspath(os.path.dirname(__file__))
logger.info('Version: {v}'.format(v=__version__))
logger.info('----------')
logger.info('Start domain: {d}'.format(d=domain))
self.engines = engines
self.proxy = proxy
self.data = {}
self.domain = domain
self.skip_rsc = skip_rsc
self.split = split
self.multiresolve = multiresolve
self.skey = shodan_key
self.fofa_struct = fofa
self.conf = configparser.ConfigParser()
self.zoomeye_struct = zoomeye
self.censys_struct = censys
self.stable_dns_servers = ['1.1.1.1', '223.5.5.5']
if dns_servers is None:
dns_servers = [
'223.5.5.5', # AliDNS
'114.114.114.114', # 114DNS
'1.1.1.1', # Cloudflare
'119.29.29.29', # DNSPod
'1.2.4.8', # sDNS
# '11.1.1.1' # test DNS, not available
# '8.8.8.8', # Google DNS, 延时太高了
]
random.shuffle(dns_servers)
self.dns_servers = dns_servers
self.resolver = None
self.loop = asyncio.get_event_loop()
self.general_dicts = []
# Mark whether the current domain name is a pan-resolved domain name
self.is_wildcard_domain = False
# Use a nonexistent domain name to determine whether
# there is a pan-resolve based on the DNS resolution result
self.wildcard_sub = 'feei-esd-{random}'.format(random=random.randint(0, 9999))
self.wildcard_sub3 = 'feei-esd-{random}.{random}'.format(random=random.randint(0, 9999))
# There is no domain name DNS resolution IP
self.wildcard_ips = []
# No domain name response HTML
self.wildcard_html = None
self.wildcard_html_len = 0
self.wildcard_html3 = None
self.wildcard_html3_len = 0
# Subdomains that are consistent with IPs that do not have domain names
self.wildcard_subs = []
# Wildcard domains use RSC
self.wildcard_domains = {}
# Corotines count
self.coroutine_count = None
self.coroutine_count_dns = 100000
self.coroutine_count_request = 100
# dnsaio resolve timeout
self.resolve_timeout = 2
# RSC ratio
self.rsc_ratio = 0.8
self.remainder = 0
self.count = 0
# Request Header
self.request_headers = {
'Connection': 'keep-alive',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Baiduspider',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'DNT': '1',
'Referer': 'http://www.baidu.com/',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'}
# Filter the domain's response(regex)
self.response_filter = response_filter
# debug mode
self.debug = debug
if self.debug:
logger.setLevel(logging.DEBUG)
# collect redirecting domains and response domains
self.domains_rs = []
self.domains_rs_processed = []
def generate_general_dicts(self, line):
"""
Generate general subdomains dicts
:param line:
:return:
"""
letter_count = line.count('{letter}')
number_count = line.count('{number}')
letters = itertools.product(string.ascii_lowercase, repeat=letter_count)
letters = [''.join(l) for l in letters]
numbers = itertools.product(string.digits, repeat=number_count)
numbers = [''.join(n) for n in numbers]
for l in letters:
iter_line = line.replace('{letter}' * letter_count, l)
self.general_dicts.append(iter_line)
number_dicts = []
for gd in self.general_dicts:
for n in numbers:
iter_line = gd.replace('{number}' * number_count, n)
number_dicts.append(iter_line)
if len(number_dicts) > 0:
return number_dicts
else:
return self.general_dicts
def load_sub_domain_dict(self):
"""
Load subdomains from files and dicts
:return:
"""
dicts = []
if self.debug:
path = '{pd}/subs-test.esd'.format(pd=self.project_directory)
else:
path = '{pd}/subs.esd'.format(pd=self.project_directory)
with open(path, encoding='utf-8') as f:
for line in f:
line = line.strip().lower()
# skip comments and space
if '#' in line or line == '':
continue
if '{letter}' in line or '{number}' in line:
self.general_dicts = []
dicts_general = self.generate_general_dicts(line)
dicts += dicts_general
else:
# compatibility other dicts
line = line.strip('.')
dicts.append(line)
dicts = list(set(dicts))
# split dict
if self.split is not None:
s = self.split.split('/')
dicts_choose = int(s[0])
dicts_count = int(s[1])
dicts_every = int(math.ceil(len(dicts) / dicts_count))
dicts = [dicts[i:i + dicts_every] for i in range(0, len(dicts), dicts_every)][dicts_choose - 1]
logger.info('Sub domain dict split {count} and get {choose}st'.format(count=dicts_count, choose=dicts_choose))
# root domain
dicts.append('@')
return dicts
async def query(self, sub):
"""
Query domain
:param sub:
:return:
"""
ret = None
# root domain
if sub == '@' or sub == '':
sub_domain = self.domain
else:
sub = ''.join(sub.rsplit(self.domain, 1)).rstrip('.')
sub_domain = '{sub}.{domain}'.format(sub=sub, domain=self.domain)
try:
ret = await self.resolver.query(sub_domain, 'A')
except aiodns.error.DNSError as e:
err_code, err_msg = e.args[0], e.args[1]
# 1: DNS server returned answer with no data
# 4: Domain name not found
# 11: Could not contact DNS servers
# 12: Timeout while contacting DNS servers
if err_code not in [1, 4, 11, 12]:
logger.warning('{domain} {exception}'.format(domain=sub_domain, exception=e))
except Exception as e:
logger.info(sub_domain)
logger.warning(traceback.format_exc())
else:
ret = [r.host for r in ret]
domain_ips = [s for s in ret]
# It is a wildcard domain name and
# the subdomain IP that is burst is consistent with the IP
# that does not exist in the domain name resolution,
# the response similarity is discarded for further processing.
if self.is_wildcard_domain and (sorted(self.wildcard_ips) == sorted(domain_ips) or set(domain_ips).issubset(self.wildcard_ips)):
if self.skip_rsc:
logger.debug('{sub} maybe wildcard subdomain, but it is --skip-rsc mode now, it will be drop this subdomain in results'.format(sub=sub_domain))
else:
logger.debug('{r} maybe wildcard domain, continue RSC {sub}'.format(r=self.remainder, sub=sub_domain, ips=domain_ips))
else:
if sub != self.wildcard_sub:
self.data[sub_domain] = sorted(domain_ips)
print('', end='\n')
self.count += 1
logger.info('{r} {sub} {ips}'.format(r=self.remainder, sub=sub_domain, ips=domain_ips))
self.remainder += -1
return sub_domain, ret
@staticmethod
def limited_concurrency_coroutines(coros, limit):
futures = [
asyncio.ensure_future(c)
for c in islice(coros, 0, limit)
]
async def first_to_finish():
while True:
await asyncio.sleep(0)
for f in futures:
if f.done():
futures.remove(f)
try:
nf = next(coros)
futures.append(asyncio.ensure_future(nf))
except StopIteration:
pass
return f.result()
while len(futures) > 0:
yield first_to_finish()
async def start(self, tasks, tasks_num):
"""
Limit the number of coroutines for reduce memory footprint
:param tasks:
:return:
"""
for res in tqdm(self.limited_concurrency_coroutines(tasks, self.coroutine_count), bar_format="%s{l_bar}%s{bar}%s{r_bar}%s" % (Fore.YELLOW, Fore.YELLOW, Fore.YELLOW, Fore.RESET), total=tasks_num):
await res
@staticmethod
def data_clean(data):
try:
html = re.sub(r'\s', '', data)
html = re.sub(r'<script(?!.*?src=).*?>.*?</script>', '', html)
return html
except BaseException:
return data
@staticmethod
@backoff.on_exception(backoff.expo, TimeoutError, max_tries=3)
async def fetch(session, url):
"""
Fetch url response with session
:param session:
:param url:
:return:
"""
try:
async with async_timeout.timeout(20):
async with session.get(url) as response:
return await response.text(), response.history
except Exception as e:
# TODO 当在随机DNS场景中只做响应相似度比对的话，如果域名没有Web服务会导致相似度比对失败从而丢弃
logger.warning('fetch exception: {e} {u}'.format(e=type(e).__name__, u=url))
return None, None
async def similarity(self, sub):
"""
Enumerate subdomains by responding to similarities
:param sub:
:return:
"""
# root domain
if sub == '@' or sub == '':
sub_domain = self.domain
else:
sub = ''.join(sub.rsplit(self.domain, 1)).rstrip('.')
sub_domain = '{sub}.{domain}'.format(sub=sub, domain=self.domain)
if sub_domain in self.domains_rs:
self.domains_rs.remove(sub_domain)
full_domain = 'http://{sub_domain}'.format(sub_domain=sub_domain)
# 如果跳转中的域名是以下情况则不加入下一轮RSC
skip_domain_with_history = [
# 跳到主域名了
'{domain}'.format(domain=self.domain),
'www.{domain}'.format(domain=self.domain),
# 跳到自己本身了，比如HTTP跳HTTPS
'{domain}'.format(domain=sub_domain),
]
try:
regex_domain = r"((?!\/)(?:(?:[a-z\d-]*\.)+{d}))".format(d=self.domain)
resolver = AsyncResolver(nameservers=self.dns_servers)
conn = aiohttp.TCPConnector(resolver=resolver)
async with aiohttp.ClientSession(connector=conn, headers=self.request_headers) as session:
html, history = await self.fetch(session, full_domain)
html = self.data_clean(html)
if history is not None and len(history) > 0:
location = str(history[-1].headers['location'])
if '.' in location:
location_split = location.split('/')
if len(location_split) > 2:
location = location_split[2]
else:
location = location
try:
location = re.match(regex_domain, location).group(0)
except AttributeError:
location = location
status = history[-1].status
if location in skip_domain_with_history and len(history) >= 2:
logger.debug('domain in skip: {s} {r} {l}'.format(s=sub_domain, r=status, l=location))
return
else:
# cnsuning.com suning.com
if location[-len(self.domain) - 1:] == '.{d}'.format(d=self.domain):
# collect redirecting's domains
if sub_domain != location and location not in self.domains_rs and location not in self.domains_rs_processed:
print('', end='\n')
logger.info('[{sd}] add redirect domain: {l}({len})'.format(sd=sub_domain, l=location, len=len(self.domains_rs)))
self.domains_rs.append(location)
self.domains_rs_processed.append(location)
else:
print('', end='\n')
logger.info('not same domain: {l}'.format(l=location))
else:
print('', end='\n')
logger.info('not domain(maybe path): {l}'.format(l=location))
if html is None:
print('', end='\n')
logger.warning('domain\'s html is none: {s}'.format(s=sub_domain))
return
# collect response html's domains
response_domains = re.findall(regex_domain, html)
response_domains = list(set(response_domains) - set([sub_domain]))
for rd in response_domains:
rd = rd.strip().strip('.')
if rd.count('.') >= sub_domain.count('.') and rd[-len(sub_domain):] == sub_domain:
continue
if rd not in self.domains_rs:
if rd not in self.domains_rs_processed:
print('', end='\n')
logger.info('[{sd}] add response domain: {s}({l})'.format(sd=sub_domain, s=rd, l=len(self.domains_rs)))
self.domains_rs.append(rd)
self.domains_rs_processed.append(rd)
if len(html) == self.wildcard_html_len:
ratio = 1
else:
# SPEED 4 2 1, but here is still the bottleneck
# real_quick_ratio() > quick_ratio() > ratio()
# TODO bottleneck
if sub.count('.') == 0: # secondary sub, ex: www
ratio = SequenceMatcher(None, html, self.wildcard_html).real_quick_ratio()
ratio = round(ratio, 3)
else: # tertiary sub, ex: home.dev
ratio = SequenceMatcher(None, html, self.wildcard_html3).real_quick_ratio()
ratio = round(ratio, 3)
self.remainder += -1
if ratio > self.rsc_ratio:
# passed
logger.debug('{r} RSC ratio: {ratio} (passed) {sub}'.format(r=self.remainder, sub=sub_domain, ratio=ratio))
else:
# added
# for def distinct func
# self.wildcard_domains[sub_domain] = html
if self.response_filter is not None:
for resp_filter in self.response_filter.split(','):
if resp_filter in html:
logger.debug('{r} RSC filter in response (passed) {sub}'.format(r=self.remainder, sub=sub_domain))
return
else:
continue
self.data[sub_domain] = self.wildcard_ips
else:
self.data[sub_domain] = self.wildcard_ips
print('', end='\n')
logger.info('{r} RSC ratio: {ratio} (added) {sub}'.format(r=self.remainder, sub=sub_domain, ratio=ratio))
except Exception as e:
logger.debug(traceback.format_exc())
return
def distinct(self):
for domain, html in self.wildcard_domains.items():
for domain2, html2 in self.wildcard_domains.items():
ratio = SequenceMatcher(None, html, html2).real_quick_ratio()
if ratio > self.rsc_ratio:
# remove this domain
if domain2 in self.data:
del self.data[domain2]
m = 'Remove'
else:
m = 'Stay'
logger.info('{d} : {d2} {ratio} {m}'.format(d=domain, d2=domain2, ratio=ratio, m=m))
def dnspod(self):
"""
http://feei.cn/esd
:return:
"""
# noinspection PyBroadException
try:
content = requests.get('http://www.dnspod.cn/proxy_diagnose/recordscan/{domain}?callback=feei'.format(domain=self.domain), timeout=5).text
domains = re.findall(r'[^": ]*{domain}'.format(domain=self.domain), content)
domains = list(set(domains))
tasks = (self.query(''.join(domain.rsplit(self.domain, 1)).rstrip('.')) for domain in domains)
self.loop.run_until_complete(self.start(tasks, len(domains)))
except Exception as e:
domains = []
return domains
def check(self, dns):
logger.info("Checking if DNS server {dns} is available".format(dns=dns))
msg = b'\x5c\x6d\x01\x00\x00\x01\x00\x00\x00\x00\x00\x00\x03www\x05baidu\x03com\x00\x00\x01\x00\x01'
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.settimeout(3)
repeat = {
1: 'first',
2: 'second',
3: 'third'
}
for i in range(3):
logger.info("Sending message to DNS server a {times} time".format(times=repeat[i + 1]))
sock.sendto(msg, (dns, 53))
try:
sock.recv(4096)
break
except socket.timeout as e:
logger.warning('Failed!')
if i == 2:
return False
return True
def run(self):
"""
Run
:return:
"""
start_time = time.time()
subs = self.load_sub_domain_dict()
self.remainder = len(subs)
logger.info('Sub domain dict count: {c}'.format(c=len(subs)))
logger.info('Generate coroutines...')
# Verify that all DNS server results are consistent
stable_dns = []
wildcard_ips = None
last_dns = []
only_similarity = False
for dns in self.dns_servers:
delay = self.check(dns)
if not delay:
logger.warning("@{dns} is not available, skip this DNS server".format(dns=dns))
continue
self.resolver = aiodns.DNSResolver(loop=self.loop, nameservers=[dns], timeout=self.resolve_timeout)
job = self.query(self.wildcard_sub)
sub, ret = self.loop.run_until_complete(job)
logger.info('@{dns} {sub} {ips}'.format(dns=dns, sub=sub, ips=ret))
if ret is None:
ret = None
else:
ret = sorted(ret)
if dns in self.stable_dns_servers:
wildcard_ips = ret
stable_dns.append(ret)
if ret:
equal = [False for r in ret if r not in last_dns]
if len(last_dns) != 0 and False in equal:
only_similarity = self.is_wildcard_domain = True
logger.info('Is a random resolve subdomain.')
break
else:
last_dns = ret
is_all_stable_dns = stable_dns.count(stable_dns[0]) == len(stable_dns)
if not is_all_stable_dns:
logger.info('Is all stable dns: NO, use the default dns server')
self.resolver = aiodns.DNSResolver(loop=self.loop, nameservers=self.stable_dns_servers, timeout=self.resolve_timeout)
# Wildcard domain
is_wildcard_domain = not (stable_dns.count(None) == len(stable_dns))
if is_wildcard_domain or self.is_wildcard_domain:
if not self.skip_rsc:
logger.info('This is a wildcard domain, will enumeration subdomains use by DNS+RSC.')
else:
logger.info('This is a wildcard domain, but it is --skip-rsc mode now, it will be drop all random resolve subdomains in results')
self.is_wildcard_domain = True
if wildcard_ips is not None:
self.wildcard_ips = wildcard_ips
else:
self.wildcard_ips = stable_dns[0]
logger.info('Wildcard IPS: {ips}'.format(ips=self.wildcard_ips))
if not self.skip_rsc:
try:
self.wildcard_html = requests.get('http://{w_sub}.{domain}'.format(w_sub=self.wildcard_sub, domain=self.domain), headers=self.request_headers, timeout=10, verify=False).text
self.wildcard_html = self.data_clean(self.wildcard_html)
self.wildcard_html_len = len(self.wildcard_html)
self.wildcard_html3 = requests.get('http://{w_sub}.{domain}'.format(w_sub=self.wildcard_sub3, domain=self.domain), headers=self.request_headers, timeout=10, verify=False).text
self.wildcard_html3 = self.data_clean(self.wildcard_html3)
self.wildcard_html3_len = len(self.wildcard_html3)
logger.info('Wildcard domain response html length: {len} 3length: {len2}'.format(len=self.wildcard_html_len, len2=self.wildcard_html3_len))
except requests.exceptions.SSLError:
logger.warning('SSL Certificate Error!')
except requests.exceptions.ConnectTimeout:
logger.warning('Request response content failed, check network please!')
except requests.exceptions.ReadTimeout:
self.wildcard_html = self.wildcard_html3 = ''
self.wildcard_html_len = self.wildcard_html3_len = 0
logger.warning('Request response content timeout, {w_sub}.{domain} and {w_sub3}.{domain} maybe not a http service, content will be set to blank!'.format(w_sub=self.wildcard_sub,
domain=self.domain,
w_sub3=self.wildcard_sub3))
except requests.exceptions.ConnectionError:
logger.error('ESD can\'t get the response text so the rsc will be skipped. ')
self.skip_rsc = True
else:
logger.info('Not a wildcard domain')
if not only_similarity:
self.coroutine_count = self.coroutine_count_dns
tasks = (self.query(sub) for sub in subs)
self.loop.run_until_complete(self.start(tasks, len(subs)))
logger.info("Brute Force subdomain count: {total}".format(total=self.count))
dns_time = time.time()
time_consume_dns = int(dns_time - start_time)
# DNSPod JSONP API
logger.info('Collect DNSPod JSONP API\'s subdomains...')
dnspod_domains = self.dnspod()
logger.info('DNSPod JSONP API Count: {c}'.format(c=len(dnspod_domains)))
# CA subdomain info
ca_subdomains = []
logger.info('Collect subdomains in CA...')
ca_subdomains = CAInfo(self.domain).get_subdomains()
if len(ca_subdomains):
tasks = (self.query(sub) for sub in ca_subdomains)
self.loop.run_until_complete(self.start(tasks, len(ca_subdomains)))
logger.info('CA subdomain count: {c}'.format(c=len(ca_subdomains)))
# DNS Transfer Vulnerability
transfer_info = []
logger.info('Check DNS Transfer Vulnerability in {domain}'.format(domain=self.domain))
transfer_info = DNSTransfer(self.domain).transfer_info()
if len(transfer_info):
logger.warning('DNS Transfer Vulnerability found in {domain}!'.format(domain=self.domain))
tasks = (self.query(sub) for sub in transfer_info)
self.loop.run_until_complete(self.start(tasks, len(transfer_info)))
logger.info('DNS Transfer subdomain count: {c}'.format(c=len(transfer_info)))
# Use search engines to enumerate subdomains (support Baidu,Bing,Google,Yahoo)
subdomains = []
if self.engines:
logger.info('Enumerating subdomains with search engine')
subdomains_queue = multiprocessing.Manager().list()
enums = [enum(self.domain, q=subdomains_queue, verbose=False, proxy=self.proxy) for enum in self.engines]
for enum in enums:
enum.start()
for enum in enums:
enum.join()
subdomains = set(subdomains_queue)
if len(subdomains):
tasks = (self.query(sub) for sub in subdomains)
self.loop.run_until_complete(self.start(tasks, len(subdomains)))
logger.info('Search engines subdomain count: {subdomains_count}'.format(subdomains_count=len(subdomains)))
# Use shodan to enumerate subdomains (need key and money)
shodan_result = []
base_dir = os.path.dirname(os.path.abspath(__file__))
self.conf.read(base_dir + "/key.ini")
shodan = ShodanEngine(self.skey, self.conf, self.domain)
is_success = shodan.initialize(base_dir)
if is_success:
logger.info('Enumerating subdomains with Shodan')
shodan_result = shodan.search()
if len(shodan_result):
tasks = (self.query(sub) for sub in shodan_result)
self.loop.run_until_complete(self.start(tasks, len(shodan_result)))
logger.info("Shodan subdomain count: {subdomains_count}".format(subdomains_count=len(shodan_result)))
# Use fofa to enumerate subdomains (need key and money)
fofa_result = []
fofa = FofaEngine(self.fofa_struct, self.conf, self.domain)
is_success = fofa.initialize(base_dir)
if is_success:
logger.info("Enumerating subdomains with Fofa")
fofa_result = fofa.search()
if len(fofa_result):
tasks = (self.query(sub) for sub in fofa_result)
self.loop.run_until_complete(self.start(tasks, len(fofa_result)))
logger.info("Fofa subdomain count: {subdomains_count}".format(subdomains_count=len(fofa_result)))
# Use zoomeye to enumerate subdomains (need account or money)
zoomeye_result = []
zoomeye = ZoomeyeEngine(self.domain, self.zoomeye_struct, self.conf)
is_success = zoomeye.initialize(base_dir)
if is_success:
logger.info("Enumerating subdomains with Zoomeye")
zoomeye_result = zoomeye.enumerate()
if len(zoomeye_result):
tasks = (self.query(sub) for sub in zoomeye_result)
self.loop.run_until_complete(self.start(tasks, len(zoomeye_result)))
logger.info("Zoomeye subdomain count: {subdomains_count}".format(subdomains_count=len(zoomeye_result)))
censys_result = []
censys = CensysEngine(self.domain, self.censys_struct, self.conf)
is_success = censys.initialize(base_dir)
if is_success:
logger.info("Enumerating subdomains with Censys")
censys_result = censys.search()
if len(censys_result):
tasks = (self.query(sub) for sub in censys_result)
self.loop.run_until_complete(self.start(tasks, len(censys_result)))
logger.info("Censys subdomain count: {subdomains_count}".format(subdomains_count=len(censys_result)))
total_subs = set(subs + dnspod_domains + list(subdomains) + transfer_info + ca_subdomains + list(shodan_result) + fofa_result + zoomeye_result + censys_result)
# Use TXT,SOA,MX,AAAA record to find sub domains
if self.multiresolve:
logger.info('Enumerating subdomains with TXT, SOA, MX, AAAA record...')
dnsquery = DNSQuery(self.domain, total_subs, self.domain)
record_info = dnsquery.dns_query()
tasks = (self.query(record[:record.find('.')]) for record in record_info)
self.loop.run_until_complete(self.start(tasks, len(record_info)))
logger.info('DNS record subdomain count: {c}'.format(c=len(record_info)))
if self.is_wildcard_domain and not self.skip_rsc:
# Response similarity comparison
total_subs = set(subs + dnspod_domains + list(subdomains) + transfer_info + ca_subdomains)
self.wildcard_subs = list(set(subs).union(total_subs))
logger.info('Enumerates {len} sub domains by DNS mode in {tcd}.'.format(len=len(self.data), tcd=str(datetime.timedelta(seconds=time_consume_dns))))
logger.info('Will continue to test the distinct({len_subs}-{len_exist})={len_remain} domains used by RSC, the speed will be affected.'.format(len_subs=len(subs), len_exist=len(self.data),
len_remain=len(self.wildcard_subs)))
self.coroutine_count = self.coroutine_count_request
self.remainder = len(self.wildcard_subs)
tasks = (self.similarity(sub) for sub in self.wildcard_subs)
self.loop.run_until_complete(self.start(tasks, len(self.wildcard_subs)))
# Distinct last domains use RSC
# Maybe misinformation
# self.distinct()
time_consume_request = int(time.time() - dns_time)
logger.info('Requests time consume {tcr}'.format(tcr=str(datetime.timedelta(seconds=time_consume_request))))
# RS(redirect/response) domains
while len(self.domains_rs) != 0:
logger.info('RS(redirect/response) domains({l})...'.format(l=len(self.domains_rs)))
tasks = (self.similarity(''.join(domain.rsplit(self.domain, 1)).rstrip('.')) for domain in self.domains_rs)
self.loop.run_until_complete(self.start(tasks, len(self.domains_rs)))
# write output
tmp_dir = '/tmp/esd'
if not os.path.isdir(tmp_dir):
os.mkdir(tmp_dir, 0o777)
output_path_with_time = '{td}/.{domain}_{time}.esd'.format(td=tmp_dir, domain=self.domain, time=datetime.datetime.now().strftime("%Y-%m_%d_%H-%M"))
output_path = '{td}/.{domain}.esd'.format(td=tmp_dir, domain=self.domain)
if len(self.data):
max_domain_len = max(map(len, self.data)) + 2
else:
max_domain_len = 2
output_format = '%-{0}s%-s\n'.format(max_domain_len)
with open(output_path_with_time, 'w') as opt, open(output_path, 'w') as op:
for domain, ips in self.data.items():
# The format is consistent with other scanners to ensure that they are
# invoked at the same time without increasing the cost of
# resolution
if ips is None or len(ips) == 0:
ips_split = ''
else:
ips_split = ','.join(ips)
con = output_format % (domain, ips_split)
op.write(con)
opt.write(con)
logger.info('Output: {op}'.format(op=output_path))
logger.info('Output with time: {op}'.format(op=output_path_with_time))
logger.info('Total domain: {td}'.format(td=len(self.data)))
time_consume = int(time.time() - start_time)
logger.info('Time consume: {tc}'.format(tc=str(datetime.timedelta(seconds=time_consume))))
return self.data
def banner():
print("""\033[94m
______ _____ _____
| ____| / ____| | __ \
| |__ | (___ | | | |
| __| \___ \ | | | |
| |____ ____) | | |__| |
|______| |_____/ |_____/\033[0m\033[93m
# Enumeration sub domains @version: %s\033[92m
""" % __version__)
def main():
banner()
parser = OptionParser('Usage: python ESD.py -d feei.cn -F response_filter -e baidu,google,bing,yahoo -p user:pass@host:port')
parser.add_option('-d', '--domain', dest='domains', help='The domains that you want to enumerate')
parser.add_option('-f', '--file', dest='input', help='Import domains from this file')
parser.add_option('-F', '--filter', dest='filter', help='Response filter')
parser.add_option('-s', '--skip-rsc', dest='skiprsc', help='Skip response similary compare', action='store_true', default=False)
parser.add_option('-e', '--engines', dest='engines', help='Choose an engine in baidu,google,bing or yahoo, split with ","')
parser.add_option('-S', '--split', dest='split', help='Split the dict into several parts', default='1/1')
parser.add_option('-p', '--proxy', dest='proxy', help='Use socks5 proxy to access Google and Yahoo')
parser.add_option('-m', '--multi-resolve', dest='multiresolve', help='Use TXT, AAAA, MX, SOA record to find subdomains', action='store_true', default=False)
parser.add_option('--skey', '--shodan-key', dest='shodankey', help='Define the api of shodan')
parser.add_option('--fkey', '--fofa-key', dest='fofakey', help='Define the key of fofa')
parser.add_option('--femail', '--fofa-email', dest='fofaemail', help='The email of your fofa account')
parser.add_option('--zusername', '--zoomeye-username', dest='zoomeyeusername', help='The username of your zoomeye account')
parser.add_option('--zpassword', '--zoomeye-password', dest='zoomeyepassword', help='The password of your zoomeye account')
parser.add_option('--cuid', '--censys-uid', dest='censysuid', help="The uid of your censys account")
parser.add_option('--csecret', '--censys-secret', dest='censyssecret', help='The secret of your censys account')
(options, args) = parser.parse_args()
support_engines = {
'baidu': Baidu,
'google': Google,
'bing': Bing,
'yahoo': Yahoo,
}
domains = []
engines = []
response_filter = options.filter
skip_rsc = options.skiprsc
split_list = options.split.split('/')
split = options.split
multiresolve = options.multiresolve
skey = options.shodankey
fofa_struct = {
'fkey': options.fofakey,
'femail': options.fofaemail,
}
zoomeye_struct = {
'username': options.zoomeyeusername,
'password': options.zoomeyepassword,
}
censys_struct = {
'uid': options.censysuid,
'secret': options.censyssecret,
}
try:
if len(split_list) != 2 or int(split_list[0]) > int(split_list[1]):
logger.error('Invaild split parameter,can not split the dict')
split = None
except:
logger.error('Split validation failed: {d}'.format(d=split_list))
exit(0)
if options.proxy:
proxy = {
'http': 'socks5h://%s' % options.proxy,
'https': 'socks5h://%s' % options.proxy
}
else:
proxy = {}
if options.engines:
for engine in options.engines.split(','):
if engine.lower() in support_engines:
engines.append(support_engines[engine])
else:
engines = [Baidu, Google, Bing, Yahoo]
if options.domains is not None:
for p in options.domains.split(','):
p = p.strip().lower()
re_domain = re.findall(r'^(([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,})$', p)
if len(re_domain) > 0 and re_domain[0][0] == p and tldextract.extract(p).suffix != '':
domains.append(p.strip())
else:
logger.error('Domain validation failed: {d}'.format(d=p))
elif options.input and os.path.isfile(options.input):
with open(options.input) as fh:
for line_domain in fh:
line_domain = line_domain.strip().lower()
re_domain = re.findall(r'^(([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,})$', line_domain)
if len(re_domain) > 0 and re_domain[0][0] == line_domain and tldextract.extract(line_domain).suffix != '':
domains.append(line_domain)
else:
logger.error('Domain validation failed: {d}'.format(d=line_domain))
else:
logger.error('Please input vaild parameter. ie: "esd -d feei.cn" or "esd -f /Users/root/domains.txt"')
if 'esd' in os.environ:
debug = os.environ['esd']
else:
debug = False
logger.info('Debug: {d}'.format(d=debug))
logger.info('--skip-rsc: {rsc}'.format(rsc=skip_rsc))
logger.info('Total target domains: {ttd}'.format(ttd=len(domains)))
try:
for d in domains:
esd = EnumSubDomain(d, response_filter, skip_rsc=skip_rsc, debug=debug, split=split, engines=engines, proxy=proxy,
multiresolve=multiresolve, shodan_key=skey, fofa=fofa_struct, zoomeye=zoomeye_struct, censys=censys_struct)
esd.run()
except KeyboardInterrupt:
print('', end='\n')
logger.info('Bye :)')
exit(0)
if __name__ == '__main__':
main()
