# # Copyright (c) 2013-2015 Luigi Mori <l@isidora.org> # # Permission to use, copy, modify, and distribute this software for any # purpose with or without fee is hereby granted, provided that the above # copyright notice and this permission notice appear in all copies. # # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR # ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # # Quick and dirty hub of public ip lists for Palo Alto Networks devices import webapp2 import logging import address import types import os from google.appengine.ext.webapp import template from google.appengine.api import memcache from google.appengine.api import urlfetch from google.appengine.ext import db import lxml.html import lxml.etree class BlockList(db.Model): """DB model of iplist""" tag = db.StringProperty(required=True) time = db.DateTimeProperty(auto_now=True) iplist = db.StringListProperty(indexed=False, required=True) class BlockListJob(webapp2.RequestHandler): """Base class for block list update jobs get method is periodically called by GAE as configured in cron.yml, unauthorized direct access is prevented via app.yaml """ url = "" tag = "BlockListJobERROR" def handle_line(self, line): return address.create_address(line) def get(self): """Convert ip list retrieved from self.url into a list of IPs/NETs/RANGEs using address.optimize_list. The resulting iplist is then stored in the GAE DataStore with tag self.tag. Memcache entry for the iplist is invalidated. """ blist = urlfetch.fetch(self.url, deadline=60) if blist.status_code != 200: logging.error(self.tag+" returned: "+str(blist.status_code)) self.response.headers['Content-Type'] = 'text/plain' self.response.write(self.tag+" returned: "+str(blist.status_code)) return nets = set() for l in blist.content.splitlines(): l = l.strip() if l == None or len(l) == 0: continue addr = self.handle_line(l) if addr == None: continue if type(addr) != types.ListType: addr = [addr] for a in addr: if not a in nets: nets.add(a) nets = address.optimize_list([ x for x in nets ]) self.response.headers['Content-Type'] = 'text/plain' self.response.write(self.tag+" list\n"+"# entries: "+str(len(nets))+"\n\n"+'\n'.join(map(repr, nets))) dblist = BlockList(tag=self.tag, iplist=map(repr, nets)) dblist.put() memcache.delete("l"+self.tag) memcache.delete("t"+self.tag) logging.info(self.tag+" refreshed") class SpamhausList(BlockListJob): """Base BlockListJob class for Spamhaus Lists""" url = "" tag = "SpamhausListERROR" def handle_line(self, line): if line[0] == ";": return None net = line.partition(";")[0] return address.create_address(net) class SpamhausDrop(SpamhausList): url = "http://www.spamhaus.org/drop/drop.txt" tag = "SpamhausDROP" class SpamhausEDrop(SpamhausList): url = "http://www.spamhaus.org/drop/edrop.txt" tag = "SpamhausEDROP" class OpenBLIpList(BlockListJob): url = "http://www.openbl.org/lists/base.txt" tag = "OpenBLIpList" def handle_line(self, line): if line[0] == '#': return None return address.create_address(line) class MalwareDomainList(BlockListJob): url = "http://www.malwaredomainlist.com/hostslist/ip.txt" tag = "MalwareDomainList" class EmergingThreatsCompromisedList(BlockListJob): url = "http://rules.emergingthreats.net/open/suricata/rules/compromised-ips.txt" tag = "EmergingThreatsCompromisedList" class BruteForceBlockerList(BlockListJob): url = "http://danger.rulez.sk/projects/bruteforceblocker/blist.php" tag = "BruteForceBlockerList" def handle_line(self, line): if line[0] == '#': return None return address.create_address(line) class SnortRules(BlockListJob): """Base BlockListJob class for simple SnortRules block lists""" url = "" tag = "SnortRulesERROR" def handle_line(self, line): if line[0] == '#': return None source = line.split(None, 2)[2] if source[0] == '[': source = source[1:source.index(']')] source = source.split(',') else: source = [source] result = [] for a in source: try: result.append(address.create_address(a)) except: logging.exception('Error decoding: {!r}'.format(a)) pass return result class EmergingThreatsRBN(SnortRules): url = "http://rules.emergingthreats.net/blockrules/emerging-rbn-BLOCK.rules" tag = "EmergingThreatsRBN" class EmergingThreatsTOR(SnortRules): url = "http://rules.emergingthreats.net/open/suricata/rules/tor.rules" tag = "EmergingThreatsTOR" class DshieldBlockList(BlockListJob): url = "http://feeds.dshield.org/block.txt" tag = "DshieldBlockList" def handle_line(self, line): if line[0] == '#': return None if line[0] == 'S': return None start, end = line.split()[:2] return address.create_address(start+"-"+end) class SSLAbuseIPList(BlockListJob): url = "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv" tag = "SSLAbuseIPList" def handle_line(self, line): if line[0] == "#": return None net = line.split(",")[0] return address.create_address(net) class ZeusTrackerBadIPsList(BlockListJob): url = "https://zeustracker.abuse.ch/blocklist.php?download=badips" tag = "ZeusTrackerBadIPsList" def handle_line(self, line): if line[0] == '#': return None return address.create_address(line) class TalosIntelIPFilter(BlockListJob): url = "http://talosintel.com/feeds/ip-filter.blf" tag = "TalosIntelIPFilter" class Office365NetBlocks(webapp2.RequestHandler): """Not used, MS web page is unreliable, includes ProPlus, Office Online and RCA""" url = "https://support.content.office.net/en-us/static/O365IPAddresses.xml" tag = "Office365NetBlocks" def get(self): page = urlfetch.fetch(self.url, deadline=60) if page.status_code != 200: logging.error(self.tag+" returned: "+str(page.status_code)) self.response.headers['Content-Type'] = 'text/plain' self.response.write(self.tag+" returned: "+str(page.status_code)) return ltree = lxml.etree.fromstring(page.content) nets = set() ipv4 = ltree.xpath("/products/product[@name='o365']/addresslist[@type='IPv4']/address") for a in ipv4: addr = address.create_address(a.text) if addr == None: continue if type(addr) != types.ListType: addr = [addr] for a in addr: if not a in nets: nets.add(a) ipv4 = ltree.xpath("/products/product[@name='ProPlus']/addresslist[@type='IPv4']/address") for a in ipv4: addr = address.create_address(a.text) if addr == None: continue if type(addr) != types.ListType: addr = [addr] for a in addr: if not a in nets: nets.add(a) ipv4 = ltree.xpath("/products/product[@name='RCA']/addresslist[@type='IPv4']/address") for a in ipv4: addr = address.create_address(a.text) if addr == None: continue if type(addr) != types.ListType: addr = [addr] for a in addr: if not a in nets: nets.add(a) ipv4 = ltree.xpath("/products/product[@name='WAC']/addresslist[@type='IPv4']/address") for a in ipv4: addr = address.create_address(a.text) if addr == None: continue if type(addr) != types.ListType: addr = [addr] for a in addr: if not a in nets: nets.add(a) nets = address.optimize_list([ x for x in nets ]) self.response.headers['Content-Type'] = 'text/plain' self.response.write(self.tag+" list\n"+"# entries: "+str(len(nets))+"\n\n"+'\n'.join(map(repr, nets))) dblist = BlockList(tag=self.tag, iplist=map(repr, nets)) dblist.put() memcache.delete("l"+self.tag) memcache.delete("t"+self.tag) logging.info(self.tag+" refreshed") class ExchangeOnlineNetBlocks(webapp2.RequestHandler): """Not used, MS web page is unreliable, includes Exchange Online Protection""" url = "https://support.content.office.net/en-us/static/O365IPAddresses.xml" tag = "ExchangeOnlineNetBlocks" def get(self): page = urlfetch.fetch(self.url, deadline=60) if page.status_code != 200: logging.error(self.tag+" returned: "+str(page.status_code)) self.response.headers['Content-Type'] = 'text/plain' self.response.write(self.tag+" returned: "+str(page.status_code)) return ltree = lxml.etree.fromstring(page.content) nets = set() ipv4 = ltree.xpath("/products/product[@name='EXO']/addresslist[@type='IPv4']/address") for a in ipv4: addr = address.create_address(a.text) if addr == None: continue if type(addr) != types.ListType: addr = [addr] for a in addr: if not a in nets: nets.add(a) ipv4 = ltree.xpath("/products/product[@name='EOP']/addresslist[@type='IPv4']/address") for a in ipv4: addr = address.create_address(a.text) if addr == None: continue if type(addr) != types.ListType: addr = [addr] for a in addr: if not a in nets: nets.add(a) nets = address.optimize_list([ x for x in nets ]) self.response.headers['Content-Type'] = 'text/plain' self.response.write(self.tag+" list\n"+"# entries: "+str(len(nets))+"\n\n"+'\n'.join(map(repr, nets))) dblist = BlockList(tag=self.tag, iplist=map(repr, nets)) dblist.put() memcache.delete("l"+self.tag) memcache.delete("t"+self.tag) logging.info(self.tag+" refreshed") class LyncOnlineNetBlocks(webapp2.RequestHandler): """Not used, MS web page is unreliable, includes Skype for Business Online""" url = "https://support.content.office.net/en-us/static/O365IPAddresses.xml" tag = "LyncOnlineNetBlocks" def get(self): page = urlfetch.fetch(self.url, deadline=60) if page.status_code != 200: logging.error(self.tag+" returned: "+str(page.status_code)) self.response.headers['Content-Type'] = 'text/plain' self.response.write(self.tag+" returned: "+str(page.status_code)) return ltree = lxml.etree.fromstring(page.content) nets = set() ipv4 = ltree.xpath("/products/product[@name='LYO']/addresslist[@type='IPv4']/address") for a in ipv4: addr = address.create_address(a.text) if addr == None: continue if type(addr) != types.ListType: addr = [addr] for a in addr: if not a in nets: nets.add(a) nets = address.optimize_list([ x for x in nets ]) self.response.headers['Content-Type'] = 'text/plain' self.response.write(self.tag+" list\n"+"# entries: "+str(len(nets))+"\n\n"+'\n'.join(map(repr, nets))) dblist = BlockList(tag=self.tag, iplist=map(repr, nets)) dblist.put() memcache.delete("l"+self.tag) memcache.delete("t"+self.tag) logging.info(self.tag+" refreshed") class SharepointOnlineNetBlocks(webapp2.RequestHandler): """Not used, MS web page is unreliable""" url = "https://support.content.office.net/en-us/static/O365IPAddresses.xml" tag = "SharepointOnlineNetBlocks" def get(self): page = urlfetch.fetch(self.url, deadline=60) if page.status_code != 200: logging.error(self.tag+" returned: "+str(page.status_code)) self.response.headers['Content-Type'] = 'text/plain' self.response.write(self.tag+" returned: "+str(page.status_code)) return ltree = lxml.etree.fromstring(page.content) nets = set() ipv4 = ltree.xpath("/products/product[@name='SPO']/addresslist[@type='IPv4']/address") for a in ipv4: addr = address.create_address(a.text) if addr == None: continue if type(addr) != types.ListType: addr = [addr] for a in addr: if not a in nets: nets.add(a) nets = address.optimize_list([ x for x in nets ]) self.response.headers['Content-Type'] = 'text/plain' self.response.write(self.tag+" list\n"+"# entries: "+str(len(nets))+"\n\n"+'\n'.join(map(repr, nets))) dblist = BlockList(tag=self.tag, iplist=map(repr, nets)) dblist.put() memcache.delete("l"+self.tag) memcache.delete("t"+self.tag) logging.info(self.tag+" refreshed") class GetBlockList(webapp2.RequestHandler): """Base class for ip list retrive requests""" tag = "GetBlockListERROR" copyright = None def __get_iplist(self): iplist = memcache.get("l"+self.tag) if iplist is not None: ipltime = memcache.get("t"+self.tag) return ipltime, iplist q = BlockList.all() q.filter("tag =", self.tag) q.order("-time") iplist = q.get() if iplist is None: iplist = [] ipltime = None else: ipltime = iplist.time.strftime('%d %b %Y %H:%M %Z') iplist = iplist.iplist memcache.set("t"+self.tag, ipltime, 60*60*24) memcache.set("l"+self.tag, iplist, 60*60*24) return ipltime, iplist def get(self): ipltime, iplist = self.__get_iplist() n = self.request.get('n', None) if n is None: n = len(iplist) else: try: n = int(n) except: logging.error("Invalid value for n: %s", n) n = len(iplist) if n < 0: logging.error("Invalid value for n: %d", n) n = len(iplist) s = self.request.get('s', None) if s is None: s = 0 else: try: s = int(s) except: logging.error("Invalid value for s: %s", s) s = 0 if s < 0: logging.error("Invalid value for s: %s", s) s = 0 self.response.headers['Content-Type'] = 'text/plain' if ipltime is not None or self.copyright is not None: header = '#' if self.copyright is not None: header += ' '+self.copyright if ipltime is not None: header += ' Retrieved: %s'%ipltime self.response.write('%s\n' % header) self.response.write('\n'.join(iplist[s:s+n])) class GetEmergingThreatsRBN(GetBlockList): tag = "EmergingThreatsRBN" class GetSpamhausDrop(GetBlockList): copyright = "Spamhaus DROP (c) Do not use after 2 days." tag = "SpamhausDROP" class GetSpamhausEDrop(GetBlockList): copyright = "Spamhaus EDROP (c) Do not use after 2 days." tag = "SpamhausEDROP" class GetOpenBLIpList(GetBlockList): tag = "OpenBLIpList" class GetMalwareDomainList(GetBlockList): tag = "MalwareDomainList" class GetEmergingThreatsCompromisedList(GetBlockList): tag = "EmergingThreatsCompromisedList" class GetBruteForceBlockerList(GetBlockList): tag = "BruteForceBlockerList" class GetEmergingThreatsTOR(GetBlockList): tag = "EmergingThreatsTOR" class GetDshieldBlockList(GetBlockList): tag = "DshieldBlockList" class GetGoogleNetBlocks(GetBlockList): tag = "GoogleNetBlocks" class GetSSLAbuseIPList(GetBlockList): tag = "SSLAbuseIPList" class GetZeusTrackerBadIPsList(GetBlockList): tag = "ZeusTrackerBadIPsList" class GetTalosIntelIPFilter(GetBlockList): tag = "TalosIntelIPFilter" class MainPage(webapp2.RequestHandler): """Main page renderer""" def __get_iplist_info(self, tag): ipltime = memcache.get("t"+tag) iplist = memcache.get("l"+tag) if (ipltime is not None) and (iplist is not None): iplist = len(iplist) return ipltime, iplist else: q = BlockList.all() q.filter("tag =", tag) q.order("-time") iplist = q.get() if iplist is None: return '--', '--' else: ipltime = iplist.time.strftime('%d %b %Y %H:%M %Z') memcache.set_multi({"t"+tag: ipltime, "l"+tag: iplist.iplist}, 60*60*24) return ipltime, len(iplist.iplist) def get(self): tags = ["EmergingThreatsRBN", "SpamhausDROP", "SpamhausEDROP", "OpenBLIpList", "MalwareDomainList", "EmergingThreatsCompromisedList", "BruteForceBlockerList", "EmergingThreatsTOR", "DshieldBlockList", "SSLAbuseIPList", "ZeusTrackerBadIPsList"] template_values = {} for t in tags: ipld, ipln = self.__get_iplist_info(t) template_values['n'+t] = ipln template_values['d'+t] = ipld path = os.path.join(os.path.dirname(__file__), 'index.html') self.response.out.write(template.render(path, template_values)) app = webapp2.WSGIApplication([('/', MainPage), ('/jobs/shdropjob', SpamhausDrop), ('/jobs/shedropjob', SpamhausEDrop), ('/jobs/mdljob', MalwareDomainList), # ('/jobs/openbljob', OpenBLIpList), ('/jobs/bruteforceblockerjob', BruteForceBlockerList), ('/jobs/etrbnjob', EmergingThreatsRBN), ('/jobs/ettorjob', EmergingThreatsTOR), ('/jobs/etcompromisedjob', EmergingThreatsCompromisedList), ('/jobs/dshieldbljob', DshieldBlockList), ('/jobs/sslabuseiplistjob', SSLAbuseIPList), ('/jobs/zeustrackerbadipsjob', ZeusTrackerBadIPsList), # ('/jobs/office365netblocksjob', Office365NetBlocks), # ('/jobs/exchangeonlinenetblocksjob', ExchangeOnlineNetBlocks), # ('/jobs/lynconlinenetblocksjob', LyncOnlineNetBlocks), # ('/jobs/sharepointonlinenetblocksjob', SharepointOnlineNetBlocks), ('/jobs/talosintelipfilterjob', TalosIntelIPFilter), ('/lists/shdrop.txt', GetSpamhausDrop), ('/lists/shedrop.txt', GetSpamhausEDrop), ('/lists/mdl.txt', GetMalwareDomainList), # ('/lists/openbl.txt', GetOpenBLIpList), ('/lists/bruteforceblocker.txt', GetBruteForceBlockerList), ('/lists/etrbn.txt', GetEmergingThreatsRBN), ('/lists/ettor.txt', GetEmergingThreatsTOR), ('/lists/etcompromised.txt', GetEmergingThreatsCompromisedList), ('/lists/dshieldbl.txt', GetDshieldBlockList), ('/lists/sslabuseiplist.txt', GetSSLAbuseIPList), ('/lists/zeustrackerbadips.txt', GetZeusTrackerBadIPsList), # ('/lists/office365netblocks.txt', GetOffice365NetBlocks), # ('/lists/exchangeonlinenetblocks.txt', GetExchangeOnlineNetBlocks), # ('/lists/lynconlinenetblocks.txt', GetLyncOnlineNetBlocks), # ('/lists/sharepointonlinenetblocks.txt', GetSharepointOnlineNetBlocks), ('/lists/talosintelipfilter.txt', GetTalosIntelIPFilter) ], debug=False)