import flask import flask.json import common.url import common.rpc from www import server from www import login from www import history import re import datetime import pytz import asyncio import sqlalchemy @server.app.route('/spam') @login.require_mod async def spam(session): link_spam = "link_spam" in flask.request.values data = await common.rpc.bot.get_data('link_spam_rules' if link_spam else 'spam_rules') return flask.render_template("spam.html", rules=data, link_spam=link_spam, session=session) def verify_rules(rules): for ix, rule in enumerate(rules): # Test the regular expression is valid try: re_rule = re.compile(rule['re']) except re.error as ex: return {"msg": str(ex), "row": ix, "col": 0} # Test the response message uses the right groups try: rule['message'] % {str(i + 1): "" for i in range(re_rule.groups)} except KeyError as ex: return {"msg": "No group %s" % ex, "row": ix, "col": 1} except TypeError: return {"msg": "Must use named placeholders", "row": ix, "col": 1} # Check the type setting if rule['type'] not in ('spam', 'censor'): return {"msg": "Incorrect type", "row": ix, "col": 2} @server.app.route('/spam/submit', methods=['POST']) @login.require_mod async def spam_submit(session): link_spam = "link_spam" in flask.request.values data = flask.json.loads(flask.request.values['data']) # Validation checks error = verify_rules(data) if error: return flask.json.jsonify(error=error, csrf_token=server.app.csrf_token()) if link_spam: await common.rpc.bot.link_spam.modify_link_spam_rules(data) else: await common.rpc.bot.spam.modify_spam_rules(data) history.store("link_spam" if link_spam else "spam", session['user']['id'], data) return flask.json.jsonify(success='OK', csrf_token=server.app.csrf_token()) def do_check(line, rules): for rule in rules: matches = rule['re'].search(line) if matches: groups = {str(i+1):v for i,v in enumerate(matches.groups())} return rule['message'] % groups return None async def do_check_links(message, rules): re_url = await common.url.url_regex() urls = [] for match in re_url.finditer(message): for url in match.groups(): if url is not None: urls.append(url) break canonical_urls = await asyncio.gather(*map(common.url.canonical_url, urls)) for url_chain in canonical_urls: for url in url_chain: for rule in rules: match = rule["re"].search(url) if match is not None: return rule["message"] % {str(i+1): v for i, v in enumerate(match.groups())} @server.app.route('/spam/redirects') @login.require_mod async def spam_redirects(session): redirects = await common.url.canonical_url(flask.request.values["url"].strip()) return flask.json.jsonify(redirects=redirects, csrf_token=server.app.csrf_token()) @server.app.route('/spam/test', methods=['POST']) @login.require_mod async def spam_test(session): link_spam = "link_spam" in flask.request.values rules = flask.json.loads(flask.request.values['data']) message = flask.request.values['message'] # Validation checks error = verify_rules(rules) if error: return flask.json.jsonify(error=error, csrf_token=server.app.csrf_token()) for rule in rules: rule['re'] = re.compile(rule['re']) result = [] check = do_check_links if link_spam else asyncio.coroutine(do_check) re_twitchchat = re.compile("^\w*:\s*(.*)$") re_irc = re.compile("<[^<>]*>\s*(.*)$") lines = message.split('\n') for line in lines: res = await check(line, rules) if res is None: match = re_twitchchat.search(line) if match: res = await check(match.group(1), rules) if res is None: match = re_irc.search(line) if match: res = await check(match.group(1), rules) if res is not None: result.append({ 'line': line, 'spam': True, 'message': res, }) else: result.append({ 'line': line, 'spam': False, }) return flask.json.jsonify(result=result, csrf_token=server.app.csrf_token()) @server.app.route('/spam/find') @login.require_mod async def spam_find(session): rules = await common.rpc.bot.get_data('spam_rules') for rule in rules: rule['re'] = re.compile(rule['re']) starttime = datetime.datetime.now(tz=pytz.utc) - datetime.timedelta(days=14) log = server.db.metadata.tables["log"] with server.db.engine.begin() as conn: res = conn.execute(sqlalchemy.select([log.c.source, log.c.message, log.c.time]) .where((log.c.time >= starttime) & log.c.specialuser.any('cleared')) .order_by(log.c.time.asc())) data = [tuple(row) + (do_check(row[1], rules),) for row in res] return flask.render_template("spam_find.html", data=data, session=session)