Python validators.url() Examples

def __init__(self, url, max_worker=10, timeout=3,
                 scan_dict=None, verbose=False, status=None):
        self.site_lang = ''
        self.raw_base_url = url
        self.base_url = url
        self.max_worker = max_worker
        self.timeout = timeout
        self.scan_dict = scan_dict
        self.verbose = verbose
        self.first_item = ''
        self.dict_data = {}
        self.first_queue = []
        self.found_items = {}
        if status is None or len(status) == 0:
            self.status = [200, 301, 302, 304, 401, 403]
        else:
            self.status = [int(t) for t in status] 

def on_response(self, url, item, method, response, queue):
        if response.code in self.status:
            if item in self.found_items:
                return
            self.found_items[item] = None
            logger.warning('[Y] %s %s %s' % (response.code, method, url))
            # 自动对找到的代码文件扫描编辑器生成的备份文件
            if any(map(item.endswith, ['.php', '.asp', '.jsp'])):
                bak_list = self.make_bak_file_list(item)
                bak_list = [(t, 'HEAD') for t in bak_list]
                queue.extendleft(bak_list)
        else:
            if response.code == 405 and method != 'POST':
                queue.appendleft((item, 'POST'))

            if self.verbose:
                logger.info('[N] %s %s %s' % (response.code, method, url)) 

def hashchecking( ):

    print ("\n")
    print(mycolors.reset)
    print("Main Antivirus Reports")
    print("-" * 25 + "\n")

    vtresult = vtshow(hashtemp,url,param)

    if vtresult == 'Not Found':
        if(bkg == 1):
            print(mycolors.foreground.lightred + "Malware sample was not found in Virus Total.")
        else:
            print(mycolors.foreground.red + "Malware sample was not found in Virus Total.")

    print(mycolors.reset)

    hashow(hashtemp)
    if (down == 1):
         downhash(hashtemp)
    print(mycolors.reset)
    exit(0) 

def auth_login(self, password, api_key):
        self.username = ''
        self.api_key = api_key
        self.password = password
        url = self.server_url + self.routes['auth_login']
        payload = {'api_key': str(api_key), 'pass1': str(password)}
        ret = self.get_info()  # Check SSL certificate
        if not ret[0]:
            return ret
        r = self.s.post(url, data=payload)
        if r.ok:
            self.auth_string = 'Bearer ' + r.json()['access_token']
            self.token_start_time = time.time()
            return True, self.auth_string
        else:
            return False, '{}: {} - {}'.format(r.status_code, r.reason, r.text[0:1024]) 

def clean_data(self, dataset_name, **kwargs):
        url = self.server_url + self.routes['clean_data'] + urllib.parse.quote(dataset_name, safe='')
        headers = self.get_auth_header()
        parameters = kwargs
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        r = self.s.post(url, headers=headers, json=parameters)
        if not r.ok and 'Please run analyze data' in r.text:
            print("Raw profile not found. Running analyze_data")
            char_encoding = parameters['char_encoding'] if 'char_encoding' in parameters else 'utf-8'
            r = self.analyze_data(dataset_name, char_encoding=char_encoding)
            if r[0]:
                r = self.s.post(url, headers=headers, json=parameters)
            else:
                return r
        return self.get_return_info(r)

    # Create risk information for a datatset 

def validate_repo_url(url):
    """
    Validates and formats `url` to be valid URL pointing to a repo on bitbucket.org or github.com
    :param url: str, URL
    :return: str, valid URL if valid repo, emptry string otherwise
    """
    try:
        if "github.com" in url:
            return re.findall(r"https?://w?w?w?.?github.com/[\w\-]+/[\w.-]+", url)[0]
        elif "bitbucket.org" in url:
            return re.findall(r"https?://bitbucket.org/[\w.-]+/[\w.-]+", url)[0] + "/src/"
        elif "launchpad.net" in url:
            return re.findall(r"https?://launchpad.net/[\w.-]+", url)[0]
        elif "sourceforge.net" in url:
            mo = re.match(r"https?://sourceforge.net/projects/"
                          r"([\w.-]+)/", url, re.I)
            template = "https://sourceforge.net/p/{}/code/HEAD/tree/trunk/src/"
            return template.format(mo.groups()[0])
    except (IndexError, AttributeError):
        pass
    return "" 

def find_release_page(session, repo_url):
    if "github.com" in repo_url:
        logger.debug("Unable to find changelog on {}, try release page".format(repo_url))
        try:
            username, reponame = repo_url.split("/")[3:5]
            # try to fetch the release page. if it 200s, yield the release page
            # api URL for further processing
            resp = session.get("https://github.com/{username}/{reponame}/releases".format(
                username=username, reponame=reponame
            ))
            if resp.status_code == 200:
                yield "https://api.github.com/repos/{username}/{reponame}/releases".format(
                    username=username, reponame=reponame
                )
        except IndexError:
            logger.debug("Unable to construct releases url for {}".format(repo_url)) 

def upload_dataset(self, dataset_path, dataset_name=None, has_header=True):
        if dataset_name is None:
            head, tail = os.path.split(dataset_path)
            dataset_name = tail
            # dataset_name = dataset_path.split('/')[-1]
        url = self.server_url + self.routes['upload_dataset']
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        if not os.path.isfile(dataset_path):
            return False, "File not found"
        with open(dataset_path, 'rb') as f:
            form = encoder.MultipartEncoder({
                "dataset": (str(dataset_path), f, 'text/csv/h5'),
                'dataset_name': str(dataset_name),
                'has_header': str(has_header)
            })
            headers.update({"Prefer": "respond-async", "Content-Type": form.content_type})
            r = self.s.post(url, headers=headers, data=form)
        return self.get_return_info(r) 

def lookup_artifact(self, type=None):
        url = self.server_url + self.routes['lookup_artifact']
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        payload = {'type': type}
        r = self.s.get(url, headers=headers, params=payload)
        return self.get_return_info(r) 

def validate_result(current, default, type):
    """
    Validates the data, whether it needs to be url, twitter, linkedin link etc.
    """
    if current is None:
        current = ""
    if default is None:
        default = ""
    if type == "URL" and validators.url(current, require_tld=True) and not validators.url(default, require_tld=True):
        return current
    if type == "EMAIL" and validators.email(current) and not validators.email(default):
        return current
    return default 

def lookup_artifact_name(self, artifact_name):
        url = self.server_url + self.routes['lookup_artifact_name']
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        r = self.s.get(url + urllib.parse.quote(artifact_name, safe=''), headers=headers)
        return self.get_return_info(r) 

def lookup_limits(self):
        url = self.server_url + self.routes['lookup_limits']
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        r = self.s.get(url, headers=headers)
        return self.get_return_info(r) 

def create_risk_info(self, failure_data, timeseries_data, job_name=None, artifact_name=None, **kwargs):
        url = self.server_url + self.routes['create_risk_info'] + failure_data + '/' + timeseries_data
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        parameters = kwargs
        if 'job_name' not in parameters and job_name is not None:
            parameters['job_name'] = job_name
        if 'artifact_name' not in parameters and artifact_name is not None:
            parameters['artifact_name'] = artifact_name
        r = self.s.post(url, headers=headers, json=parameters)
        return self.get_return_info(r)

    # Run a model on some dataset 

def delete_artifact(self, artifact_name):
        url = self.server_url + self.routes['delete_artifact'] + urllib.parse.quote(artifact_name, safe='')
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        r = self.s.delete(url, headers=headers)
        return self.get_return_info(r) 

def download_model(self, model_name, path=None, model_type=None, model_format=None):
        """
        Download a model and data profiler given a model_name and location
        If location is not supplied, it will download to the current directory
        :param model_name: Model name to download
        :param path: Path where the model and data profiler are supposed to be downloaded
        :param model_type: Model type of the model
        :param model_format: Format of the model to be downloaded
        :return: Response if the download was successful or not
        """
        headers = {'Authorization': self.auth_string}
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        url = self.server_url + self.routes['download_model'] + urllib.parse.quote(model_name, safe='')
        payload = {}
        if model_type:
            payload['model_type'] = model_type
        if model_format:
            payload['model_format'] = model_format
        r = self.s.get(url, headers=headers, stream=True, params=payload)
        try:
            z = zipfile.ZipFile(io.BytesIO(r.content))
            z.extractall(path=path)
        except:
            return False, "Error while downloading models"
        return True, None

    # Analyze a model or data set 

def analyze_data(self, dataset_name, **kwargs):
        url = self.server_url + self.routes['analyze_data'] + urllib.parse.quote(dataset_name, safe='')
        headers = self.get_auth_header()
        parameters = kwargs
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        r = self.s.post(url, headers=headers, json=parameters)
        return self.get_return_info(r)

    # Analyze global feature importances 

def analyze_model(self, model_name, job_name=None, artifact_name=None, category_name=None, model_type=None):
        url = self.server_url + self.routes['analyze_model'] + urllib.parse.quote(model_name, safe='')
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        payload = {'job_name': job_name, 'artifact_name': artifact_name,
                   'category_name': category_name, 'model_type': model_type}
        r = self.s.post(url, headers=headers, data=payload)
        return self.get_return_info(r)

    # Analyze sample-wise feature importances 

def evaluate_model(self, dataset_name, model_name):
        # replace v1 with v2
        assert self.server_url.endswith('/v1/')
        url = self.server_url.replace('/v1/', '/v2/') + 'model/' + model_name + '/evaluate'
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        parameters = {'dataset_ids': [dataset_name]}
        r = self.s.post(url, headers=headers, json=parameters)
        return self.get_return_info(r)

    # Interactive help 

def lookup_model(self):
        url = self.server_url + self.routes['lookup_model']
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        r = self.s.get(url, headers=headers)
        return self.get_return_info(r) 

def stop_job(self, job_name):
        url = self.server_url + self.routes['stop_job']
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        r = self.s.patch(url + urllib.parse.quote(job_name, safe=''), headers=headers)
        return self.get_return_info(r)

    # Get model or dataset metadata 

def lookup_job_status_name(self, job_name):
        url = self.server_url + self.routes['lookup_job_status_name']
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        r = self.s.get(url + urllib.parse.quote(job_name, safe=''), headers=headers)
        return self.get_return_info(r) 

def lookup_job_status(self, age=None, status=None):
        url = self.server_url + self.routes['lookup_job_status']
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        payload = {'age': age, 'status': status}
        r = self.s.get(url, headers=headers, params=payload)
        return self.get_return_info(r) 

def auth_delete_user(self, username):
        url = self.server_url + self.routes['auth_delete_user'] + urllib.parse.quote(username, safe='')
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        r = self.s.delete(url, headers=headers)
        return self.get_return_info(r)

    # Conveniences 

def auth_set_email(self, username, email):
        url = self.server_url + self.routes['auth_set_email']
        headers = self.get_auth_header()
        if headers is None:
            return False, "Cannot get Auth token. Please log in."
        payload = {'username': username, 'email': email}
        r = self.s.patch(url, headers=headers, data=payload)
        return self.get_return_info(r) 

def auth_reset_password(self, username):
        url = self.server_url + self.routes['auth_reset_password']
        headers = self.get_auth_header()
        payload = {'username': str(username)}
        r = self.s.patch(url, headers=headers, data=payload)
        return self.get_return_info(r) 

def auth_register_user(self, username, password, email):
        self.username = username
        self.password = password
        url = self.server_url + self.routes['auth_register_user']
        headers = {'Authorization': self.auth_string}
        payload = {'username': str(username), 'pass1': str(password), 'pass2': str(password), 'email': str(email)}
        r = self.s.post(url, headers=headers, data=payload)
        if r.ok:
            self.auth_string = 'Bearer ' + r.json()['access_token']
            self.token_start_time = time.time()
            return True, self.auth_string
        else:
            return False, '{}: {} - {}'.format(r.status_code, r.reason, r.text[0:1024]) 

def auth_register(self, password, api_key, email):
        self.username = ''
        self.password = password
        self.api_key = api_key
        url = self.server_url + self.routes['auth_register']
        headers = {'Authorization': self.auth_string}
        payload = {'api_key': str(api_key), 'pass1': str(password), 'pass2': str(password), 'email': str(email)}
        r = self.s.post(url, headers=headers, data=payload)
        if r.ok:
            self.auth_string = 'Bearer ' + r.json()['access_token']
            self.token_start_time = time.time()
            return True, self.auth_string
        else:
            return False, '{}: {} - {}'.format(r.status_code, r.reason, r.text[0:1024]) 

def auth_login_user(self, username, password):
        self.username = username
        self.password = password
        url = self.server_url + self.routes['auth_login_user']
        payload = {'username': str(username), 'pass1': str(password)}
        ret = self.get_info()  # Check SSL certificate
        if not ret[0]:
            return ret
        r = self.s.post(url, data=payload)
        if r.ok:
            self.auth_string = 'Bearer ' + r.json()['access_token']
            self.token_start_time = time.time()
            return True, self.auth_string
        else:
            return False, '{}: {} - {}'.format(r.status_code, r.reason, r.text[0:1024]) 

def set_url(self, url, version='v1'):
        if validators.url(url):
            self.server_url = url
            self.version = version
            return True, self.server_url
        else:
            return False, 'invalid url' 

def prepare_url(self):
        url_parsed = urlparse(self.raw_base_url)
        items = url_parsed.path.split('/')
        if len(items) > 0:
            item = items[-1]
            items = items[:-1]
            new_path = '/'.join(items)
        else:
            item = ''
            new_path = url_parsed.path
        url = urlunparse((url_parsed.scheme, url_parsed.netloc, new_path, '', '', ''))

        if item.endswith('.php'):
            self.site_lang = 'php'
        elif item.endswith('.asp'):
            self.site_lang = 'asp'
        elif item.endswith('.aspx'):
            self.site_lang = 'aspx'

        if self.site_lang != '':
            logger.info('site_lang: %s' % self.site_lang)
        self.base_url = url
        self.first_item = item
        logger.info('base_url: %s' % url)
        logger.info('first_item: %s' % item)