Python urllib.error() Examples

def password_length_validator(password):
    """ Test password for length.

    :param password: Plain text password to check

    :return: ``None`` if password conforms to length requirements,
     a list of error/suggestions if not.

    .. versionadded:: 3.4.0

    """
    if len(password) < config_value("PASSWORD_LENGTH_MIN") or len(password) > 128:
        return [
            get_message(
                "PASSWORD_INVALID_LENGTH", length=config_value("PASSWORD_LENGTH_MIN")
            )[0]
        ]
    return None 

def update_progress(progress):
    barLength = 20  # Modify this value to change the length of the progress bar
    status = ""
    if isinstance(progress, int):
        progress = float(progress)
    if not isinstance(progress, float):
        progress = 0
        status = "error: progress var must be float\r\n"
    if progress < 0:
        progress = 0
        status = Fore.RED + "Halt!\r\n"
    if progress >= .999:
        progress = 1
        status = Fore.GREEN + " Complete!\r\n"
    block = int(round(barLength*progress))
    text = "\r[*] Progress: [{0}] {1}% {2}".format("#"*block + "-"*(barLength-block), round(progress*100), status)
    sys.stdout.write(text)
    sys.stdout.flush() 

def get_web_page(url, headers, cookies):
    try:
        logging.info(f'Fetching {url}')
        request = urllib.request.Request(url, None, headers)
        request.add_header('Authorization', cookies)
        response = urllib.request.urlopen(request)
        if response.info().get('Content-Encoding') == 'gzip':
            buf = BytesIO(response.read())
            f = gzip.GzipFile(fileobj=buf)
            r = f.read()
        else:
            r = response.read()
        return r
    except urllib.error.HTTPError as e:
        logging.info(f"Error processing webpage: {e}")
        if e.code == ALREADY_CLICKED_CODE:
            return ALREADY_CLICKED_CODE
        if e.code == UNAUTHORIZED:
            return UNAUTHORIZED
        return None 

def emplid_to_userid(emplid):
    """
    Fetch userid from ITS API for emplid -> userid mapping.

    Admin contact for the API is George Lee in the Learning & Community Platforms Group
    """
    qs = urllib.parse.urlencode({'art': EMPLID_SECRET, 'sfuid': str(emplid)})
    url = USERID_BASE_URL + qs
    try:
        req = urllib.request.urlopen(url, timeout=30)
        jsondata = req.read().decode('utf8')
        data = json.loads(jsondata)
    except ValueError:
        # can't decode JSON
        return None
    except (urllib.error.HTTPError, urllib.error.URLError, http.client.HTTPException):
        # network problem, or 404 (if userid doesn't exist)
        return None

    if 'username' not in data:
        raise ValueError("No 'username' returned in response.")

    userids = data['username']
    return userids.split(',')[0].strip() 

def userid_to_emplid(userid):
    """
    Fetch emplid from ITS API for userid -> emplid mapping.

    Admin contact for the API is George Lee in the Learning & Community Platforms Group
    """
    qs = urllib.parse.urlencode({'art': EMPLID_SECRET, 'username': userid})
    url = EMPLID_BASE_URL + qs
    try:
        req = urllib.request.urlopen(url, timeout=30)
        jsondata = req.read().decode('utf8')
        data = json.loads(jsondata)
    except ValueError:
        # can't decode JSON
        return None
    except (urllib.error.HTTPError, urllib.error.URLError, http.client.HTTPException, socket.timeout, socket.error):
        # network problem, or 404 (if userid doesn't exist)
        return None

    if 'sfuid' not in data:
        return None

    return data['sfuid'] 

def outlines_data_json(offering):
    url = outlines_api_url(offering)
    try:
        req = urllib.request.urlopen(url, timeout=30)
        jsondata = req.read()
        data = json.loads(jsondata.decode('utf8'))
    except ValueError:
        data = {'internal_error': 'could not decode JSON'}
    except (urllib.error.HTTPError, urllib.error.URLError, socket.timeout, socket.error):
        data = {'internal_error': 'could not retrieve outline data from API'}

    if 'info' in data and 'outlinePath' in data['info']:
        data['outlineurl'] = OUTLINES_FRONTEND_BASE + '?' + data['info']['outlinePath']

    return json.dumps(data, indent=1)



##############################################################################
# Emplid -> Userid APIs 

def acad_plan_count(acad_plan, strm):
    """
    Return number of majors in academic plan (e.g. 'CMPTMAJ') in the semester or a SIMSProblem instance (error message).
    """
    db = SIMSConn()

    # most recent acad_plan, most recent acad_plan *in this program* for each student active this semester
    last_prog_plan = "(SELECT ap.emplid, max(ap.effdt) as effdtprog, max(ap2.effdt) as effdt " \
                     "FROM ps_acad_plan ap, ps_acad_plan ap2, ps_stdnt_car_term ct "\
                     "WHERE ap.emplid=ct.emplid and ap.emplid=ap2.emplid and ct.strm=%s and ap.acad_plan=%s " \
                     "and ct.stdnt_car_nbr=ap.stdnt_car_nbr GROUP BY ap.emplid)"

    # select those whose most recent program is in this acad_plan
    db.execute("SELECT count(*) FROM " + last_prog_plan + " WHERE effdtprog=effdt", (strm, acad_plan))
    return db.fetchone()[0]
    #for row in db:
    #    print row 

def password_breached_validator(password):
    """ Check if password on breached list.
    Does nothing unless :py:data:`SECURITY_PASSWORD_CHECK_BREACHED` is set.
    If password is found on the breached list, return an error if the count is
    greater than or equal to :py:data:`SECURITY_PASSWORD_BREACHED_COUNT`.

    :param password: Plain text password to check

    :return: ``None`` if password passes breached tests, else a list of error messages.

    .. versionadded:: 3.4.0
    """
    pwn = config_value("PASSWORD_CHECK_BREACHED")
    if pwn:
        try:
            cnt = pwned(password)
            if cnt >= config_value("PASSWORD_BREACHED_COUNT"):
                return [get_message("PASSWORD_BREACHED")[0]]
        except Exception:
            if pwn == "strict":
                return [get_message("PASSWORD_BREACHED_SITE_ERROR")[0]]
    return None 

def _check_file_create(directory):
    """
    Check that files can be created in the given directory.

    Returns error message, or None if okay
    """
    filename = os.path.join(directory, 'filewrite-' + str(os.getpid()) + '.tmp')

    if not os.path.isdir(directory):
        return 'directory does not exist'

    try:
        fh = open(filename, 'w')
    except IOError:
        return 'could not write to a file'
    else:
        fh.write('test file: may safely delete')
        fh.close()
        os.unlink(filename) 

def _check_cert(filename):
    """
    Does this certificate file look okay?

    Returns error message, or None if okay
    """
    try:
        st = os.stat(filename)
    except OSError:
        return filename + " doesn't exist"
    else:
        good_perm = stat.S_IFREG | stat.S_IRUSR # | stat.S_IWUSR
        if (st[stat.ST_UID], st[stat.ST_GID]) != (0,0):
            return 'not owned by root.root'
        perm = st[stat.ST_MODE]
        if good_perm != perm:
            return "expected permissions %o but found %o." % (good_perm, perm) 

def parsefootnotes(self):
    "Parse footnotes options."
    if not Options.footnotes:
      return
    Options.marginfoot = False
    Options.letterfoot = False
    Options.hoverfoot = False
    options = Options.footnotes.split(',')
    for option in options:
      footoption = option + 'foot'
      if hasattr(Options, footoption):
        setattr(Options, footoption, True)
      else:
        Trace.error('Unknown footnotes option: ' + option)
    if not Options.endfoot and not Options.marginfoot and not Options.hoverfoot:
      Options.hoverfoot = True
    if not Options.numberfoot and not Options.symbolfoot:
      Options.letterfoot = True 

def __install_logproxy(self):
		# pylint: disable=W0212
		class LogProxy():
			def __init__(self, parent_logger, log_prefix):
				self.parent_logger = parent_logger
				self.log_prefix    = log_prefix
			def debug(self, msg, *args):
				self.parent_logger._debug   (" [{}] -> ".format(self.log_prefix) + msg, *args)
			def info(self, msg, *args):
				self.parent_logger._info    (" [{}] -> ".format(self.log_prefix) + msg, *args)
			def error(self, msg, *args):
				self.parent_logger._error   (" [{}] -> ".format(self.log_prefix) + msg, *args)
			def critical(self, msg, *args):
				self.parent_logger._critical(" [{}] -> ".format(self.log_prefix) + msg, *args)
			def warning(self, msg, *args):
				self.parent_logger._warning (" [{}] -> ".format(self.log_prefix) + msg, *args)
			def warn(self, msg, *args):
				self.parent_logger._warning (" [{}] -> ".format(self.log_prefix) + msg, *args)

		self.wg.log  = LogProxy(self, "WebGet")
		self.cwg.log = LogProxy(self, "CWebGet")
		self.log     = LogProxy(self, "MainRPCAgent") 

def regex(ioc_type):
    ioc_patts = {
        "ip":b"((?:(?:[12]\d?\d?|[1-9]\d|[1-9])(?:\[\.\]|\.)){3}(?:[12]\d?\d?|[\d+]{1,2}))",
        "domain":b"([A-Za-z0-9]+(?:[\-|\.][A-Za-z0-9]+)*(?:\[\.\]|\.)(?:com|net|edu|ru|org|de|uk|jp|br|pl|info|fr|it|cn|in|su|pw|biz|co|eu|nl|kr|me))",
        "md5":b"\W([A-Fa-f0-9]{32})(?:\W|$)",
        "sha1":b"\W([A-Fa-f0-9]{40})(?:\W|$)",
        "sha256":b"\W([A-Fa-f0-9]{64})(?:\W|$)",
        "email":b"[a-zA-Z0-9_]+(?:\.[A-Za-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?!([a-zA-Z0-9]*\.[a-zA-Z0-9]*\.[a-zA-Z0-9]*\.))(?:[A-Za-z0-9](?:[a-zA-Z0-9-]*[A-Za-z0-9])?\.)+[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?",
        "URL":b"((?:http|ftp|https)\:\/\/(?:[\w+?\.\w+])+[a-zA-Z0-9\~\!\@\#\$\%\^\&\*\(\)_\-\=\+\\\/\?\.\:\;]+)",
        "yara":b"(rule\s[\w\W]{,30}\{[\w\W\s]*\})"
    }

    try:
        pattern = re.compile(ioc_patts[ioc_type])
    except re.error:
        print('[!] Invalid type specified.')
        sys.exit(0)

    return pattern 

def calculate_all(request, course_slug, activity_slug):
    course = get_object_or_404(CourseOffering, slug=course_slug)
    activity = get_object_or_404(CalNumericActivity, slug=activity_slug, offering=course, deleted=False)
    
    try:
        ignored, hiding_info = calculate_numeric_grade(course,activity)
        if hiding_info:
            messages.warning(request, "This activity is released to students, but the calculation uses unreleased grades. Calculations done with unreleased activities as zero to prevent leaking hidden info to students.")
        if ignored==1:
            messages.warning(request, "Did not calculate grade for 1 manually-graded student.")
        elif ignored>1:
            messages.warning(request, "Did not calculate grade for %i manually-graded students." % (ignored))
    except ValidationError as e:
        messages.error(request, e.args[0])
    except EvalException as e:
        messages.error(request, e.args[0])
    except NotImplementedError:
        return NotFoundResponse(request)

    return HttpResponseRedirect(activity.get_absolute_url()) 

def calculate_all_lettergrades(request, course_slug, activity_slug):
    course = get_object_or_404(CourseOffering, slug=course_slug)
    activity = get_object_or_404(CalLetterActivity, slug=activity_slug, offering=course, deleted=False)
    
    try:
        ignored = calculate_letter_grade(course,activity)
        if ignored==1:
            messages.warning(request, "Did not calculate letter grade for 1 manually-graded student.")
        elif ignored>1:
            messages.warning(request, "Did not calculate letter grade for %i manually-graded students." % (ignored))
    except ValidationError as e:
        messages.error(request, e.args[0])
    except NotImplementedError:
        return NotFoundResponse(request)

    return HttpResponseRedirect(activity.get_absolute_url()) 

def getTaskStatus(taskId):
  """Retrieve status of one or more long-running tasks.

  Args:
    taskId: ID of the task or a list of multiple IDs.

  Returns:
    List containing one object for each queried task, in the same order as
    the input array, each object containing the following values:
      id (string) ID of the task.
      state (string) State of the task, one of READY, RUNNING, COMPLETED,
        FAILED, CANCELLED; or UNKNOWN if the task with the specified ID
        doesn't exist.
      error_message (string) For a FAILED task, a description of the error.
  """
  if isinstance(taskId, six.string_types):
    taskId = [taskId]
  args = {'q': ','.join(taskId)}
  return send_('/taskstatus', args, 'GET') 

def DeleteSharedCredential(self, id_or_shared_credential):
        """
        Delete a shared credential from the security console.
        An id or dl_nexpose.SharedCredential object is expected.
        This function will return a boolean indicating success.
        NOTE: no official documentation exists except the Rapid7 Nexpose Ruby API.
        """
        # TODO: To be equal with the other delete functions, this one should raise an exception
        try:
            if isinstance(id_or_shared_credential, SharedCredentialBase):
                id_or_shared_credential = id_or_shared_credential.id
            return self.ExecuteFormPost('data/credential/shared/delete?credid={0}'.format(id_or_shared_credential), '') == 'true'
        except urllib.error.HTTPError:
            return False

    #
    # The following functions implement the Backup Management API (not documented):
    # =========================================================== 

def ExecuteDelete_JSON(session_id, uri, sub_url, timeout):
    headers = CreateHeadersWithSessionCookieAndCustomHeader(session_id)
    uri = uri + sub_url
    try:
        ExecuteWebRequest(uri, None, headers, timeout, lambda: 'DELETE')
    except urllib.error.HTTPError as e:
        if e.code == 410:
            return True
        raise e
    try:
        ExecuteGet_JSON(session_id, uri, sub_url, timeout)
    except urllib.error.HTTPError as e:
        if e.code == 404:
            return True
        raise e
    return False 

def VerifySuccess(self, response):
        """
        If the response indicates a failure, the error message is extracted and
        a NexposeFailureException is raised. Otherwise the response is returned.
        """
        if response.tag == 'Failure':
            message = get_content_of(response, 'Exception/Message')
            if message is None:
                message = get_content_of(response, 'Message')
            raise NexposeFailureException(message)
        if get_attribute(response, 'success') == '0':
            message = get_content_of(response, 'Failure/message')
            if message is None:
                message = get_content_of(response, 'Failure/Exception/message')
            if message is None:
                message = get_content_of(response, 'Error')  # Used by unofficial API's (for example: TestAdminCredentialsResult)
            raise NexposeFailureException(message)
        return response

    #
    # The following functions implement the Site Management API:
    # ========================================================= 

def Open(self):
        """
        Opens a session to the nexpose appliance by logging in.
        This function with raise an exception on error or if the session is already open.
        """
        if self._session_id:
            raise SessionIsNotClosedException("Please close the session first!")
        try:
            response = self._Execute_APIv1d1(self._login_request)
        except NexposeConnectionException as ex:
            if isinstance(ex.inner_exception, etree.XMLSyntaxError):
                raise NexposeException("Unexpected error! Is the Nexpose appliance activated?")
            raise ex
        if response.tag == "LoginResponse":
            if response.attrib["success"] == "1":
                self._session_id = response.attrib["session-id"]
        if not self._session_id:
            raise NexposeFailureException("Login failure!") 

def process(self, container, list):
    "Add allowed containers, clone cloned containers and add the clone."
    name = container.__class__.__name__
    if name in self.allowed:
      list.append(container)
    elif name in self.cloned:
      list.append(self.safeclone(container))
    else:
      Trace.error('Unknown container class ' + name) 

def kegg_rest_request(query):
    url = 'http://rest.kegg.jp/%s' % (query)
    print(url)
    try:
        data = urllib.request.urlopen(url).read()
    except urllib.error.HTTPError as e:
        print("HTTP error: %d" % e.code)
    except urllib.error.URLError as e:
        print("Network error: %s" % e.reason.args[1])
    
    return data 

def print_error(e):
    sys.stdout = sys.__stdout__
    if isinstance(e, ValueError):
        print("Unknown url type")

    if isinstance(e, urllib.error.HTTPError):
            print("[!] URL Returned an HTTP error: {}".format(
                colorize(str(e.code), 'error')))

    if isinstance(e, urllib.error.URLError):
            if "CERTIFICATE_VERIFY_FAILED" in str(e.reason):
                print("SSL: Certificate validation error.\nIf you want to \
    ignore it run the program with the \"-d\" option.")
            else:
                print("Target host seems to be unreachable ({})".format(e.reason)) 

def normalize(target):
    try:
        if (socket.inet_aton(target)):
            target = 'http://' + target
    except (ValueError, socket.error):
        pass
    finally:
        return target 

def colorize(string, alert):
    color = {
        'error':    bcolors.FAIL + string + bcolors.ENDC,
        'warning':  bcolors.WARNING + string + bcolors.ENDC,
        'ok':       bcolors.OKGREEN + string + bcolors.ENDC,
        'info':     bcolors.OKBLUE + string + bcolors.ENDC
    }
    return color[alert] if alert in color else string 

def _error(self, msg, *args):
		tmp = self.logname + " [ERROR] ->" + msg % args
		self.local_logger.error(tmp)
		self.out_buffer.append(tmp) 

def other_links(url):
    """
    Uses Microsoft's Cognitive API to evaluate the quality of a webpage, and suggest
    better information if possible.
    """
    link_verified = verified_links(url)
    if link_verified == "not verified":

        st = url_title(url)
        import http.client, urllib.request, urllib.parse, urllib.error
        headers = {
            'Ocp-Apim-Subscription-Key': MICROSOFT_SEARCH_SUBSCRIPTION_KEY,}
        params = urllib.parse.urlencode({'q': st, 'count': '10', 'offset': '0', 'mkt': 'en-us','safesearch': 'Moderate',})
        try:
            conn = http.client.HTTPSConnection('api.cognitive.microsoft.com')
            conn.request("GET", "/bing/v5.0/search?%s" % params, "", headers)
            response = conn.getresponse()
            data = response.read()
            #print(data)
            data = json.loads(data.decode("utf-8"))

            for alt_url in data['webPages']['value']:
                if alt_url['displayUrl'] != url:
                    urlscores = verified_links(alt_url['displayUrl'])
                    if urlscores == "verified":
                        alternative_summary = "Non verified. Better Verified Info is : "+summarization(alt_url['displayUrl'])
                        return alternative_summary
            conn.close()
            return "no verified links"

        except Exception as e:
            print("[Errno {0}] {1}".format(e.errno, e.strerror))
    else:
        return link_verified 

def scan_one(url, data=None, headers=None, encoding="UTF-8"):
    """扫描单个URL漏洞"""
    click.secho('[+] 正在扫描URL:' + url, fg='green')
    ss = [s(url, data, headers, encoding) for s in s2_list]
    with futures.ThreadPoolExecutor(max_workers=10) as executor:
        results = list(executor.map(check_one, ss))
    results = {r for r in results if r}
    click.secho('[*] ----------------results------------------'.format(url=url), fg='green')
    if (not results) and (not is_quiet):
        click.secho('[*] {url} 未发现漏洞'.format(url=url), fg='red')
    for r in results:
        if r.startswith("ERROR:"):
            click.secho('[ERROR] {url} 访问出错: {error}'.format(url=url, error=r[6:]), fg='red')
        else:
            click.secho('[*] {url} 存在漏洞: {name}'.format(url=url, name=r), fg='red') 

def createAssetHome(requestedId):
  """Attempts to create a home root folder for the current user ("users/joe").

  Results in an error if the user already has a home root folder or the
  requested ID is unavailable.

  Args:
    requestedId: The requested ID of the home folder (e.g. "users/joe").
  """
  send_('/createbucket', {'id': requestedId}) 

def write(self, strings):
    "Write a list of strings"
    for string in strings:
      if not isinstance(string, str):
        Trace.error('Not a string: ' + str(string) + ' in ' + str(strings))
        return
      self.writestring(string)