Python urllib2.HTTPSHandler() Examples

def get_access_token(self, code, state=None):
        '''
        In callback url: http://host/callback?code=123&state=xyz
        use code and state to get an access token.
        '''
        kw = dict(client_id=self._client_id, client_secret=self._client_secret, code=code)
        if self._redirect_uri:
            kw['redirect_uri'] = self._redirect_uri
        if state:
            kw['state'] = state
        opener = build_opener(HTTPSHandler)
        request = Request('https://github.com/login/oauth/access_token', data=_encode_params(kw))
        request.get_method = _METHOD_MAP['POST']
        request.add_header('Accept', 'application/json')
        try:
            response = opener.open(request, timeout=TIMEOUT)
            r = _parse_json(response.read())
            if 'error' in r:
                raise ApiAuthError(str(r.error))
            return str(r.access_token)
        except HTTPError as e:
            raise ApiAuthError('HTTPError when get access token') 

def download_python(self):
        """
        Download Python
        :return: None
        """

        self._prepare_for_download()

        logger.info('Beginning download of python')

        with open(os.path.join(DOWNLOADS_DIR, 'python-installer.msi'), 'wb') as f:

            if self.PROXY is not None:
                opener = urllib2.build_opener(
                    urllib2.HTTPHandler(),
                    urllib2.HTTPSHandler(),
                    urllib2.ProxyHandler({'http': self.PROXY, 'https': self.PROXY})
                )
                urllib2.install_opener(opener)

            f.write(urllib2.urlopen(self.PYTHON_DOWNLOAD_URL, timeout=self.DOWNLOAD_TIMEOUT).read())

        logger.debug('Download of python complete') 

def get_access_token(self, code, state=None):
        '''
        In callback url: http://host/callback?code=123&state=xyz

        use code and state to get an access token.        
        '''
        kw = dict(client_id=self._client_id, client_secret=self._client_secret, code=code)
        if self._redirect_uri:
            kw['redirect_uri'] = self._redirect_uri
        if state:
            kw['state'] = state
        opener = build_opener(HTTPSHandler)
        request = Request('https://github.com/login/oauth/access_token', data=_encode_params(kw))
        request.get_method = _METHOD_MAP['POST']
        request.add_header('Accept', 'application/json')
        try:
            response = opener.open(request, timeout=TIMEOUT)
            r = _parse_json(response.read())
            if 'error' in r:
                raise ApiAuthError(str(r.error))
            return str(r.access_token)
        except HTTPError as e:
            raise ApiAuthError('HTTPError when get access token') 

def __init__(self, user_id='0', tv_auth_token='0'):
        '''
        Creates the TV API interface. The user identified by the supplied user
        id must have a TV profile created by a call to
        QuizduellApi.create_tv_user() and a personal TV auth token.
        
        @param user_id: Quizduell user id
        @type user_id: str
        
        @param tv_auth_token: TV auth token returned by
                              QuizduellApi.create_tv_user()
        @type tv_auth_token: str
        ''' 
        self._user_id = user_id
        self._tv_auth_token = tv_auth_token
        self._opener = urllib2.build_opener(
            urllib2.HTTPRedirectHandler(),
            urllib2.HTTPHandler(debuglevel=0),
            urllib2.HTTPSHandler(debuglevel=0)
        ) 

def download_vcpython27(self):
        """
        Download vcpython27 since some Windows 7 boxes have it and some don't.
        :return: None
        """

        self._prepare_for_download()

        logger.info('Beginning download of vcpython27... this may take a few minutes...')

        with open(os.path.join(DOWNLOADS_DIR, 'vcpython27.msi'), 'wb') as f:

            if self.PROXY is not None:
                opener = urllib2.build_opener(
                    urllib2.HTTPHandler(),
                    urllib2.HTTPSHandler(),
                    urllib2.ProxyHandler({'http': self.PROXY, 'https': self.PROXY})
                )
                urllib2.install_opener(opener)

            f.write(urllib2.urlopen(self.VCPYTHON27_DOWNLOAD_URL, timeout=self.DOWNLOAD_TIMEOUT).read())

        logger.debug('Download of vcpython27 complete') 

def get_urldata(url, urldata, method):
    # create a handler
    handler = urllib2.HTTPSHandler()
    # create an openerdirector instance
    opener = urllib2.build_opener(handler)
    # encode urldata
    body = urllib.urlencode(urldata)
    # build a request
    req = urllib2.Request(url, data=body)
    # add any other information you want
    req.add_header('Accept', 'application/json')
    req.add_header('User-Agent', __useragent__)
    # overload the get method function
    req.get_method = lambda: method
    try:
        #response = urllib2.urlopen(req)
        connection = opener.open(req)
    except urllib2.HTTPError,e:
        connection = e 

def login():
	cookie_filename = "cookies.txt"
	cookiejar = cookielib.MozillaCookieJar(cookie_filename)
	opener = urllib2.build_opener(urllib2.HTTPRedirectHandler(),urllib2.HTTPHandler(debuglevel=0),urllib2.HTTPSHandler(debuglevel=0),urllib2.HTTPCookieProcessor(cookiejar))
	page = loadPage(opener, "https://www.linkedin.com/")
	parse = BeautifulSoup(page, "html.parser")

	csrf = parse.find(id="loginCsrfParam-login")['value']
	
	login_data = urllib.urlencode({'session_key': username, 'session_password': password, 'loginCsrfParam': csrf})
	page = loadPage(opener,"https://www.linkedin.com/uas/login-submit", login_data)
	
	parse = BeautifulSoup(page, "html.parser")
	cookie = ""
	
	try:
		cookie = cookiejar._cookies['.www.linkedin.com']['/']['li_at'].value
	except:
		sys.exit(0)
	
	cookiejar.save()
	os.remove(cookie_filename)
	return cookie 

def scan(url, redirect, insecure, useragent, postdata, proxy):
    request = urllib2.Request(url.geturl())
    request.add_header('User-Agent', useragent)
    request.add_header('Origin', 'http://hsecscan.com')
    request.add_header('Accept', '*/*')
    if postdata:
        request.add_data(urllib.urlencode(postdata))
    build = [urllib2.HTTPHandler()]
    if redirect:
        build.append(RedirectHandler())
    if proxy:
        build.append(urllib2.ProxyHandler({'http': proxy, 'https': proxy}))
    if insecure:
        context = ssl._create_unverified_context()
        build.append(urllib2.HTTPSHandler(context=context))
    urllib2.install_opener(urllib2.build_opener(*build))
    response = urllib2.urlopen(request)
    print '>> RESPONSE INFO <<'
    print_response(response.geturl(), response.getcode(), response.info())
    print '>> RESPONSE HEADERS DETAILS <<'
    for header in response.info().items():
        check_header(header)
    print '>> RESPONSE MISSING HEADERS <<'
    missing_headers(response.info().items(), url.scheme) 

def reverseip(url):
    """return domains from given the same server"""

    # get only domain name
    url = urlparse(url).netloc if urlparse(url).netloc != '' else urlparse(url).path.split("/")[0]

    source = "http://domains.yougetsignal.com/domains.php"
    useragent = useragents.get()
    contenttype = "application/x-www-form-urlencoded; charset=UTF-8"

    # POST method
    opener = urllib2.build_opener(
        urllib2.HTTPHandler(), urllib2.HTTPSHandler())
    data = urllib.urlencode([('remoteAddress', url), ('key', '')])

    request = urllib2.Request(source, data)
    request.add_header("Content-type", contenttype)
    request.add_header("User-Agent", useragent)

    try:
        result = urllib2.urlopen(request).read()

    except urllib2.HTTPError, e:
        print >> sys.stderr, "[{}] HTTP error".format(e.code) 

def openUrl(self,url, ischunkDownloading=False):
        try:
            post=None
            openner = urllib2.build_opener(urllib2.HTTPHandler, urllib2.HTTPSHandler)

            if post:
                req = urllib2.Request(url, post)
            else:
                req = urllib2.Request(url)
            
            ua_header=False
            if self.clientHeader:
                for n,v in self.clientHeader:
                    req.add_header(n,v)
                    if n=='User-Agent':
                        ua_header=True

            if not ua_header:
                req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36')
            #response = urllib2.urlopen(req)
            if self.proxy and (  (not ischunkDownloading) or self.use_proxy_for_chunks ):
                req.set_proxy(self.proxy, 'http')
            response = openner.open(req)

            return response
        except:
            print 'Error in getUrl'
            traceback.print_exc()
        return None 

def __init__(self, username, password):
        super(PyLinuxQQ, self).__init__()
        self.cookies = cookielib.CookieJar()
        self.opener = urllib2.build_opener(
            urllib2.HTTPHandler(),
            urllib2.HTTPSHandler(),
            urllib2.HTTPCookieProcessor(self.cookies),
        )
        urllib2.install_opener(self.opener)
        self.username = username
        self.password = password 

def hashed_download(url, temp, digest):
    """Download ``url`` to ``temp``, make sure it has the SHA-256 ``digest``,
    and return its path."""
    # Based on pip 1.4.1's URLOpener but with cert verification removed. Python
    # >=2.7.9 verifies HTTPS certs itself, and, in any case, the cert
    # authenticity has only privacy (not arbitrary code execution)
    # implications, since we're checking hashes.
    def opener(using_https=True):
        opener = build_opener(HTTPSHandler())
        if using_https:
            # Strip out HTTPHandler to prevent MITM spoof:
            for handler in opener.handlers:
                if isinstance(handler, HTTPHandler):
                    opener.handlers.remove(handler)
        return opener

    def read_chunks(response, chunk_size):
        while True:
            chunk = response.read(chunk_size)
            if not chunk:
                break
            yield chunk

    parsed_url = urlparse(url)
    response = opener(using_https=parsed_url.scheme == 'https').open(url)
    path = join(temp, parsed_url.path.split('/')[-1])
    actual_hash = sha256()
    with open(path, 'wb') as file:
        for chunk in read_chunks(response, 4096):
            file.write(chunk)
            actual_hash.update(chunk)

    actual_digest = actual_hash.hexdigest()
    if actual_digest != digest:
        raise HashError(url, path, actual_digest, digest)
    return path 

def __init__(self, connection_class=VerifiedHTTPSConnection):
        self.specialized_conn_class = connection_class
        urllib2.HTTPSHandler.__init__(self) 

def _http(self, _method, _path, **kw):
        data = None
        params = None
        if _method=='GET' and kw:
            _path = '%s?%s' % (_path, _encode_params(kw))
        if _method in ['POST', 'PATCH', 'PUT']:
            data = bytes(_encode_json(kw), 'utf-8')
        url = '%s%s' % (_URL, _path)
        opener = build_opener(HTTPSHandler)
        request = Request(url, data=data)
        request.get_method = _METHOD_MAP[_method]
        if self._authorization:
            request.add_header('Authorization', self._authorization)
        if _method in ['POST', 'PATCH', 'PUT']:
            request.add_header('Content-Type', 'application/x-www-form-urlencoded')
        try:
            response = opener.open(request, timeout=TIMEOUT)
            is_json = self._process_resp(response.headers)
            if is_json:
                return _parse_json(response.read().decode('utf-8'))
        except HTTPError as e:
            is_json = self._process_resp(e.headers)
            if is_json:
                json = _parse_json(e.read().decode('utf-8'))
            else:
                json = e.read().decode('utf-8')
            req = JsonObject(method=_method, url=url)
            resp = JsonObject(code=e.code, json=json)
            if resp.code==404:
                raise ApiNotFoundError(url, req, resp)
            raise ApiError(url, req, resp) 

def __init__(self, connection_class=VerifiedHTTPSConnection):
        self.specialized_conn_class = connection_class
        urllib2.HTTPSHandler.__init__(self) 

def __init__(self, auth_file):
        urllib2.HTTPSHandler.__init__(self)
        self.auth_file = auth_file 

def get_csrf():
    """
    get CSRF token from login page to use in POST requests
    """
    global csrf_token

    print(bcolors.WARNING + "[+] Getting CSRF Token: " + bcolors.ENDC)

    try:
        opener = rq.build_opener(rq.HTTPHandler(), rq.HTTPSHandler())
        opener.addheaders = [('User-agent', 'Mozilla/5.0')]
        rq.install_opener(opener)

        request = rq.Request('https://www.instagram.com/')
        try:
            # python 2
            headers = rq.urlopen(request).info().headers
        except Exception:
            # python 3
            headers = rq.urlopen(request).info().get_all('Set-Cookie')

        for header in headers:
            if header.find('csrftoken') != -1:
                csrf_token = header.partition(';')[0].partition('=')[2]
                print(bcolors.OKGREEN + "[+] CSRF Token :", csrf_token, "\n" + bcolors.ENDC)
    except Exception as err:
        print(bcolors.FAIL + "[!] Can't get CSRF token , please use -d for debug" + bcolors.ENDC)

        if _debug:
            logger.error(err)

        print(bcolors.FAIL + "[!] Exiting..." + bcolors.ENDC)
        exit(3) 

def check_proxy(q):
    """
    check proxy for and append to working proxies
    :param q:
    """
    if not q.empty():

        proxy = q.get(False)
        proxy = proxy.replace("\r", "").replace("\n", "")

        try:
            opener = rq.build_opener(
                rq.ProxyHandler({'https': 'https://' + proxy}),
                rq.HTTPHandler(),
                rq.HTTPSHandler()
            )

            opener.addheaders = [('User-agent', 'Mozilla/5.0')]
            rq.install_opener(opener)

            req = rq.Request('https://api.ipify.org/')

            if rq.urlopen(req).read().decode() == proxy.partition(':')[0]:
                proxys_working_list.update({proxy: proxy})
                if _verbose:
                    print(bcolors.OKGREEN + " --[+] ", proxy, " | PASS" + bcolors.ENDC)
            else:
                if _verbose:
                    print(" --[!] ", proxy, " | FAILED")

        except Exception as err:
            if _verbose:
                print(" --[!] ", proxy, " | FAILED")
            if _debug:
                logger.error(err)
            pass 

def __init__(self, connection_class=UnverifiedHTTPSConnection):
        self.specialized_conn_class = connection_class
        urllib2.HTTPSHandler.__init__(self) 

def request(target, httpsproxy=None, useragent=None):
	global contenttype

	if not useragent:
		useragent = "Mozilla/5.0 (X11; Linux x86_64; rv:22.0) Gecko/20100101 Firefox/22.0 Iceweasel/22.0"
	else:
		print "["+ bc.G + "+" + bc.ENDC + "] User-Agent: " + useragent

	if httpsproxy:
		print "["+ bc.G + "+" + bc.ENDC + "] Proxy: " + httpsproxy + "\n"
		opener = urllib2.build_opener(
                urllib2.HTTPHandler(),
                urllib2.HTTPSHandler(),
                urllib2.ProxyHandler({'http': 'http://' + httpsproxy}))
		urllib2.install_opener(opener)

	postdata = [('remoteAddress',target),('key','')]
	postdata = urllib.urlencode(postdata)

	request = urllib2.Request(url, postdata)

	request.add_header("Content-type", contenttype)
	request.add_header("User-Agent", useragent)
	try:
		result = urllib2.urlopen(request).read()
	except urllib2.HTTPError, e:
		print "Error: " + e.code 

def get(self, uri, params={}, headers={}, with_status_code=False, timeout=10, user=None, password=None):
        data = None  # always none in GET
        
        if params:
            uri = "%s?%s" % (uri, urlencode(params))
        
        # SSL, user/password and basic
        # NOTE: currently don't manage ssl & user/password
        if uri.startswith('https://'):
            handler = HTTPSHandler(context=self.ssl_context)
        elif user and password:
            passwordMgr = HTTPPasswordMgrWithDefaultRealm()
            passwordMgr.add_password(None, uri, user, password)
            handler = HTTPBasicAuthHandler(passwordMgr)
        else:
            handler = HTTPHandler
        
        url_opener = build_opener(handler)
        
        req = Request(uri, data)
        req.get_method = lambda: 'GET'
        for (k, v) in headers.items():
            req.add_header(k, v)
        
        request = url_opener.open(req, timeout=timeout)
        
        response = request.read()
        status_code = request.code
        request.close()
        
        if not with_status_code:
            return response
        else:
            return (status_code, response) 

def detect_tapatalk(self):
        # request page, check for tapatalk banner
        handlers = [
                    urllib2.HTTPHandler(debuglevel=self.debug),
                    urllib2.HTTPSHandler(debuglevel=self.debug),

                    ]
        ua = urllib2.build_opener(*handlers)
        ua.addheaders = [('User-agent', 'Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3')]
        data = ua.open(self.target).read()
        if self.debug:
            print data
        if not "tapatalkDetect()" in data:
            print "[xx] could not detect tapatalk. bye..."
            return None
            
        # extract tapatalk version
        print "[ i] Taptalk detected ... ",
        path = "".join(re.findall(r"^\s*<link href=[\s'\"]?(http://.*?/)smartbanner/appbanner.css", data, re.MULTILINE|re.DOTALL))
        path+="mobiquo.php"
        print "'%s' ... "%path,
        data = urllib.urlopen(path).read()
        version = "".join(re.findall(r"Current Tapatalk plugin version:\s*([\d\.a-zA-Z]+)", data))
        if LooseVersion(version) <= LooseVersion("5.2.1"):
            print "v.%s  :) - OK"%version    
            return path
        print "v.%s :( - not vulnerable"%version
        return None 

def __init__(self,baseurl,params, debuglevel=1):
        self.cookies = cookielib.LWPCookieJar()
        handlers = [
                    urllib2.HTTPHandler(debuglevel=debuglevel),
                    urllib2.HTTPSHandler(debuglevel=debuglevel),
                    urllib2.HTTPCookieProcessor(self.cookies)
                    ]
        self.browser = urllib2.build_opener(*handlers)
        self.baseurl=baseurl
        self.params = params 

def __init__(self,baseurl,params, debuglevel=1):
        self.cookies = cookielib.LWPCookieJar()
        handlers = [
                    urllib2.HTTPHandler(debuglevel=debuglevel),
                    urllib2.HTTPSHandler(debuglevel=debuglevel),
                    urllib2.HTTPCookieProcessor(self.cookies)
                    ]
        self.browser = urllib2.build_opener(*handlers)
        self.baseurl=baseurl
        self.params = params 

def __init__(self, *args, **kwargs):
        try:
            kwargs['context'] = ssl._create_unverified_context()
        except AttributeError:
            # Python prior to 2.7.9 doesn't have default-enabled certificate
            # verification
            pass

        urllib2.HTTPSHandler.__init__(self, *args, **kwargs) 

def __init__(self, cookie_jar=None):
        '''
        Creates the API interface. Expects either an authentication cookie within
        the user supplied cookie jar or a subsequent call to
        QuizduellApi.login_user() or QuizduellApi.create_user().
        
        @param cookie_jar: Stores authentication tokens with each request made
        @type cookie_jar: cookielib.CookieJar or None
        '''
        self._opener = urllib2.build_opener(
            urllib2.HTTPRedirectHandler(),
            urllib2.HTTPHandler(debuglevel=0),
            urllib2.HTTPSHandler(debuglevel=0),
            urllib2.HTTPCookieProcessor(cookie_jar)
        ) 

def __init__(self, cacert_file):
        self.cacert_file = cacert_file
        request.HTTPSHandler.__init__(self) 

def __init__(self, ctx):
		urllib2.HTTPSHandler.__init__(self)
		self.ctx = ctx 

def __init__(self, auth_file):
        urllib2.HTTPSHandler.__init__(self)
        self.auth_file = auth_file 

def __init__(self, auth_file):
        urllib2.HTTPSHandler.__init__(self)
        self.auth_file = auth_file