Python requests.utils() Examples

The following are code examples for showing how to use requests.utils(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: AutomaticEssayGrading   Author: SahilC   File: py_bing_search.py    MIT License 6 votes vote down vote up
def _search(self, limit, format):
        '''
        Returns a list of result objects, with the url for the next page bing search url.
        '''
        url = self.QUERY_URL.format(requests.utils.quote("'{}'".format(self.query)), min(50, limit), self.current_offset, format)
        r = requests.get(url, auth=("", self.api_key))
        try:
            json_results = r.json()
        except ValueError as vE:
            if not self.safe:
                raise PyBingWebException("Request returned with code %s, error msg: %s" % (r.status_code, r.text))
            else:
                print ("[ERROR] Request returned with code %s, error msg: %s. \nContinuing in 5 seconds." % (r.status_code, r.text))
                time.sleep(5)
        packaged_results = [WebResult(single_result_json) for single_result_json in json_results['d']['results']]
        self.current_offset += min(50, limit, len(packaged_results))
        return packaged_results 
Example 2
Project: AutomaticEssayGrading   Author: SahilC   File: py_bing_search.py    MIT License 6 votes vote down vote up
def _search(self, limit, format):
        '''
        Returns a list of result objects, with the url for the next page bing search url.
        '''
        filters = requests.utils.quote("'{}'".format(self.image_filters))
        url = self.QUERY_URL.format(requests.utils.quote("'{}'".format(self.query)), min(50, limit), self.current_offset, format, filters)
        r = requests.get(url, auth=("", self.api_key))
        try:
            json_results = r.json()
        except ValueError as vE:
            if not self.safe:
                raise PyBingImageException("Request returned with code %s, error msg: %s" % (r.status_code, r.text))
            else:
                print ("[ERROR] Request returned with code %s, error msg: %s. \nContinuing in 5 seconds." % (r.status_code, r.text))
                time.sleep(5)
        packaged_results = [ImageResult(single_result_json) for single_result_json in json_results['d']['results']]
        self.current_offset += min(50, limit, len(packaged_results))
        return packaged_results 
Example 3
Project: AutomaticEssayGrading   Author: SahilC   File: py_bing_search.py    MIT License 6 votes vote down vote up
def _search(self, limit, format):
        '''
        Returns a list of result objects, with the url for the next page bing search url.
        '''
        url = self.QUERY_URL.format(requests.utils.quote("'{}'".format(self.query)), min(50, limit), self.current_offset, format)
        r = requests.get(url, auth=("", self.api_key))
        try:
            json_results = r.json()
        except ValueError as vE:
            if not self.safe:
                raise PyBingVideoException("Request returned with code %s, error msg: %s" % (r.status_code, r.text))
            else:
                print ("[ERROR] Request returned with code %s, error msg: %s. \nContinuing in 5 seconds." % (r.status_code, r.text))
                time.sleep(5)
        packaged_results = [VideoResult(single_result_json) for single_result_json in json_results['d']['results']]
        self.current_offset += min(50, limit, len(packaged_results))
        return packaged_results 
Example 4
Project: AutomaticEssayGrading   Author: SahilC   File: py_bing_search.py    MIT License 6 votes vote down vote up
def _search(self, limit, format):
        '''
        Returns a list of result objects, with the url for the next page bing search url.
        '''
        url = self.QUERY_URL.format(requests.utils.quote("'{}'".format(self.query)), min(50, limit), self.current_offset, format)
        r = requests.get(url, auth=("", self.api_key))
        try:
            json_results = r.json()
        except ValueError as vE:
            if not self.safe:
                raise PyBingNewsException("Request returned with code %s, error msg: %s" % (r.status_code, r.text))
            else:
                print ("[ERROR] Request returned with code %s, error msg: %s. \nContinuing in 5 seconds." % (r.status_code, r.text))
                time.sleep(5)
        packaged_results = [NewsResult(single_result_json) for single_result_json in json_results['d']['results']]
        self.current_offset += min(50, limit, len(packaged_results))
        return packaged_results 
Example 5
Project: python_practice   Author: zhengbomo   File: Server.py    MIT License 6 votes vote down vote up
def login(url, data):
        headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
            "Accept-Encoding": "gzip, deflate, sdch",
            "Accept-Language": "zh-CN,zh;q=0.8,en;q=0.6",
            "Connection": "keep-alive",
            "Content-Type": "application/x-www-form-urlencoded",
            "Host": "ac.ppdai.com",
            "Origin": "https://ac.ppdai.com",
            "Referer": "https://ac.ppdai.com/User/Login?message=&Redirect=",
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) "
                          "Chrome/51.0.2704.103 Safari/537.36",
            "X-Requested-With": "XMLHttpRequest"
        }

        cookies, exists = Server.__get_cookie()
        if not exists:
            session = requests.Session()
            session.headers = headers
            response = session.post(url, data=data)

            # 写入cookie
            with open('cookie', 'w') as f:
                pickle.dump(requests.utils.dict_from_cookiejar(response.cookies), f) 
Example 6
Project: OpenCouture-Dev   Author: 9-9-0   File: bodegaREQ.py    GNU General Public License v3.0 6 votes vote down vote up
def loadCartAndCheckout(self):
        #Import Cookies
        driver = webdriver.Chrome(executable_path="./chromedriver")
        driver.delete_all_cookies()
        driver.get(self.URL_cart)

        cookies = requests.utils.dict_from_cookiejar(self.user_session.cookies)
        
        for cookie in cookies.items():
            cookie_dict = {'name': '',
                           'value': '',
                           'path': '/'}
            cookie_dict['name'] = cookie[0]
            cookie_dict['value'] = cookie[1]
            driver.add_cookie(cookie_dict)
                          
        driver.get(self.URL_cart)
        #time.sleep(5)
        #driver.quit() 
Example 7
Project: instabotai   Author: instabotai   File: api.py    Apache License 2.0 6 votes vote down vote up
def load_cookie(self, fname):
        # Python2 compatibility
        if PY2:
            FileNotFoundError = IOError

        try:
            with open(fname, 'r') as f:
                self.session = requests.Session()
                self.session.cookies = requests.utils.cookiejar_from_dict(json.load(f))
            cookie_username = self.cookie_dict['ds_user']
            assert cookie_username == self.username
        except FileNotFoundError:
            raise Exception('Cookie file `{}` not found'.format(fname))
        except (TypeError, EOFError):
            os.remove(fname)
            msg = ('An error occured opening the cookie `{}`, '
                   'it will be removed an recreated.')
            raise Exception(msg.format(fname))
        except AssertionError:
            msg = 'The loaded cookie was for {} instead of {}.'
            raise Exception(msg.format(cookie_username, self.username)) 
Example 8
Project: Clawer   Author: Saviio   File: utils.py    MIT License 6 votes vote down vote up
def douban_tranlation(matched_name): 
    import urllib2
    priority = {'movie':10, 'tv':9}
    qs = utils.url_encode({'q': matched_name})
    search_api = 'http://api.douban.com/v2/movie/search?%s' % (qs)
    rs = json.loads(urllib2.urlopen(search_api).read())

    items = rs['subjects']
    ret = None

    for item in items:
        if ret == None or (priority[ret['subtype']] < priority[item['subtype']]):
            ret = {
                'origin'  : item['original_title'], 
                'subtype' : item['subtype']
            }

    return ret 
Example 9
Project: google-alerts   Author: 9b   File: __init__.py    MIT License 6 votes vote down vote up
def _session_check(self):
        """Attempt to authenticate the user through a session file.

        This process is done to avoid having to authenticate the user every
        single time. It uses a session file that is saved when a valid session
        is captured and then reused. Because sessions can expire, we need to
        test the session prior to calling the user authenticated. Right now
        that is done with a test string found in an unauthenticated session.
        This approach is not an ideal method, but it works.
        """
        if not os.path.exists(SESSION_FILE):
            self._log.debug("Session file does not exist")
            return False
        with open(SESSION_FILE, 'rb') as f:
            cookies = requests.utils.cookiejar_from_dict(pickle.load(f))
            self._session.cookies = cookies
            self._log.debug("Loaded cookies from session file")
        response = self._session.get(url=self.TEST_URL, headers=self.HEADERS)
        if self.TEST_KEY in str(response.content):
            self._log.debug("Session file appears invalid")
            return False
        self._is_authenticated = True
        self._process_state()
        return True 
Example 10
Project: zhihu-api   Author: lzjun567   File: auth.py    MIT License 6 votes vote down vote up
def authenticated(func):
    def wrapper(self, *args, **kwargs):
        success = False
        # 先判断有没有cookie文件, 再判断cookie是否有效
        if 'z_c0' in requests.utils.dict_from_cookiejar(self.cookies):
            from ..url import URL
            r = self._execute(method="get", url=URL.profile(user_slug="zhijun-liu"))
            success = r.ok
        while not success:
            account = input("请输入Email或者手机号码:")
            password = input("请输入密码:")
            obj = Account()
            data = obj.login(account, password)
            if "error" not in data:
                success = True
                self.cookies = obj.cookies
            else:
                print(data["error"]["message"])
        else:
            return func(self, *args, **kwargs)

    return wrapper 
Example 11
Project: Office365-REST-Python-Client   Author: vgrem   File: saml_token_provider.py    MIT License 6 votes vote down vote up
def acquire_authentication_cookie(self, options):
        """Retrieve SPO auth cookie"""
        logger = self.logger(self.acquire_authentication_cookie.__name__)

        url = options['endpoint']

        session = requests.session()
        logger.debug_secrets("session: %s\nsession.post(%s, data=%s)", session, url, self.token)
        session.post(url, data=self.token, headers={'Content-Type': 'application/x-www-form-urlencoded'})
        logger.debug_secrets("session.cookies: %s", session.cookies)
        cookies = requests.utils.dict_from_cookiejar(session.cookies)
        logger.debug_secrets("cookies: %s", cookies)
        if 'FedAuth' in cookies and 'rtFa' in cookies:
            self.FedAuth = cookies['FedAuth']
            self.rtFa = cookies['rtFa']
            return True
        self.error = "An error occurred while retrieving auth cookies"
        logger.error(self.error)
        return False 
Example 12
Project: instaloader   Author: instaloader   File: instaloadercontext.py    MIT License 5 votes vote down vote up
def copy_session(session: requests.Session) -> requests.Session:
    """Duplicates a requests.Session."""
    new = requests.Session()
    new.cookies = requests.utils.cookiejar_from_dict(requests.utils.dict_from_cookiejar(session.cookies))
    new.headers = session.headers.copy() # type: ignore
    return new 
Example 13
Project: instaloader   Author: instaloader   File: instaloadercontext.py    MIT License 5 votes vote down vote up
def save_session_to_file(self, sessionfile):
        """Not meant to be used directly, use :meth:`Instaloader.save_session_to_file`."""
        pickle.dump(requests.utils.dict_from_cookiejar(self._session.cookies), sessionfile) 
Example 14
Project: instaloader   Author: instaloader   File: instaloadercontext.py    MIT License 5 votes vote down vote up
def load_session_from_file(self, username, sessionfile):
        """Not meant to be used directly, use :meth:`Instaloader.load_session_from_file`."""
        session = requests.Session()
        session.cookies = requests.utils.cookiejar_from_dict(pickle.load(sessionfile))
        session.headers.update(self._default_http_header())
        session.headers.update({'X-CSRFToken': session.cookies.get_dict()['csrftoken']})
        self._session = session
        self.username = username 
Example 15
Project: python_practice   Author: zhengbomo   File: Server.py    MIT License 5 votes vote down vote up
def __get_cookie():
        path = os.getcwd()
        path = os.path.join(path, 'cookie')

        if os.path.isfile(path):
            with open('cookie') as f:
                cookies = requests.utils.cookiejar_from_dict(pickle.load(f))
                return cookies, True
        else:
            return None, False 
Example 16
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_auth_from_url(self):
        url = 'http://user:[email protected]/path?query=yes'
        assert ('user', 'pass') == requests.utils.get_auth_from_url(url) 
Example 17
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_auth_from_url_encoded_spaces(self):
        url = 'http://user:pass%[email protected]/path?query=yes'
        assert ('user', 'pass pass') == requests.utils.get_auth_from_url(url) 
Example 18
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_auth_from_url_not_encoded_spaces(self):
        url = 'http://user:pass [email protected]/path?query=yes'
        assert ('user', 'pass pass') == requests.utils.get_auth_from_url(url) 
Example 19
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_auth_from_url_percent_chars(self):
        url = 'http://user%25user:[email protected]/path?query=yes'
        assert ('user%user', 'pass') == requests.utils.get_auth_from_url(url) 
Example 20
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_auth_from_url_encoded_hashes(self):
        url = 'http://user:pass%[email protected]/path?query=yes'
        assert ('user', 'pass#pass') == requests.utils.get_auth_from_url(url) 
Example 21
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_html_charset(self):
        """HTML5 meta charset attribute"""
        content = '<meta charset="UTF-8">'
        encodings = requests.utils.get_encodings_from_content(content)
        assert len(encodings) == 1
        assert encodings[0] == 'UTF-8' 
Example 22
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_html4_pragma(self):
        """HTML4 pragma directive"""
        content = '<meta http-equiv="Content-type" content="text/html;charset=UTF-8">'
        encodings = requests.utils.get_encodings_from_content(content)
        assert len(encodings) == 1
        assert encodings[0] == 'UTF-8' 
Example 23
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_xhtml_pragma(self):
        """XHTML 1.x served with text/html MIME type"""
        content = '<meta http-equiv="Content-type" content="text/html;charset=UTF-8" />'
        encodings = requests.utils.get_encodings_from_content(content)
        assert len(encodings) == 1
        assert encodings[0] == 'UTF-8' 
Example 24
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_xml(self):
        """XHTML 1.x served as XML"""
        content = '<?xml version="1.0" encoding="UTF-8"?>'
        encodings = requests.utils.get_encodings_from_content(content)
        assert len(encodings) == 1
        assert encodings[0] == 'UTF-8' 
Example 25
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_precedence(self):
        content = '''
        <?xml version="1.0" encoding="XML"?>
        <meta charset="HTML5">
        <meta http-equiv="Content-type" content="text/html;charset=HTML4" />
        '''.strip()
        encodings = requests.utils.get_encodings_from_content(content)
        assert encodings == ['HTML5', 'HTML4', 'XML'] 
Example 26
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_environ_proxies_ip_ranges(self):
        """Ensures that IP addresses are correctly matches with ranges
        in no_proxy variable."""
        from requests.utils import get_environ_proxies
        os.environ['no_proxy'] = "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1"
        assert get_environ_proxies('http://192.168.0.1:5000/') == {}
        assert get_environ_proxies('http://192.168.0.1/') == {}
        assert get_environ_proxies('http://172.16.1.1/') == {}
        assert get_environ_proxies('http://172.16.1.1:5000/') == {}
        assert get_environ_proxies('http://192.168.1.1:5000/') != {}
        assert get_environ_proxies('http://192.168.1.1/') != {} 
Example 27
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_environ_proxies(self):
        """Ensures that IP addresses are correctly matches with ranges
        in no_proxy variable."""
        from requests.utils import get_environ_proxies
        os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1"
        assert get_environ_proxies(
            'http://localhost.localdomain:5000/v1.0/') == {}
        assert get_environ_proxies('http://www.requests.com/') != {} 
Example 28
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_guess_filename_when_int(self):
        from requests.utils import guess_filename
        assert None is guess_filename(1) 
Example 29
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_guess_filename_when_filename_is_an_int(self):
        from requests.utils import guess_filename
        fake = type('Fake', (object,), {'name': 1})()
        assert None is guess_filename(fake) 
Example 30
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_guess_filename_with_file_like_obj(self):
        from requests.utils import guess_filename
        from requests import compat
        fake = type('Fake', (object,), {'name': b'value'})()
        guessed_name = guess_filename(fake)
        assert b'value' == guessed_name
        assert isinstance(guessed_name, compat.bytes) 
Example 31
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_is_ipv4_address(self):
        from requests.utils import is_ipv4_address
        assert is_ipv4_address('8.8.8.8')
        assert not is_ipv4_address('8.8.8.8.8')
        assert not is_ipv4_address('localhost.localdomain') 
Example 32
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_is_valid_cidr(self):
        from requests.utils import is_valid_cidr
        assert not is_valid_cidr('8.8.8.8')
        assert is_valid_cidr('192.168.1.0/24') 
Example 33
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_dotted_netmask(self):
        from requests.utils import dotted_netmask
        assert dotted_netmask(8) == '255.0.0.0'
        assert dotted_netmask(24) == '255.255.255.0'
        assert dotted_netmask(25) == '255.255.255.128' 
Example 34
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_address_in_network(self):
        from requests.utils import address_in_network
        assert address_in_network('192.168.1.1', '192.168.1.0/24')
        assert not address_in_network('172.16.0.1', '192.168.1.0/24') 
Example 35
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_auth_from_url(self):
        """Ensures that username and password in well-encoded URI as per
        RFC 3986 are correclty extracted."""
        from requests.utils import get_auth_from_url
        from requests.compat import quote
        percent_encoding_test_chars = "%!*'();:@&=+$,/?#[] "
        url_address = "request.com/url.html#test"
        url = "http://" + quote(
            percent_encoding_test_chars, '') + ':' + quote(
            percent_encoding_test_chars, '') + '@' + url_address
        (username, password) = get_auth_from_url(url)
        assert username == percent_encoding_test_chars
        assert password == percent_encoding_test_chars 
Example 36
Project: SublimeBart   Author: ganemone   File: test_requests.py    MIT License 5 votes vote down vote up
def test_requote_uri_properly_requotes(self):
        """Ensure requoting doesn't break expectations."""
        from requests.utils import requote_uri
        quoted = 'http://example.com/fiz?buz=%25ppicture'
        assert quoted == requote_uri(quoted) 
Example 37
Project: instabotai   Author: instabotai   File: api.py    Apache License 2.0 5 votes vote down vote up
def save_cookie(self, fname):
        with open(fname, 'w') as f:
            json.dump(requests.utils.dict_from_cookiejar(self.session.cookies), f) 
Example 38
Project: Clawer   Author: Saviio   File: utils.py    MIT License 5 votes vote down vote up
def save_cookies(session):
    with open(cookie_path, 'w') as f:
        pickle.dump(requests.utils.dict_from_cookiejar(session.cookies), f) 
Example 39
Project: Clawer   Author: Saviio   File: utils.py    MIT License 5 votes vote down vote up
def start_session(forceNew = False):
    session = requests.session()
    session.headers.update(headers)
    if os.path.exists(cookie_path) and forceNew != True:
        with open(cookie_path) as f:
            cookies = requests.utils.cookiejar_from_dict(pickle.load(f))
            session.cookies.update(cookies)
            session._new_ = False
    else:
        session._new_ = True
    return session 
Example 40
Project: kook-insta-bot   Author: bojbaj   File: login.py    MIT License 5 votes vote down vote up
def reload_session(self):
    with open('cookies/cookie_' + self.user_id + '.ck') as f:
        cookies = requests.utils.cookiejar_from_dict(pickle.load(f))
        self.s.cookies = cookies 
Example 41
Project: google-alerts   Author: 9b   File: __init__.py    MIT License 5 votes vote down vote up
def authenticate(self):
        """Authenticate the user and setup our state."""
        valid = self._session_check()
        if self._is_authenticated and valid:
            self._log.debug("[!] User has already authenticated")
            return
        init = self._session.get(url=self.LOGIN_URL, headers=self.HEADERS)
        soup = BeautifulSoup(init.content, "html.parser")
        soup_login = soup.find('form').find_all('input')
        post_data = dict()
        for u in soup_login:
            if u.has_attr('name') and u.has_attr('value'):
                post_data[u['name']] = u['value']
        post_data['Email'] = self._email
        post_data['Passwd'] = self._password
        response = self._session.post(url=self.AUTH_URL, data=post_data,
                                      headers=self.HEADERS)
        if self.CAPTCHA_KEY in str(response.content):
            raise AccountCaptcha('Google is forcing a CAPTCHA. To get around this issue, run the google-alerts with the seed option to open an interactive authentication session. Once authenticated, this module will cache your session and load that in the future')
        cookies = [x.name for x in response.cookies]
        if 'SIDCC' not in cookies:
            raise InvalidCredentials("Email or password was incorrect.")
        with open(SESSION_FILE, 'wb') as f:
            cookies = requests.utils.dict_from_cookiejar(self._session.cookies)
            pickle.dump(cookies, f, protocol=2)
            self._log.debug("Saved session to disk for future reference")
        self._log.debug("User successfully authenticated")
        self._is_authenticated = True
        self._process_state()
        return 
Example 42
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_auth_from_url(self):
        url = 'http://user:[email protected]/path?query=yes'
        assert ('user', 'pass') == requests.utils.get_auth_from_url(url) 
Example 43
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_auth_from_url_encoded_spaces(self):
        url = 'http://user:pass%[email protected]/path?query=yes'
        assert ('user', 'pass pass') == requests.utils.get_auth_from_url(url) 
Example 44
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_auth_from_url_not_encoded_spaces(self):
        url = 'http://user:pass [email protected]/path?query=yes'
        assert ('user', 'pass pass') == requests.utils.get_auth_from_url(url) 
Example 45
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_auth_from_url_percent_chars(self):
        url = 'http://user%25user:[email protected]/path?query=yes'
        assert ('user%user', 'pass') == requests.utils.get_auth_from_url(url) 
Example 46
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_auth_from_url_encoded_hashes(self):
        url = 'http://user:pass%[email protected]/path?query=yes'
        assert ('user', 'pass#pass') == requests.utils.get_auth_from_url(url) 
Example 47
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_html_charset(self):
        """HTML5 meta charset attribute"""
        content = '<meta charset="UTF-8">'
        encodings = requests.utils.get_encodings_from_content(content)
        assert len(encodings) == 1
        assert encodings[0] == 'UTF-8' 
Example 48
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_html4_pragma(self):
        """HTML4 pragma directive"""
        content = '<meta http-equiv="Content-type" content="text/html;charset=UTF-8">'
        encodings = requests.utils.get_encodings_from_content(content)
        assert len(encodings) == 1
        assert encodings[0] == 'UTF-8' 
Example 49
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_xhtml_pragma(self):
        """XHTML 1.x served with text/html MIME type"""
        content = '<meta http-equiv="Content-type" content="text/html;charset=UTF-8" />'
        encodings = requests.utils.get_encodings_from_content(content)
        assert len(encodings) == 1
        assert encodings[0] == 'UTF-8' 
Example 50
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_xml(self):
        """XHTML 1.x served as XML"""
        content = '<?xml version="1.0" encoding="UTF-8"?>'
        encodings = requests.utils.get_encodings_from_content(content)
        assert len(encodings) == 1
        assert encodings[0] == 'UTF-8' 
Example 51
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_precedence(self):
        content = '''
        <?xml version="1.0" encoding="XML"?>
        <meta charset="HTML5">
        <meta http-equiv="Content-type" content="text/html;charset=HTML4" />
        '''.strip()
        encodings = requests.utils.get_encodings_from_content(content)
        assert encodings == ['HTML5', 'HTML4', 'XML'] 
Example 52
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_super_len_correctly_calculates_len_of_partially_read_file(self):
        """Ensure that we handle partially consumed file like objects."""
        from requests.utils import super_len
        s = StringIO.StringIO()
        s.write('foobarbogus')
        assert super_len(s) == 0 
Example 53
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_environ_proxies_ip_ranges(self):
        """Ensures that IP addresses are correctly matches with ranges
        in no_proxy variable."""
        from requests.utils import get_environ_proxies
        os.environ['no_proxy'] = "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1"
        assert get_environ_proxies('http://192.168.0.1:5000/') == {}
        assert get_environ_proxies('http://192.168.0.1/') == {}
        assert get_environ_proxies('http://172.16.1.1/') == {}
        assert get_environ_proxies('http://172.16.1.1:5000/') == {}
        assert get_environ_proxies('http://192.168.1.1:5000/') != {}
        assert get_environ_proxies('http://192.168.1.1/') != {} 
Example 54
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_environ_proxies(self):
        """Ensures that IP addresses are correctly matches with ranges
        in no_proxy variable."""
        from requests.utils import get_environ_proxies
        os.environ['no_proxy'] = "127.0.0.1,localhost.localdomain,192.168.0.0/24,172.16.1.1"
        assert get_environ_proxies(
            'http://localhost.localdomain:5000/v1.0/') == {}
        assert get_environ_proxies('http://www.requests.com/') != {} 
Example 55
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_select_proxies(self):
        """Make sure we can select per-host proxies correctly."""
        from requests.utils import select_proxy
        proxies = {'http': 'http://http.proxy',
                   'http://some.host': 'http://some.host.proxy'}
        assert select_proxy('hTTp://u:[email protected]/path', proxies) == 'http://some.host.proxy'
        assert select_proxy('hTTp://u:[email protected]/path', proxies) == 'http://http.proxy'
        assert select_proxy('hTTps://Other.Host', proxies) is None 
Example 56
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_guess_filename_when_int(self):
        from requests.utils import guess_filename
        assert None is guess_filename(1) 
Example 57
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_guess_filename_with_file_like_obj(self):
        from requests.utils import guess_filename
        from requests import compat
        fake = type('Fake', (object,), {'name': b'value'})()
        guessed_name = guess_filename(fake)
        assert b'value' == guessed_name
        assert isinstance(guessed_name, compat.bytes) 
Example 58
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_guess_filename_with_unicode_name(self):
        from requests.utils import guess_filename
        from requests import compat
        filename = b'value'.decode('utf-8')
        fake = type('Fake', (object,), {'name': filename})()
        guessed_name = guess_filename(fake)
        assert filename == guessed_name
        assert isinstance(guessed_name, compat.str) 
Example 59
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_is_ipv4_address(self):
        from requests.utils import is_ipv4_address
        assert is_ipv4_address('8.8.8.8')
        assert not is_ipv4_address('8.8.8.8.8')
        assert not is_ipv4_address('localhost.localdomain') 
Example 60
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_is_valid_cidr(self):
        from requests.utils import is_valid_cidr
        assert not is_valid_cidr('8.8.8.8')
        assert is_valid_cidr('192.168.1.0/24') 
Example 61
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_dotted_netmask(self):
        from requests.utils import dotted_netmask
        assert dotted_netmask(8) == '255.0.0.0'
        assert dotted_netmask(24) == '255.255.255.0'
        assert dotted_netmask(25) == '255.255.255.128' 
Example 62
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_get_auth_from_url(self):
        """Ensures that username and password in well-encoded URI as per
        RFC 3986 are correclty extracted."""
        from requests.utils import get_auth_from_url
        from requests.compat import quote
        percent_encoding_test_chars = "%!*'();:@&=+$,/?#[] "
        url_address = "request.com/url.html#test"
        url = "http://" + quote(
            percent_encoding_test_chars, '') + ':' + quote(
            percent_encoding_test_chars, '') + '@' + url_address
        (username, password) = get_auth_from_url(url)
        assert username == percent_encoding_test_chars
        assert password == percent_encoding_test_chars 
Example 63
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_requote_uri_with_unquoted_percents(self):
        """Ensure we handle unquoted percent signs in redirects.

        See: https://github.com/kennethreitz/requests/issues/2356
        """
        from requests.utils import requote_uri
        bad_uri = 'http://example.com/fiz?buz=%ppicture'
        quoted = 'http://example.com/fiz?buz=%25ppicture'
        assert quoted == requote_uri(bad_uri) 
Example 64
Project: PythonGroceryList   Author: The01000001Team   File: test_requests.py    MIT License 5 votes vote down vote up
def test_requote_uri_properly_requotes(self):
        """Ensure requoting doesn't break expectations."""
        from requests.utils import requote_uri
        quoted = 'http://example.com/fiz?buz=%25ppicture'
        assert quoted == requote_uri(quoted) 
Example 65
Project: renrenBackup   Author: whusnoopy   File: crawler.py    MIT License 5 votes vote down vote up
def load_cookie(cls):
        cookies = None

        if os.path.exists(config.COOKIE_FILE):
            with open(config.COOKIE_FILE) as fp:
                try:
                    cookies = requests.utils.cookiejar_from_dict(json.load(fp))
                    logger.info('load cookies from {filename}'.format(filename=config.COOKIE_FILE))
                except json.decoder.JSONDecodeError:
                    cookies = None

        return cookies 
Example 66
Project: renrenBackup   Author: whusnoopy   File: crawler.py    MIT License 5 votes vote down vote up
def dump_cookie(self):
        cookies = self.session.cookies
        for cookie in cookies:
            if cookie.name == 't' and cookie.path != '/':
                cookies.clear(cookie.domain, cookie.path, cookie.name)
        with open(config.COOKIE_FILE, 'w') as fp:
            json.dump(requests.utils.dict_from_cookiejar(cookies), fp) 
Example 67
Project: adicart   Author: zweed4u   File: adicart.py    MIT License 5 votes vote down vote up
def _request(self, method, url, params=None, data=None, headers=None):
		# add passed headers
		if headers is not None:
			for key in headers.keys():
				self.headers[key] = headers[key]
		# implement method params
		self.response = self.session.request(method, url, headers=self.headers)
		#print('\tCookies for EditThisCookie\t')
		#print(f'####################################################################################################################################{self.colorText}')
		#for key, value in requests.utils.dict_from_cookiejar(self.response.cookies).items():
		#	print(f'{key}: {value}\n')
		#print(f'{COLOR_END}####################################################################################################################################')
		self.build_edit_this_cookie(requests.utils.dict_from_cookiejar(self.response.cookies)) 
Example 68
Project: python-nicolive-download   Author: ydixon   File: NicoSession.py    GNU General Public License v2.0 5 votes vote down vote up
def saveSessionCookie(self, session, outputPath):
    	with open(outputPath, 'w') as f:
    	    pickle.dump(requests.utils.dict_from_cookiejar(session.cookies), f) 
Example 69
Project: python-nicolive-download   Author: ydixon   File: NicoSession.py    GNU General Public License v2.0 5 votes vote down vote up
def loadSessionCookie(self, inputPath):
    	if not os.path.exists('cookiejar'):
    		return None

    	with open(inputPath) as f:
        	cookies = requests.utils.cookiejar_from_dict(pickle.load(f))
        	session = requests.session()
    		session.cookies = cookies
    	return session 
Example 70
Project: kook-insta-bot   Author: bojbaj   File: login.py    MIT License 4 votes vote down vote up
def login(self):
    self.login_post = {
        'username': self.user_login,
        'password': self.user_password
    }

    self.s.headers.update({
        'Accept': '*/*',
        'Accept-Language': self.accept_language,
        'Accept-Encoding': 'gzip, deflate, br',
        'Connection': 'keep-alive',
        'Content-Length': '0',
        'Host': 'www.instagram.com',
        'Origin': 'https://www.instagram.com',
        'Referer': 'https://www.instagram.com/',
        'User-Agent': self.user_agent,
        'X-Instagram-AJAX': '1',
        'Content-Type': 'application/x-www-form-urlencoded',
        'X-Requested-With': 'XMLHttpRequest'
    })
    r = self.s.get(url)
    self.s.headers.update({'X-CSRFToken': r.cookies['csrftoken']})
    time.sleep(1 * random.random())

    login = self.s.post(
        url_login, data=self.login_post, allow_redirects=True)
    self.s.headers.update({'X-CSRFToken': login.cookies['csrftoken']})
    self.csrftoken = login.cookies['csrftoken']
    # ig_vw=1536; ig_pr=1.25; ig_vh=772;  ig_or=landscape-primary;
    self.s.cookies['ig_vw'] = '1536'
    self.s.cookies['ig_pr'] = '1.25'
    self.s.cookies['ig_vh'] = '772'
    self.s.cookies['ig_or'] = 'landscape-primary'
    time.sleep(3 * random.random())

    if login.status_code == 200:
        response = json.loads(login.text)
        self.login_status = response['authenticated']
        if self.login_status == True:
            self.user_id = response['userId']
            with open('cookies/cookie_' + self.user_id + '.ck', 'w') as f:
                pickle.dump(requests.utils.dict_from_cookiejar(
                    self.s.cookies), f)           
    else:
        self.login_status = False

    return self.login_status, self.csrftoken, self.user_id