Python requests.response() Examples

The following are 30 code examples of requests.response(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module requests , or try the search function .
Example #1
Source File: base.py    From container-pipeline-service with GNU General Public License v3.0 6 votes vote down vote up
def response_data(response, bad_json=False):
        """
        Extracts data from a requests response object
        :param response: The response from which we need to extract information
        :type response: requests.response
        :param bad_json: Default False: If true, uses ast eval instead of json
        load
        :type bad_json bool
        :return: response result in Pythonic form, if it can get it, Else None
        :raises Exception
        """
        data = None
        if response:
            if not bad_json:
                try:
                    data = json_to_python(response.text)
                except Exception:
                    data = parse_literals(response.text)
            else:
                data = parse_literals(response.text)
        return data 
Example #2
Source File: test_integration.py    From pygmy with MIT License 6 votes vote down vote up
def test_login_shorten(self):
        response = requests.post(self.url + '/signup', data=self.user_data, headers=self.headers)
        with requests.Session() as sess:
            response = sess.post(self.url + '/login', data=self.login_data, headers=self.headers)
            self.cookies = sess.cookies
            self.assertTrue('Welcome Test' in response.text)

            # Shorten the URL
            data = self.data
            data['long_url'] = 'https://example.com/1'
            response = sess.post(self.url + '/shorten', data=data, headers=self.headers)
            short_url = self._get_short_url_from_response(response)

            self.assertEqual(response.status_code, 200)
            self.assertTrue('value="{}"'.format(short_url) in response.text)
            self.assertTrue('Welcome Test' in response.text)
            self.assertTrue('Copy To Clipboard' in response.text)
            self.assertTrue('Shorten Another Link' in response.text)

            # verify its on dashboard
            response = sess.get(self.url + '/dashboard')
            short_code = short_url.split('/')[-1]
            self.assertEqual(response.status_code, 200)
            self.assertTrue('{}+'.format(short_code) in response.text)
            self.assertTrue('https://example.com/1' in response.text) 
Example #3
Source File: test_integration.py    From pygmy with MIT License 6 votes vote down vote up
def test_link_hits(self):
        data = self.data
        data['long_url'] = 'http://example.com/index'
        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        short_url = self._get_short_url_from_response(response)

        # Open link
        for i in range(2):
            requests.get(short_url)
            stats_page = requests.get(short_url + '+')
            self.assertEqual(stats_page.status_code, 200)
            self.assertTrue('Total Hits: {}'.format(i+1) in stats_page.text)

    # def test_link_stats(self):
    #     pass
    #
    # def test_secret_link_stats(self):
    #     pass
    #
    # def test_expired_link_stats(self):
    #     pass
    # 
Example #4
Source File: test_integration.py    From pygmy with MIT License 6 votes vote down vote up
def test_custom_link_stats(self):
        data = self.data
        data['custom_url'] = 'ninja'
        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        short_url = self._get_short_url_from_response(response)

        # Open link
        for i in range(2):
            requests.get(short_url)
            stats_page = requests.get(short_url + '+')
            self.assertEqual(stats_page.status_code, 200)
            self.assertTrue('Total Hits: {}'.format(i+1) in stats_page.text)

    # #######################
    # # Test static resources
    # ####################### 
Example #5
Source File: test_integration.py    From pygmy with MIT License 6 votes vote down vote up
def test_shorten(self):
        data = self.data
        response = self.client.call('/shorten', data=data, headers=self.headers)
        short_url = self._get_short_url_from_response(response)
        self.assertEqual(response.status_code, 200)
        self.assertTrue('value="{}"'.format(short_url) in response.text)
        self.assertTrue('Copy To Clipboard' in response.text)
        self.assertTrue('Shorten Another Link' in response.text)
        self.assertTrue('{}+'.format(short_url) in response.text)

        # Repeat shorten should return the same result
        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        self.assertEqual(response.status_code, 200)
        self.assertTrue('value="{}"'.format(short_url) in response.text)

        # Test a different url
        data['long_url'] = 'https://www.python.org/'
        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        short_url = self._get_short_url_from_response(response)
        self.assertEqual(response.status_code, 200)
        self.assertTrue('value="{}"'.format(short_url) in response.text) 
Example #6
Source File: __init__.py    From ocl_web with Mozilla Public License 2.0 6 votes vote down vote up
def update_resource_version(self, owner_type, owner_id,
                                resource_id, version_id, resource_type, base_data):
        """
        Update source version. Limits update to only the description and released fields for now.
        :param owner_type: 'orgs' or 'users'
        :param owner_id: ID of the org/user owner
        :param resource_id: ID of the source/collection
        :param version_id: ID of the source/collection_version
        :param resource_type: 'source' or 'collection'
        :param base_data: Dictionary of fields to update
        :returns: response object
        """
        data = {}
        if 'description' in base_data:
            data['description'] = base_data['description']
        if 'released' in base_data:
            data['released'] = base_data['released']
        if 'retired' in base_data:
            data['retired'] = base_data['retired']
        if 'version_external_id' in base_data:
            data['version_external_id'] = base_data['version_external_id']

        result = self.put(owner_type, owner_id, resource_type, resource_id, version_id, **data)
        return result 
Example #7
Source File: __init__.py    From ocl_web with Mozilla Public License 2.0 6 votes vote down vote up
def get_json(self, *args):
        """
        Smarter GET request when you really want a json object back.
        Note: This is experimental -- not sure if this is the right abstraction.
        :param *args: All positional arguments are appended to the request URL.
        :returns: json string or None if error.
        :exception: Will raise exception if response status code is not 200.
        """
        results = self.get(*args)
        if results.status_code != requests.codes.ok:
            results.raise_for_status()
        if len(results.content) > 0:
            return results.json()
        else:
            return None


    # TODO: Retire get_by_url? 
Example #8
Source File: __init__.py    From ocl_web with Mozilla Public License 2.0 6 votes vote down vote up
def head(self, *args, **kwargs):
        """
        Issue HEAD request to API.
        :param *args: All positional arguments are appended to the request URL.
        :param **kwargs: These are not used at the moment, since this is a get request TODO
        :returns: requests.response object.
        """
        self.url = '%s/' % (self.host)
        if len(args) > 0:
            self.url = self.url + '/'.join(args) + '/'
        if self.debug:
            self.logger.debug('HEAD %s %s %s' % (self.url, json.dumps(kwargs), self.headers))

        # look for optional keyword argument params for constructing URL param
        # i.e. ?f1=v1&f2=v2
        params = kwargs.get('params')

        results = requests.head(self.url, params=params,
                                headers=self.headers)
        self.status_code = results.status_code
        if self.debug:
            self.debug_result(results)
        return results 
Example #9
Source File: __init__.py    From ocl_web with Mozilla Public License 2.0 6 votes vote down vote up
def post(self, type_name, *args, **kwargs):
        """
        Issue POST request to API.
        :param type_name: is a string specifying the type of the object according to the API.
        :param *args: The rest of the positional arguments will be appended to the post URL
        :param *kwargs: all the keyword arguments will become post data.
        :returns: response object from requests.
        """
        url = '%s/%s/' % (self.host, type_name)
        if len(args) > 0:
            url = url + '/'.join(args) + '/'
        if self.debug:
            self.logger.debug('POST %s %s %s' % (url, json.dumps(kwargs),
                                                 self.headers))

        results = requests.post(url, data=json.dumps(kwargs),
                                headers=self.headers)
        self.status_code = results.status_code
        if self.debug:
            self.debug_result(results)
        return results 
Example #10
Source File: requests_fun.py    From akshare with MIT License 5 votes vote down vote up
def requests_link(url: str, encoding: str = "utf-8", method: str = "get", data: Dict = None, headers: Dict = None):
    """
    利用 requests 请求网站, 爬取网站内容, 如网站链接失败, 可重复爬取 20 次
    :param url: string 网站地址
    :param encoding: string 编码类型: "utf-8", "gbk", "gb2312"
    :param method: string 访问方法: "get", "post"
    :param data: dict 上传数据: 键值对
    :param headers: dict 游览器请求头: 键值对
    :return: requests.response 爬取返回内容: response
    """
    i = 0
    while True:
        try:
            if method == "get":
                r = requests.get(url, timeout=20)
                r.encoding = encoding
                return r
            elif method == "post":
                r = requests.post(url, timeout=20, data=data, headers=headers)
                r.encoding = encoding
                return r
            else:
                raise ValueError("请提供正确的请求方式")
        except requests.exceptions.Timeout as e:
            i += 1
            print(f"第{str(i)}次链接失败, 最多尝试 20 次", e)
            time.sleep(5)
            if i > 20:
                return None 
Example #11
Source File: base.py    From iexfinance with Apache License 2.0 5 votes vote down vote up
def fetch(self, fmt_p=None, fmt_j=None):
        """Fetches latest data

        Prepares the query URL based on self.params and executes the request

        Returns
        -------
        response: requests.response
            A response object
        """
        url = self._prepare_query()
        data = self._execute_iex_query(url)
        return self._output_format(data, fmt_j=fmt_j, fmt_p=fmt_p) 
Example #12
Source File: base.py    From iexfinance with Apache License 2.0 5 votes vote down vote up
def _handle_error(self, response):
        """
        Handles all responses which return an error status code
        """
        raise IEXQueryError(response.status_code, response.text) 
Example #13
Source File: base.py    From iexfinance with Apache License 2.0 5 votes vote down vote up
def _execute_iex_query(self, url):
        """ Executes HTTP Request
        Given a URL, execute HTTP request from IEX server. If request is
        unsuccessful, attempt is made self.retry_count times with pause of
        self.pause in between.

        Parameters
        ----------
        url: str
            A properly-formatted url

        Returns
        -------
        response: requests.response
            Sends requests.response object to validator

        Raises
        ------
        IEXQueryError
            If problems arise when making the query
        """
        params = self.params
        params["token"] = self.token
        for _ in range(self.retry_count + 1):
            response = self.session.get(url=url, params=params)
            logger.debug("REQUEST: %s" % response.request.url)
            logger.debug("RESPONSE: %s" % response.status_code)
            if response.status_code == requests.codes.ok:
                return self._validate_response(response)
            time.sleep(self.pause)
        return self._handle_error(response) 
Example #14
Source File: base.py    From iexfinance with Apache License 2.0 5 votes vote down vote up
def _validate_response(self, response):
        """ Ensures response from IEX server is valid.

        Parameters
        ----------
        response: requests.response
            A requests.response object

        Returns
        -------
        response: Parsed JSON
            A json-formatted response

        Raises
        ------
        ValueError
            If a single Share symbol is invalid
        IEXQueryError
            If the JSON response is empty or throws an error

        """
        # log the number of messages used
        key = "iexcloud-messages-used"
        if key in response.headers:
            msg = response.headers[key]
        else:
            msg = "N/A"
        logger.info("MESSAGES USED: %s" % msg)

        if response.text == "Unknown symbol":
            raise IEXQueryError(response.status_code, response.text)
        try:
            json_response = response.json(
                parse_int=self.json_parse_int, parse_float=self.json_parse_float
            )
            if isinstance(json_response, str) and ("Error Message" in json_response):
                raise IEXQueryError(response.status_code, response.text)
        except ValueError:
            raise IEXQueryError(response.status_code, response.text)
        return json_response 
Example #15
Source File: test_integration.py    From pygmy with MIT License 5 votes vote down vote up
def test_logo_svg(self):
        response = requests.get(self.url + '/static/logo/logov2.svg')
        self.assertEqual(response.status_code, 200) 
Example #16
Source File: requests_fun.py    From akshare with MIT License 5 votes vote down vote up
def pandas_read_html_link(url: str, encoding: str = "utf-8", method: str = "get", data: Dict = None, headers: Dict = None):
    """
    利用 pandas 提供的 read_html 函数来直接提取网页中的表格内容, 如网站链接失败, 可重复爬取 20 次
    :param url: string 网站地址
    :param encoding: string 编码类型: "utf-8", "gbk", "gb2312"
    :param method: string 访问方法: "get", "post"
    :param data: dict 上传数据: 键值对
    :param headers: dict 游览器请求头: 键值对
    :return: requests.response 爬取返回内容: response
    """
    i = 0
    while True:
        try:
            if method == "get":
                r = requests.get(url, timeout=20)
                r.encoding = encoding
                r = pd.read_html(r.text, encoding=encoding)
                return r
            elif method == "post":
                r = requests.post(url, timeout=20, data=data, headers=headers)
                r.encoding = encoding
                r = pd.read_html(r.text, encoding=encoding)
                return r
            else:
                raise ValueError("请提供正确的请求方式")
        except requests.exceptions.Timeout as e:
            i += 1
            print(f"第{str(i)}次链接失败, 最多尝试20次", e)
            time.sleep(5)
            if i > 20:
                return None 
Example #17
Source File: test_integration.py    From pygmy with MIT License 5 votes vote down vote up
def test_custom_taken_link_shorten(self):
        custom_code = 'go'
        response = requests.get(self.url + '/check?custom_code={}'.format(custom_code))
        self.assertTrue(response.json().get('ok'))

        data = self.data
        data['custom_url'] = custom_code
        requests.post(self.url + '/shorten', data=data, headers=self.headers)

        response = requests.get(self.url + '/check?custom_code={}'.format(custom_code))
        self.assertFalse(response.json().get('ok'))

        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        self.assertEqual(response.status_code, 400) 
Example #18
Source File: test_integration.py    From pygmy with MIT License 5 votes vote down vote up
def test_custom_taken_link_availability(self):
        custom_code = 'logo'
        response = requests.get(self.url + '/check?custom_code={}'.format(custom_code))
        self.assertTrue(response.json().get('ok'))
        data = self.data
        data['custom_url'] = custom_code
        requests.post(self.url + '/shorten', data=data, headers=self.headers)
        response = requests.get(self.url + '/check?custom_code={}'.format(custom_code))
        self.assertFalse(response.json().get('ok')) 
Example #19
Source File: test_integration.py    From pygmy with MIT License 5 votes vote down vote up
def test_non_loggedin_dashboard(self):
        response = requests.get(self.url + '/dashboard')
        self.assertTrue(response.status_code == 400)
        self.assertTrue('Please log in again to continue.</h3>' in response.text)
    #
    # ###############
    # # Link Options
    # ############### 
Example #20
Source File: test_integration.py    From pygmy with MIT License 5 votes vote down vote up
def test_invalid_password_login(self):
        response = requests.post(self.url + '/signup', data=self.user_data, headers=self.headers)
        login_data = self.login_data.copy()
        login_data['password'] = 'i know it'
        response = requests.post(self.url + '/login', data=login_data, headers=self.headers)
        self.assertEqual(response.status_code, 400)
        self.assertTrue('LOGIN' in response.text)
        self.assertTrue('Invalid username or password.' in response.text) 
Example #21
Source File: test_integration.py    From pygmy with MIT License 5 votes vote down vote up
def test_logout(self):
        _ = requests.post(self.url + '/signup', data=self.user_data, headers=self.headers)
        with requests.Session() as sess:
            _ = sess.post(self.url + '/login', data=self.login_data, headers=self.headers)
            self.cookies = sess.cookies
            response = sess.get(self.url)
            self.assertEqual(response.status_code, 200)
            self.assertTrue('Welcome Test' in response.text)
            # logout
            response = sess.get(self.url + '/logout', headers=self.headers)
            self.assertEqual(response.url.strip('/'), self.url)
            self.assertEqual(response.status_code, 200)
            self.assertTrue('Welcome Test' not in response.text) 
Example #22
Source File: test_integration.py    From pygmy with MIT License 5 votes vote down vote up
def test_login(self):
        response = requests.post(self.url + '/signup', data=self.user_data, headers=self.headers)
        response = requests.get(self.url)
        self.assertTrue('LOGIN' in response.text)
        self.assertTrue('DASHBOARD' not in response.text)

        """Test redirection, hidden login, visible dashboard section, Welcome <username> section
        and dashboard table"""
        response = requests.post(self.url + '/login', data=self.login_data, headers=self.headers)
        self.assertEqual(response.status_code, 200)
        self.assertTrue(response.url == self.url + '/dashboard')
        self.assertTrue('DASHBOARD' in response.text)
        self.assertTrue('LOGIN' not in response.text)
        self.assertTrue('Welcome Test' in response.text)
        self.assertTrue('<table class="table table-bordered">' in response.text) 
Example #23
Source File: http.py    From airflow with Apache License 2.0 5 votes vote down vote up
def check_response(self, response):
        """
        Checks the status code and raise an AirflowException exception on non 2XX or 3XX
        status codes

        :param response: A requests response object
        :type response: requests.response
        """
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            self.log.error("HTTP error: %s", response.reason)
            self.log.error(response.text)
            raise AirflowException(str(response.status_code) + ":" + response.reason) 
Example #24
Source File: test_integration.py    From pygmy with MIT License 5 votes vote down vote up
def test_signup(self):
        data = self.user_data
        data['email'] = 'ninja@example.com'
        response = requests.post(self.url + '/signup', data=data, headers=self.headers)
        self.assertEqual(response.status_code, 200)
        self.assertTrue('Welcome Test' in response.text)
        self.assertIsNotNone(response.cookies.get('access_token'))

        response = requests.post(self.url + '/signup', data=self.user_data, headers=self.headers)
        self.assertEqual(response.status_code, 400)
        self.assertTrue('User exists' in response.text) 
Example #25
Source File: test_integration.py    From pygmy with MIT License 5 votes vote down vote up
def test_unshorten(self):
        data = self.data
        data['long_url'] = 'https://github.com'
        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        self.assertEqual(response.status_code, 200)
        short_url = self._get_short_url_from_response(response)

        response = requests.get(short_url)
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.url, data['long_url'])

    # User access 
Example #26
Source File: test_integration.py    From pygmy with MIT License 5 votes vote down vote up
def test_already_shortened_url_error(self):
        err_msg = 'URL is already a pygmy shortened link'
        data = self.data
        data['long_url'] = 'https://pygy.co'
        response = requests.post(self.url + '/shorten', data=data, headers=self.headers)
        self.assertEqual(response.status_code, 400)
        self.assertTrue(err_msg in response.text) 
Example #27
Source File: test_integration.py    From pygmy with MIT License 5 votes vote down vote up
def _get_short_url_from_response(self, response):
        """
        :param response: requests.response object
        :return: str
        """
        # TODO: Fix this
        resp_text = response.text
        idx = resp_text.find(self.url)
        idx_end = resp_text.find('" readonly autofocus id="short_url_blocked"')
        return resp_text[idx:idx_end] 
Example #28
Source File: __init__.py    From ocl_web with Mozilla Public License 2.0 5 votes vote down vote up
def create_source_version(self, owner_type, org_id, source_id, base_data):
        """
        Create a new source version.
        :param owner_type: 'orgs' or 'users'
        :param owner_id: ID of the org/user/ owner
        :param source_id: ID of the source
        :param base_data: Dictionary of fields for the new source version
        :returns: response object
        """
        data = {}
        data.update(base_data)
        result = self.post(owner_type, org_id, 'sources', source_id, 'versions', **data)
        return result 
Example #29
Source File: __init__.py    From ocl_web with Mozilla Public License 2.0 5 votes vote down vote up
def update_source(self, owner_type, owner_id, source_id, base_data, extras=[]):
        """
        Update source owned by org.
        :param owner_type: 'orgs' or 'users'
        :param owner_id: ID of the org/user/ owner
        :param base_data: is a dictionary of fields.
        :param extras: Extras to save to the resource
        :returns: response object.
        """
        data = {}
        data.update(base_data)
        result = self.put(owner_type, owner_id, 'sources', source_id, **data)
        return result 
Example #30
Source File: __init__.py    From ocl_web with Mozilla Public License 2.0 5 votes vote down vote up
def create_source(self, owner_type, owner_id, base_data, extras=[]):
        """
        Create source.
        :param owner_type: 'orgs' or 'users'
        :param owner_id: ID of the org/user/ owner
        :param base_data: Dictionary of fields for the new source version
        :param extras: Extras to save to the resource
        :returns: response object

        TODO(paynejd): create_sources extras not implemented
        """
        data = {}
        data.update(base_data)
        result = self.post(owner_type, owner_id, 'sources', **data)
        return result