Python requests.TooManyRedirects() Examples

def test_update_notable_too_many_redirect(self, mocked_requests_post):
        """ Test update notable with wrong URL or connection issue"""
        print("Test update_notable returns TooManyRedirects\n")
        try:
            sim_status = 1

            sim_content = '<response><sessionKey>' + self.simSessionKey + '</sessionKey></response>'
            mocked_requests_post.return_value = self._generateResponse(sim_content, 200)
            splnk_utils = splunk_utils.SplunkUtils(host=self.fake_host,
                                                   port=self.fake_port,
                                                   username=self.fake_username,
                                                   password=self.fake_password,
                                                   verify=self.verify)

            mocked_requests_post.side_effect = requests.TooManyRedirects(Mock(status=404), "Ambiguous excetpion.")

            ret = splnk_utils.update_notable(event_id=self.simEventId,
                                             comment=self.simComment,
                                             status=sim_status,
                                             cafile=self.verify)
            #
            # request post throws exception
            #
            assert False
        except splunk_utils.RequestError as e:
            assert True 

def handle_requests_error(e):
    if isinstance(e, requests.ConnectionError):
        raise HTTPError("Connection error") from e
    if isinstance(e, requests.HTTPError):
        pass  # Raised when using .raise_for_status, so should never happen
    if isinstance(e, requests.URLRequired):
        pass  # Should never happen, we always prove valid URLs
    if isinstance(e, requests.TooManyRedirects):
        pass  # TODO: Consider using allow_redirects=False to prevent this
    if isinstance(e, requests.Timeout):
        pass  # Should never happen, we don't set timeouts

    raise HTTPError("Requests error") from e 

def request_json(self, url, return_code=200):
        """
        Request JSON content and expected return code.
        :param url:  URL to request
        :param return_code: expected return code or raise error
        :return: JSON, SuccessState
        """
        try:
            header = {
                'User-Agent': str(self.ua.google)
            }
            if not validators.url(url):
                self.print_yellow(
                    " [!] Invalid URL Requested: %s" % (str(url)))
                return {}, False
            r = requests.get(url, headers=header)
            if r.status_code != return_code:
                self.print_yellow(" [!] Request returned invalid status code: (CODE): %s (EXPECTED): %s" %
                                  (str(r.status_code), str(return_code)))
                return {}, False
            return r.content, True
        except requests.ConnectTimeout as e:
            self.print_red(
                " [!] Request ConnectionTimeout: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except requests.TooManyRedirects as e:
            self.print_red(
                " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except requests.HTTPError as e:
            self.print_red(
                " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except ConnectionError as e:
            self.print_red(
                " [!] Request ConnectionError: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except Exception as e:
            self.print_red(
                " [!] Request Unknown Error: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False 

def request_content(self, url, return_code=200):
        """
        Request content and expected return code.
        :param url:  URL to request
        :param return_code: expected return code or raise error
        :return: JSON, SuccessState
        """
        try:
            header = {
                'User-Agent': str(self.ua.google)
            }
            if not validators.url(url):
                self.print_yellow(
                    " [!] Invalid URL Requested: %s" % (str(url)))
                return {}, False
            r = requests.get(url, headers=header)
            if r.status_code != return_code:
                self.print_yellow(" [!] Request returned invalid status code: (CODE): %s (EXPECTED): %s" %
                                  (str(r.status_code), str(return_code)))
                return {}, False
            return r.content, True
        except requests.ConnectTimeout as e:
            self.print_red(
                " [!] Request ConnectionTimeout: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except requests.TooManyRedirects as e:
            self.print_red(
                " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except requests.HTTPError as e:
            self.print_red(
                " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except ConnectionError as e:
            self.print_red(
                " [!] Request ConnectionError: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except Exception as e:
            self.print_red(
                " [!] Request Unknown Error: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False 

def request_text(self, url, return_code=200):
        """
        Request content and expected return code.
        :param url:  URL to request
        :param return_code: expected return code or raise error
        :return: JSON, SuccessState
        """
        try:
            header = {
                'User-Agent': str(self.ua.google)
            }
            if not validators.url(url):
                self.print_yellow(
                    " [!] Invalid URL Requested: %s" % (str(url)))
                return {}, False
            r = requests.get(url, headers=header)
            if r.status_code != return_code:
                self.print_yellow(" [!] Request returned invalid status code: (CODE): %s (EXPECTED): %s" %
                                  (str(r.status_code), str(return_code)))
                return {}, False
            return r.text, True
        except requests.ConnectTimeout as e:
            self.print_red(
                " [!] Request ConnectionTimeout: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except requests.TooManyRedirects as e:
            self.print_red(
                " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except requests.HTTPError as e:
            self.print_red(
                " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except ConnectionError as e:
            self.print_red(
                " [!] Request ConnectionError: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except Exception as e:
            self.print_red(
                " [!] Request Unknown Error: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False 

def request_raw(self, url, return_code=200):
        """
        Request content and expected return code.
        :param url:  URL to request
        :param return_code: expected return code or raise error
        :return: JSON, SuccessState
        """
        try:
            header = {
                'User-Agent': str(self.ua.google)
            }
            if not validators.url(url):
                self.print_yellow(
                    " [!] Invalid URL Requested: %s" % (str(url)))
                return {}, False
            r = requests.get(url, headers=header)
            if r.status_code != return_code:
                self.print_yellow(" [!] Request returned invalid status code: (CODE): %s (EXPECTED): %s" %
                                  (str(r.status_code), str(return_code)))
                return {}, False
            return r, True
        except requests.ConnectTimeout as e:
            self.print_red(
                " [!] Request ConnectionTimeout: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except requests.TooManyRedirects as e:
            self.print_red(
                " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except requests.HTTPError as e:
            self.print_red(
                " [!] Request TooManyRedirects: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except ConnectionError as e:
            self.print_red(
                " [!] Request ConnectionError: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False
        except Exception as e:
            self.print_red(
                " [!] Request Unknown Error: (URL): %s (ERROR): %s" % (str(url), str(e)))
            return {}, False 

def crawler20x00(url, count):
    requests = session()
    visited_urls = set()
    queued_urls = OrderedDict({ url: '' })

    while len(queued_urls) > 0:
        (u, i) = queued_urls.popitem(last=False)
        try:
            req = requests.get(u, timeout=5)
            res = req.status_code
            root = etree.HTML(req.content, base_url=u)
        except wrn.ConnectionError as e:
            res = e
            continue
        except wrn.Timeout as e:
            res = e
            continue
        except wrn.TooManyRedirects as e:
            res = e
            continue
        except ValueError as e:
            res = e
            continue
        finally:
            visited_urls.add(u)
            pfx = "{}[{}]".format(i, len(visited_urls))
            if res == 200:
                print(B+' [+] Crawling : '+GR+pfx+'  '+C+u+G+'  ('+str(res)+')'+C+color.TR2+C)
                actual_uri.append(u)
            elif res == 404:
                print(B+' [+] Crawling : '+GR+pfx+'  '+C+u+R+'  ('+str(res)+')')
            else:
                print(B+' [+] Crawling : '+GR+pfx+'  '+C+u+O+'  ('+str(res)+')'+C)

        if root is None: continue

        for a in root.xpath('//a'):
            if (len(visited_urls) + len(queued_urls) >= count):
                break
            href = a.get('href')
            if href is None: continue
            (uj, sep, ui) = urljoin(a.base, href).partition('#')
            if uj not in visited_urls and uj not in queued_urls:
                if uj.startswith('http'):
                    queued_urls[uj] = pfx
            if (len(visited_urls) >= count):
                break 

def test_wrap_requests_exception(self):
        mock_return = mock.Mock()
        mock_return.status_code = 200
        return_value = mock_return
        mock_func = mock.Mock()
        mock_func.__name__ = 'get'
        mock_func.return_value = return_value
        mock_func.side_effect = requests.TooManyRedirects
        mock_tracer = MockTracer()

        patch = mock.patch(
            'opencensus.ext.requests.trace.execution_context.'
            'get_opencensus_tracer',
            return_value=mock_tracer)

        patch_thread = mock.patch(
            'opencensus.ext.requests.trace.execution_context.'
            'is_exporter',
            return_value=False)

        wrapped = trace.wrap_requests(mock_func)

        url = 'http://localhost:8080/test'

        with patch, patch_thread:
            with self.assertRaises(requests.TooManyRedirects):
                wrapped(url)

        expected_attributes = {
            'component': 'HTTP',
            'http.host': 'localhost:8080',
            'http.method': 'GET',
            'http.path': '/test',
            'http.url': url,
        }
        expected_name = '/test'
        expected_status = status_module.Status(2, '')

        self.assertEqual(span_module.SpanKind.CLIENT,
                         mock_tracer.current_span.span_kind)
        self.assertEqual(expected_attributes,
                         mock_tracer.current_span.attributes)
        self.assertEqual(expected_name, mock_tracer.current_span.name)
        self.assertEqual(
            expected_status.__dict__,
            mock_tracer.current_span.status.__dict__
        )
        self.assertRaises(requests.TooManyRedirects, mock_func) 

def test_wrap_session_request_exception(self):
        wrapped = mock.Mock(return_value=mock.Mock(status_code=200))
        wrapped.side_effect = requests.TooManyRedirects

        mock_tracer = MockTracer(
            propagator=mock.Mock(
                to_headers=lambda x: {'x-trace': 'some-value'}))

        patch = mock.patch(
            'opencensus.ext.requests.trace.execution_context.'
            'get_opencensus_tracer',
            return_value=mock_tracer)
        patch_thread = mock.patch(
            'opencensus.ext.requests.trace.execution_context.'
            'is_exporter',
            return_value=False)

        url = 'http://localhost:8080/test'
        request_method = 'POST'
        kwargs = {}

        with patch, patch_thread:
            with self.assertRaises(requests.TooManyRedirects):
                trace.wrap_session_request(
                    wrapped, 'Session.request',
                    (request_method, url), kwargs
                )

        expected_attributes = {
            'component': 'HTTP',
            'http.host': 'localhost:8080',
            'http.method': 'POST',
            'http.path': '/test',
            'http.url': url,
        }
        expected_name = '/test'
        expected_status = status_module.Status(2, '')

        self.assertEqual(span_module.SpanKind.CLIENT,
                         mock_tracer.current_span.span_kind)
        self.assertEqual(expected_attributes,
                         mock_tracer.current_span.attributes)
        self.assertEqual(kwargs['headers']['x-trace'], 'some-value')
        self.assertEqual(expected_name, mock_tracer.current_span.name)
        self.assertEqual(
            expected_status.__dict__,
            mock_tracer.current_span.status.__dict__
        ) 

def test_add_threat_intel_item_errors(self, mocked_requests_post):
        # 1. Connect successfully
        sim_content = '<response><sessionKey>' + self.simSessionKey + '</sessionKey></response>'
        mocked_requests_post.return_value = self._generateResponse(sim_content, 200)
        splnk_utils = splunk_utils.SplunkUtils(host=self.fake_host,
                                               port=self.fake_port,
                                               username=self.fake_username,
                                               password=self.fake_password,
                                               verify=self.verify)
        # 2. Simulate wrong intel type
        try:
            splnk_utils.add_threat_intel_item("Fake type", {}, False)
        except splunk_utils.RequestError as e:
            print("Fake intel type causes exception as expected.")
            assert(True)

        # 3. Simulate RequestException
        threat_type = "ip_intel"
        mocked_requests_post.side_effect = requests.RequestException(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True

        # 4. Simulate ConnectionError
        mocked_requests_post.side_effect = requests.ConnectionError(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True

        # 5. Simulate HttpError
        mocked_requests_post.side_effect = requests.HTTPError(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True

        # 6. Simulate URLRequired
        mocked_requests_post.side_effect = requests.URLRequired(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True

        # 7. Simulate TooManyRedirects
        mocked_requests_post.side_effect = requests.TooManyRedirects(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True 

def update_notable(self, event_id, comment, status, cafile):
        """
        Update notable event
        :param event_id: event_id for notable event to be updated
        :param comment: comment to add to the notable event
        :param status: status of the notable event to change to
        :param cafile: Verify HTTPS cert or not
        :return:
        """

        headers = dict()
        headers["Authorization"] = "Splunk {}".format(self.session_key)

        args = dict()
        args["comment"] = comment
        args["status"] = status
        args["ruleUIDs"] = [event_id]

        ret = None
        url = self.base_url + "/services/notable_update"

        try:
            resp = requests.post(url,
                                 headers=headers,
                                 data=args,
                                 verify=cafile)

            #
            # We shall just return the response in json and let the post process
            # to make decision.
            #
            ret = {"status_code": resp.status_code,
                   "content": resp.json()}

        except requests.ConnectionError as e:
            raise RequestError(url, "Connection error. " + str(e))
        except requests.HTTPError as e:
            raise RequestError(url, "An HTTP error. " + str(e))
        except requests.URLRequired as e:
            raise RequestError(url, "An valid URL is required.")
        except requests.TooManyRedirects as e:
            raise RequestError(url, "Too many redirects")
        except requests.RequestException as e:
            raise RequestError(url, "Ambiguous exception when handling request. " + str(e))
        return ret 

def add_threat_intel_item(self, threat_type, threat_dict, cafile):
        """
        Add a new threat intel item to the ThreatIntelligence collections
        :param threat_type: ip_intel, file_intel, user_intel, http_intel, email_intel, service_intel
                         process_intel, registry_intel, or certificate_intel
        :param threat_dict:
        :param cafile:
        :return:
        """
        headers = dict()
        headers["Authorization"] = "Splunk {}".format(self.session_key)

        url = self.base_url + "/services/data/threat_intel/item/" + threat_type

        if threat_type not in self.SUPPORTED_THREAT_TYPE:
            raise RequestError(url, "{} is not supported")

        item = {"item": json.dumps(threat_dict)}

        try:
            resp = requests.post(url,
                                 headers=headers,
                                 data=item,
                                 verify=cafile)

            #
            # We shall just return the response in json and let the post process
            # to make decision.
            #
            ret = {"status_code": resp.status_code,
                   "content": resp.json()}

        except requests.ConnectionError as e:
            raise RequestError(url, "Connection error. " + str(e))
        except requests.HTTPError as e:
            raise RequestError(url, "An HTTP error. " + str(e))
        except requests.URLRequired as e:
            raise RequestError(url, "An valid URL is required.")
        except requests.TooManyRedirects as e:
            raise RequestError(url, "Too many redirects")
        except requests.RequestException as e:
            raise RequestError(url, "Ambiguous exception when handling request. " + str(e))
        return ret