Python requests.URLRequired() Examples

def setRemoteUrl(self):
        setRemoteUrldialog = remoteDialog.SetRemoteDialog(parent=self)
        if setRemoteUrldialog.exec_():
            self.database_url = 'http://' + setRemoteUrldialog.get_remote_url()
            self.remoteMode = setRemoteUrldialog.is_in_remote_mode()
            self.dowload_thread_num = setRemoteUrldialog.get_thread_num()
            self.server_image_list = setRemoteUrldialog.get_server_image_list()
        setRemoteUrldialog.destroy()
        print self.database_url
        if not os.path.exists(self.loadFilePath):
            os.makedirs(self.loadFilePath)
        if self.database_url:
            try:
                image_file = requests.get(
                    self.database_url + self.server_image_list)
            except requests.URLRequired as e:
                logging.error('can not get the server image list')
                return

            self.image_list = image_file.content.split('\n')[0:-1]
            self.server_image_num = len(self.image_list)
            if self.image_list:
                self.connect_remote_db = True
                self.toggleRemoteMode() 

def get_links(session, url, proxy = None):
    """
    Receive a url, and return a BeautifulSoup object
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.',
    }
    try:
        html = session.get(url, proxies = proxy, headers=headers)
        soup = BeautifulSoup(html.text, "lxml")
        return soup
    except URLRequired:
        return None
    except HTTPError:
        print("HTTPError") 

def invoke_elem(self, na_element, enable_tunneling=False):
        """Invoke the API on the server."""
        if na_element and not isinstance(na_element, NaElement):
            ValueError('NaElement must be supplied to invoke API')
        request_element = self._create_request(na_element, enable_tunneling)
        request_d = request_element.to_string()

        api_name = na_element.get_name()
        api_name_matches_regex = (re.match(self._api_trace_pattern, api_name)
                                  is not None)

        if self._trace and api_name_matches_regex:
            LOG.debug("Request: %s", request_element.to_string(pretty=True))

        if (not hasattr(self, '_session') or not self._session
                or self._refresh_conn):
            self._build_session()
        try:
            if hasattr(self, '_timeout'):
                response = self._session.post(
                    self._get_url(), data=request_d, timeout=self._timeout)
            else:
                response = self._session.post(
                    self._get_url(), data=request_d)
        except requests.HTTPError as e:
            raise NaApiError(e.errno, e.strerror)
        except requests.URLRequired as e:
            raise exception.StorageCommunicationException(six.text_type(e))
        except Exception as e:
            raise NaApiError(message=e)

        response_xml = response.text
        response_element = self._get_result(
            bytes(bytearray(response_xml, encoding='utf-8')))

        if self._trace and api_name_matches_regex:
            LOG.debug("Response: %s", response_element.to_string(pretty=True))

        return response_element 

def test_invoke_elem_urlerror(self):
        """Tests handling of URLError"""
        na_element = fake.FAKE_NA_ELEMENT
        self.mock_object(self.root, '_create_request', mock.Mock(
            return_value=fake.FAKE_NA_ELEMENT))
        self.mock_object(api, 'LOG')
        self.root._session = fake.FAKE_HTTP_SESSION
        self.mock_object(self.root, '_build_session')
        self.mock_object(self.root._session, 'post', mock.Mock(
            side_effect=requests.URLRequired()))

        self.assertRaises(exception.StorageCommunicationException,
                          self.root.invoke_elem,
                          na_element) 

def _exception_to_canonical_code(exc: Exception) -> StatusCanonicalCode:
    if isinstance(
        exc,
        (InvalidURL, InvalidSchema, MissingSchema, URLRequired, ValueError),
    ):
        return StatusCanonicalCode.INVALID_ARGUMENT
    if isinstance(exc, Timeout):
        return StatusCanonicalCode.DEADLINE_EXCEEDED
    return StatusCanonicalCode.UNKNOWN 

def test_update_notable_invalid_url(self, mocked_requests_post):
        """ Test update notable with wrong URL or connection issue"""
        print("Test update_notable returns HTTPError\n")
        try:
            sim_status = 1

            sim_content = '<response><sessionKey>' + self.simSessionKey + '</sessionKey></response>'
            mocked_requests_post.return_value = self._generateResponse(sim_content, 200)
            splnk_utils = splunk_utils.SplunkUtils(host=self.fake_host,
                                                   port=self.fake_port,
                                                   username=self.fake_username,
                                                   password=self.fake_password,
                                                   verify=self.verify)

            mocked_requests_post.side_effect = requests.URLRequired(Mock(status=404), "Ambiguous excetpion.")

            ret = splnk_utils.update_notable(event_id=self.simEventId,
                                             comment=self.simComment,
                                             status=sim_status,
                                             cafile=self.verify)
            #
            # request post throws exception
            #
            assert False
        except splunk_utils.RequestError as e:
            assert True 

def handle_requests_error(e):
    if isinstance(e, requests.ConnectionError):
        raise HTTPError("Connection error") from e
    if isinstance(e, requests.HTTPError):
        pass  # Raised when using .raise_for_status, so should never happen
    if isinstance(e, requests.URLRequired):
        pass  # Should never happen, we always prove valid URLs
    if isinstance(e, requests.TooManyRedirects):
        pass  # TODO: Consider using allow_redirects=False to prevent this
    if isinstance(e, requests.Timeout):
        pass  # Should never happen, we don't set timeouts

    raise HTTPError("Requests error") from e 

def _test_default_resolver(self, import_url, rules,
                               expected_urls_to_resolve=[],
                               expected_failure=False,
                               partial_err_msg=None):

        urls_to_resolve = []
        number_of_attempts = []

        class mock_requests_get(object):

            def __init__(self, url, timeout):
                self.status_code = 200
                self.text = 200
                number_of_attempts.append(1)
                if url not in urls_to_resolve:
                    urls_to_resolve.append(url)
                if url in [ORIGINAL_V1_URL, ORIGINAL_V2_URL, INVALID_V1_URL]:
                    raise requests.URLRequired(
                        'invalid url: {0}'.format(url))
                elif url == ILLEGAL_URL:
                    raise requests.URLRequired(
                        'unknown url type: {0}'.format(
                            url))
                elif url in [VALID_V1_URL, VALID_V2_URL]:
                    return None
                elif url == TIMEOUT_URL:
                    raise requests.ConnectionError(
                        'Timeout while trying to import')
                elif url == BAD_RESPONSE_CODE_URL:
                    self.status_code = 404
                    self.text = 404
                elif url == RETRY_URL:
                    if len(number_of_attempts) < MAX_NUMBER_RETRIES:
                        raise requests.ConnectionError(
                            'Timeout while trying to import')
                    else:
                        return None

        resolver = DefaultImportResolver(rules=rules)
        with mock.patch('requests.get', new=mock_requests_get,
                        create=True):
            with mock.patch(
                    'dsl_parser.import_resolver.abstract_import_resolver.'
                    'DEFAULT_RETRY_DELAY', new=RETRY_DELAY):
                try:
                    resolver.resolve(import_url=import_url)
                    if expected_failure:
                        err_msg = 'resolve should have been failed'
                        if partial_err_msg:
                            err_msg = \
                                '{0} with error message that contains: {1}'\
                                .format(err_msg, partial_err_msg)
                        raise AssertionError(err_msg)
                except DSLParsingLogicException, ex:
                    if not expected_failure:
                        raise ex
                    if partial_err_msg:
                        self.assertIn(partial_err_msg, str(ex)) 

def sendRequest(self, session, verb, url, postget=False):
        if (postget or verb == 'POST' or verb == 'PUT') and '??' in url:
            url, data = url.split(u'??', 1)
            try:
                data = data.encode('utf-8', 'replace')
            except UnicodeEncodeError:
                pass
            if postget:
                data = None
        else:
            data = None
        req = requests.Request(verb, url, data=data)
        try:
            prepped = session.prepare_request(req)
        except httplib.InvalidURL as e:
            err = True
            msg = unicode(e, 'utf-8')
            return err, msg
        if verb == 'POST' or verb == 'PUT':
            prepped.headers['Content-Type'] = self.taskKwargs['content-type']
        try:
            pu = prepped.url.decode('utf-8')
        except (AttributeError, UnicodeDecodeError):
            pu = u''
        try:
            pb = prepped.body.decode('utf-8')
        except (AttributeError, UnicodeDecodeError):
            pb = u''
        msg = u'Prepped URL: %s\nBody: %s' % (pu, pb)
        sys.exc_clear()
        try:
            resp = session.send(prepped, timeout=20)
            msg += u'\nStatus: %s' % resp.status_code
            resp.raise_for_status()
            err = False
            if resp.text == '':
                respmsg = u'No response received'
            else:
                respmsg = resp.text.decode('unicode_escape', 'ignore')
            msg += u'\nResponse for %s: %s' %(verb, respmsg)
            resp.close()
        except requests.ConnectionError:
            err = True
            msg = _(u'Requests Connection Error')
        except requests.HTTPError as e:
            err = True
            msg = u'%s: %s' %(_(u'Requests HTTPError'), str(e))
        except requests.URLRequired as e:
            err = True
            msg = u'%s: %s' %(_(u'Requests URLRequired Error'), str(e))
        except requests.Timeout as e:
            err = True
            msg = u'%s: %s' %(_(u'Requests Timeout Error'), str(e))
        except requests.RequestException as e:
            err = True
            msg = u'%s: %s' %(_(u'Generic Requests Error'), str(e))
        except urllib2.HTTPError, e:
            err = True
            msg = _(u'HTTPError = ') + unicode(e.code) 

def test_wrap_requests_invalid_url(self):
        mock_return = mock.Mock()
        mock_return.status_code = 200
        return_value = mock_return
        mock_func = mock.Mock()
        mock_func.__name__ = 'get'
        mock_func.return_value = return_value
        mock_func.side_effect = requests.URLRequired
        mock_tracer = MockTracer()

        patch = mock.patch(
            'opencensus.ext.requests.trace.execution_context.'
            'get_opencensus_tracer',
            return_value=mock_tracer)

        patch_thread = mock.patch(
            'opencensus.ext.requests.trace.execution_context.'
            'is_exporter',
            return_value=False)

        wrapped = trace.wrap_requests(mock_func)

        url = 'http://localhost:8080/test'

        with patch, patch_thread:
            with self.assertRaises(requests.URLRequired):
                wrapped(url)

        expected_attributes = {
            'component': 'HTTP',
            'http.host': 'localhost:8080',
            'http.method': 'GET',
            'http.path': '/test',
            'http.url': url,
        }
        expected_name = '/test'
        expected_status = status_module.Status(3, 'invalid URL')

        self.assertEqual(span_module.SpanKind.CLIENT,
                         mock_tracer.current_span.span_kind)
        self.assertEqual(expected_attributes,
                         mock_tracer.current_span.attributes)
        self.assertEqual(expected_name, mock_tracer.current_span.name)
        self.assertEqual(
            expected_status.__dict__,
            mock_tracer.current_span.status.__dict__
        )
        self.assertRaises(requests.URLRequired, mock_func) 

def test_wrap_session_request_invalid_url(self):
        wrapped = mock.Mock(return_value=mock.Mock(status_code=200))
        wrapped.side_effect = requests.URLRequired

        mock_tracer = MockTracer(
            propagator=mock.Mock(
                to_headers=lambda x: {'x-trace': 'some-value'}))

        patch = mock.patch(
            'opencensus.ext.requests.trace.execution_context.'
            'get_opencensus_tracer',
            return_value=mock_tracer)
        patch_thread = mock.patch(
            'opencensus.ext.requests.trace.execution_context.'
            'is_exporter',
            return_value=False)

        url = 'http://localhost:8080/test'
        request_method = 'POST'
        kwargs = {}

        with patch, patch_thread:
            with self.assertRaises(requests.URLRequired):
                trace.wrap_session_request(
                    wrapped, 'Session.request',
                    (request_method, url), kwargs
                )

        expected_attributes = {
            'component': 'HTTP',
            'http.host': 'localhost:8080',
            'http.method': 'POST',
            'http.path': '/test',
            'http.url': url,
        }
        expected_name = '/test'
        expected_status = status_module.Status(3, 'invalid URL')

        self.assertEqual(span_module.SpanKind.CLIENT,
                         mock_tracer.current_span.span_kind)
        self.assertEqual(expected_attributes,
                         mock_tracer.current_span.attributes)
        self.assertEqual(kwargs['headers']['x-trace'], 'some-value')
        self.assertEqual(expected_name, mock_tracer.current_span.name)
        self.assertEqual(
            expected_status.__dict__,
            mock_tracer.current_span.status.__dict__
        ) 

def test_add_threat_intel_item_errors(self, mocked_requests_post):
        # 1. Connect successfully
        sim_content = '<response><sessionKey>' + self.simSessionKey + '</sessionKey></response>'
        mocked_requests_post.return_value = self._generateResponse(sim_content, 200)
        splnk_utils = splunk_utils.SplunkUtils(host=self.fake_host,
                                               port=self.fake_port,
                                               username=self.fake_username,
                                               password=self.fake_password,
                                               verify=self.verify)
        # 2. Simulate wrong intel type
        try:
            splnk_utils.add_threat_intel_item("Fake type", {}, False)
        except splunk_utils.RequestError as e:
            print("Fake intel type causes exception as expected.")
            assert(True)

        # 3. Simulate RequestException
        threat_type = "ip_intel"
        mocked_requests_post.side_effect = requests.RequestException(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True

        # 4. Simulate ConnectionError
        mocked_requests_post.side_effect = requests.ConnectionError(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True

        # 5. Simulate HttpError
        mocked_requests_post.side_effect = requests.HTTPError(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True

        # 6. Simulate URLRequired
        mocked_requests_post.side_effect = requests.URLRequired(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True

        # 7. Simulate TooManyRedirects
        mocked_requests_post.side_effect = requests.TooManyRedirects(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True 

def update_notable(self, event_id, comment, status, cafile):
        """
        Update notable event
        :param event_id: event_id for notable event to be updated
        :param comment: comment to add to the notable event
        :param status: status of the notable event to change to
        :param cafile: Verify HTTPS cert or not
        :return:
        """

        headers = dict()
        headers["Authorization"] = "Splunk {}".format(self.session_key)

        args = dict()
        args["comment"] = comment
        args["status"] = status
        args["ruleUIDs"] = [event_id]

        ret = None
        url = self.base_url + "/services/notable_update"

        try:
            resp = requests.post(url,
                                 headers=headers,
                                 data=args,
                                 verify=cafile)

            #
            # We shall just return the response in json and let the post process
            # to make decision.
            #
            ret = {"status_code": resp.status_code,
                   "content": resp.json()}

        except requests.ConnectionError as e:
            raise RequestError(url, "Connection error. " + str(e))
        except requests.HTTPError as e:
            raise RequestError(url, "An HTTP error. " + str(e))
        except requests.URLRequired as e:
            raise RequestError(url, "An valid URL is required.")
        except requests.TooManyRedirects as e:
            raise RequestError(url, "Too many redirects")
        except requests.RequestException as e:
            raise RequestError(url, "Ambiguous exception when handling request. " + str(e))
        return ret 

def add_threat_intel_item(self, threat_type, threat_dict, cafile):
        """
        Add a new threat intel item to the ThreatIntelligence collections
        :param threat_type: ip_intel, file_intel, user_intel, http_intel, email_intel, service_intel
                         process_intel, registry_intel, or certificate_intel
        :param threat_dict:
        :param cafile:
        :return:
        """
        headers = dict()
        headers["Authorization"] = "Splunk {}".format(self.session_key)

        url = self.base_url + "/services/data/threat_intel/item/" + threat_type

        if threat_type not in self.SUPPORTED_THREAT_TYPE:
            raise RequestError(url, "{} is not supported")

        item = {"item": json.dumps(threat_dict)}

        try:
            resp = requests.post(url,
                                 headers=headers,
                                 data=item,
                                 verify=cafile)

            #
            # We shall just return the response in json and let the post process
            # to make decision.
            #
            ret = {"status_code": resp.status_code,
                   "content": resp.json()}

        except requests.ConnectionError as e:
            raise RequestError(url, "Connection error. " + str(e))
        except requests.HTTPError as e:
            raise RequestError(url, "An HTTP error. " + str(e))
        except requests.URLRequired as e:
            raise RequestError(url, "An valid URL is required.")
        except requests.TooManyRedirects as e:
            raise RequestError(url, "Too many redirects")
        except requests.RequestException as e:
            raise RequestError(url, "Ambiguous exception when handling request. " + str(e))
        return ret