Python requests.URLRequired() Examples

The following are 14 code examples for showing how to use requests.URLRequired(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module requests , or try the search function .

Example 1
Project: LabelImgTool   Author: lzx1413   File: labelImg.py    License: MIT License 6 votes vote down vote up
def setRemoteUrl(self):
        setRemoteUrldialog = remoteDialog.SetRemoteDialog(parent=self)
        if setRemoteUrldialog.exec_():
            self.database_url = 'http://' + setRemoteUrldialog.get_remote_url()
            self.remoteMode = setRemoteUrldialog.is_in_remote_mode()
            self.dowload_thread_num = setRemoteUrldialog.get_thread_num()
            self.server_image_list = setRemoteUrldialog.get_server_image_list()
        setRemoteUrldialog.destroy()
        print self.database_url
        if not os.path.exists(self.loadFilePath):
            os.makedirs(self.loadFilePath)
        if self.database_url:
            try:
                image_file = requests.get(
                    self.database_url + self.server_image_list)
            except requests.URLRequired as e:
                logging.error('can not get the server image list')
                return

            self.image_list = image_file.content.split('\n')[0:-1]
            self.server_image_num = len(self.image_list)
            if self.image_list:
                self.connect_remote_db = True
                self.toggleRemoteMode() 
Example 2
Project: crawler_examples   Author: WiseDoge   File: utils.py    License: Apache License 2.0 5 votes vote down vote up
def get_links(session, url, proxy = None):
    """
    Receive a url, and return a BeautifulSoup object
    """
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.',
    }
    try:
        html = session.get(url, proxies = proxy, headers=headers)
        soup = BeautifulSoup(html.text, "lxml")
        return soup
    except URLRequired:
        return None
    except HTTPError:
        print("HTTPError") 
Example 3
Project: manila   Author: openstack   File: api.py    License: Apache License 2.0 5 votes vote down vote up
def invoke_elem(self, na_element, enable_tunneling=False):
        """Invoke the API on the server."""
        if na_element and not isinstance(na_element, NaElement):
            ValueError('NaElement must be supplied to invoke API')
        request_element = self._create_request(na_element, enable_tunneling)
        request_d = request_element.to_string()

        api_name = na_element.get_name()
        api_name_matches_regex = (re.match(self._api_trace_pattern, api_name)
                                  is not None)

        if self._trace and api_name_matches_regex:
            LOG.debug("Request: %s", request_element.to_string(pretty=True))

        if (not hasattr(self, '_session') or not self._session
                or self._refresh_conn):
            self._build_session()
        try:
            if hasattr(self, '_timeout'):
                response = self._session.post(
                    self._get_url(), data=request_d, timeout=self._timeout)
            else:
                response = self._session.post(
                    self._get_url(), data=request_d)
        except requests.HTTPError as e:
            raise NaApiError(e.errno, e.strerror)
        except requests.URLRequired as e:
            raise exception.StorageCommunicationException(six.text_type(e))
        except Exception as e:
            raise NaApiError(message=e)

        response_xml = response.text
        response_element = self._get_result(
            bytes(bytearray(response_xml, encoding='utf-8')))

        if self._trace and api_name_matches_regex:
            LOG.debug("Response: %s", response_element.to_string(pretty=True))

        return response_element 
Example 4
Project: manila   Author: openstack   File: test_api.py    License: Apache License 2.0 5 votes vote down vote up
def test_invoke_elem_urlerror(self):
        """Tests handling of URLError"""
        na_element = fake.FAKE_NA_ELEMENT
        self.mock_object(self.root, '_create_request', mock.Mock(
            return_value=fake.FAKE_NA_ELEMENT))
        self.mock_object(api, 'LOG')
        self.root._session = fake.FAKE_HTTP_SESSION
        self.mock_object(self.root, '_build_session')
        self.mock_object(self.root._session, 'post', mock.Mock(
            side_effect=requests.URLRequired()))

        self.assertRaises(exception.StorageCommunicationException,
                          self.root.invoke_elem,
                          na_element) 
Example 5
Project: opentelemetry-python   Author: open-telemetry   File: __init__.py    License: Apache License 2.0 5 votes vote down vote up
def _exception_to_canonical_code(exc: Exception) -> StatusCanonicalCode:
    if isinstance(
        exc,
        (InvalidURL, InvalidSchema, MissingSchema, URLRequired, ValueError),
    ):
        return StatusCanonicalCode.INVALID_ARGUMENT
    if isinstance(exc, Timeout):
        return StatusCanonicalCode.DEADLINE_EXCEEDED
    return StatusCanonicalCode.UNKNOWN 
Example 6
Project: resilient-community-apps   Author: ibmresilient   File: test_splunk_utils.py    License: MIT License 5 votes vote down vote up
def test_update_notable_invalid_url(self, mocked_requests_post):
        """ Test update notable with wrong URL or connection issue"""
        print("Test update_notable returns HTTPError\n")
        try:
            sim_status = 1

            sim_content = '<response><sessionKey>' + self.simSessionKey + '</sessionKey></response>'
            mocked_requests_post.return_value = self._generateResponse(sim_content, 200)
            splnk_utils = splunk_utils.SplunkUtils(host=self.fake_host,
                                                   port=self.fake_port,
                                                   username=self.fake_username,
                                                   password=self.fake_password,
                                                   verify=self.verify)

            mocked_requests_post.side_effect = requests.URLRequired(Mock(status=404), "Ambiguous excetpion.")

            ret = splnk_utils.update_notable(event_id=self.simEventId,
                                             comment=self.simComment,
                                             status=sim_status,
                                             cafile=self.verify)
            #
            # request post throws exception
            #
            assert False
        except splunk_utils.RequestError as e:
            assert True 
Example 7
Project: fbchat   Author: carpedm20   File: _exception.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def handle_requests_error(e):
    if isinstance(e, requests.ConnectionError):
        raise HTTPError("Connection error") from e
    if isinstance(e, requests.HTTPError):
        pass  # Raised when using .raise_for_status, so should never happen
    if isinstance(e, requests.URLRequired):
        pass  # Should never happen, we always prove valid URLs
    if isinstance(e, requests.TooManyRedirects):
        pass  # TODO: Consider using allow_redirects=False to prevent this
    if isinstance(e, requests.Timeout):
        pass  # Should never happen, we don't set timeouts

    raise HTTPError("Requests error") from e 
Example 8
Project: cloudify-dsl-parser   Author: cloudify-cosmo   File: test_deafult_import_resolver.py    License: Apache License 2.0 4 votes vote down vote up
def _test_default_resolver(self, import_url, rules,
                               expected_urls_to_resolve=[],
                               expected_failure=False,
                               partial_err_msg=None):

        urls_to_resolve = []
        number_of_attempts = []

        class mock_requests_get(object):

            def __init__(self, url, timeout):
                self.status_code = 200
                self.text = 200
                number_of_attempts.append(1)
                if url not in urls_to_resolve:
                    urls_to_resolve.append(url)
                if url in [ORIGINAL_V1_URL, ORIGINAL_V2_URL, INVALID_V1_URL]:
                    raise requests.URLRequired(
                        'invalid url: {0}'.format(url))
                elif url == ILLEGAL_URL:
                    raise requests.URLRequired(
                        'unknown url type: {0}'.format(
                            url))
                elif url in [VALID_V1_URL, VALID_V2_URL]:
                    return None
                elif url == TIMEOUT_URL:
                    raise requests.ConnectionError(
                        'Timeout while trying to import')
                elif url == BAD_RESPONSE_CODE_URL:
                    self.status_code = 404
                    self.text = 404
                elif url == RETRY_URL:
                    if len(number_of_attempts) < MAX_NUMBER_RETRIES:
                        raise requests.ConnectionError(
                            'Timeout while trying to import')
                    else:
                        return None

        resolver = DefaultImportResolver(rules=rules)
        with mock.patch('requests.get', new=mock_requests_get,
                        create=True):
            with mock.patch(
                    'dsl_parser.import_resolver.abstract_import_resolver.'
                    'DEFAULT_RETRY_DELAY', new=RETRY_DELAY):
                try:
                    resolver.resolve(import_url=import_url)
                    if expected_failure:
                        err_msg = 'resolve should have been failed'
                        if partial_err_msg:
                            err_msg = \
                                '{0} with error message that contains: {1}'\
                                .format(err_msg, partial_err_msg)
                        raise AssertionError(err_msg)
                except DSLParsingLogicException, ex:
                    if not expected_failure:
                        raise ex
                    if partial_err_msg:
                        self.assertIn(partial_err_msg, str(ex)) 
Example 9
Project: script.service.kodi.callbacks   Author: KenV99   File: taskHttp.py    License: GNU General Public License v3.0 4 votes vote down vote up
def sendRequest(self, session, verb, url, postget=False):
        if (postget or verb == 'POST' or verb == 'PUT') and '??' in url:
            url, data = url.split(u'??', 1)
            try:
                data = data.encode('utf-8', 'replace')
            except UnicodeEncodeError:
                pass
            if postget:
                data = None
        else:
            data = None
        req = requests.Request(verb, url, data=data)
        try:
            prepped = session.prepare_request(req)
        except httplib.InvalidURL as e:
            err = True
            msg = unicode(e, 'utf-8')
            return err, msg
        if verb == 'POST' or verb == 'PUT':
            prepped.headers['Content-Type'] = self.taskKwargs['content-type']
        try:
            pu = prepped.url.decode('utf-8')
        except (AttributeError, UnicodeDecodeError):
            pu = u''
        try:
            pb = prepped.body.decode('utf-8')
        except (AttributeError, UnicodeDecodeError):
            pb = u''
        msg = u'Prepped URL: %s\nBody: %s' % (pu, pb)
        sys.exc_clear()
        try:
            resp = session.send(prepped, timeout=20)
            msg += u'\nStatus: %s' % resp.status_code
            resp.raise_for_status()
            err = False
            if resp.text == '':
                respmsg = u'No response received'
            else:
                respmsg = resp.text.decode('unicode_escape', 'ignore')
            msg += u'\nResponse for %s: %s' %(verb, respmsg)
            resp.close()
        except requests.ConnectionError:
            err = True
            msg = _(u'Requests Connection Error')
        except requests.HTTPError as e:
            err = True
            msg = u'%s: %s' %(_(u'Requests HTTPError'), str(e))
        except requests.URLRequired as e:
            err = True
            msg = u'%s: %s' %(_(u'Requests URLRequired Error'), str(e))
        except requests.Timeout as e:
            err = True
            msg = u'%s: %s' %(_(u'Requests Timeout Error'), str(e))
        except requests.RequestException as e:
            err = True
            msg = u'%s: %s' %(_(u'Generic Requests Error'), str(e))
        except urllib2.HTTPError, e:
            err = True
            msg = _(u'HTTPError = ') + unicode(e.code) 
Example 10
Project: opencensus-python   Author: census-instrumentation   File: test_requests_trace.py    License: Apache License 2.0 4 votes vote down vote up
def test_wrap_requests_invalid_url(self):
        mock_return = mock.Mock()
        mock_return.status_code = 200
        return_value = mock_return
        mock_func = mock.Mock()
        mock_func.__name__ = 'get'
        mock_func.return_value = return_value
        mock_func.side_effect = requests.URLRequired
        mock_tracer = MockTracer()

        patch = mock.patch(
            'opencensus.ext.requests.trace.execution_context.'
            'get_opencensus_tracer',
            return_value=mock_tracer)

        patch_thread = mock.patch(
            'opencensus.ext.requests.trace.execution_context.'
            'is_exporter',
            return_value=False)

        wrapped = trace.wrap_requests(mock_func)

        url = 'http://localhost:8080/test'

        with patch, patch_thread:
            with self.assertRaises(requests.URLRequired):
                wrapped(url)

        expected_attributes = {
            'component': 'HTTP',
            'http.host': 'localhost:8080',
            'http.method': 'GET',
            'http.path': '/test',
            'http.url': url,
        }
        expected_name = '/test'
        expected_status = status_module.Status(3, 'invalid URL')

        self.assertEqual(span_module.SpanKind.CLIENT,
                         mock_tracer.current_span.span_kind)
        self.assertEqual(expected_attributes,
                         mock_tracer.current_span.attributes)
        self.assertEqual(expected_name, mock_tracer.current_span.name)
        self.assertEqual(
            expected_status.__dict__,
            mock_tracer.current_span.status.__dict__
        )
        self.assertRaises(requests.URLRequired, mock_func) 
Example 11
Project: opencensus-python   Author: census-instrumentation   File: test_requests_trace.py    License: Apache License 2.0 4 votes vote down vote up
def test_wrap_session_request_invalid_url(self):
        wrapped = mock.Mock(return_value=mock.Mock(status_code=200))
        wrapped.side_effect = requests.URLRequired

        mock_tracer = MockTracer(
            propagator=mock.Mock(
                to_headers=lambda x: {'x-trace': 'some-value'}))

        patch = mock.patch(
            'opencensus.ext.requests.trace.execution_context.'
            'get_opencensus_tracer',
            return_value=mock_tracer)
        patch_thread = mock.patch(
            'opencensus.ext.requests.trace.execution_context.'
            'is_exporter',
            return_value=False)

        url = 'http://localhost:8080/test'
        request_method = 'POST'
        kwargs = {}

        with patch, patch_thread:
            with self.assertRaises(requests.URLRequired):
                trace.wrap_session_request(
                    wrapped, 'Session.request',
                    (request_method, url), kwargs
                )

        expected_attributes = {
            'component': 'HTTP',
            'http.host': 'localhost:8080',
            'http.method': 'POST',
            'http.path': '/test',
            'http.url': url,
        }
        expected_name = '/test'
        expected_status = status_module.Status(3, 'invalid URL')

        self.assertEqual(span_module.SpanKind.CLIENT,
                         mock_tracer.current_span.span_kind)
        self.assertEqual(expected_attributes,
                         mock_tracer.current_span.attributes)
        self.assertEqual(kwargs['headers']['x-trace'], 'some-value')
        self.assertEqual(expected_name, mock_tracer.current_span.name)
        self.assertEqual(
            expected_status.__dict__,
            mock_tracer.current_span.status.__dict__
        ) 
Example 12
Project: resilient-community-apps   Author: ibmresilient   File: test_splunk_utils.py    License: MIT License 4 votes vote down vote up
def test_add_threat_intel_item_errors(self, mocked_requests_post):
        # 1. Connect successfully
        sim_content = '<response><sessionKey>' + self.simSessionKey + '</sessionKey></response>'
        mocked_requests_post.return_value = self._generateResponse(sim_content, 200)
        splnk_utils = splunk_utils.SplunkUtils(host=self.fake_host,
                                               port=self.fake_port,
                                               username=self.fake_username,
                                               password=self.fake_password,
                                               verify=self.verify)
        # 2. Simulate wrong intel type
        try:
            splnk_utils.add_threat_intel_item("Fake type", {}, False)
        except splunk_utils.RequestError as e:
            print("Fake intel type causes exception as expected.")
            assert(True)

        # 3. Simulate RequestException
        threat_type = "ip_intel"
        mocked_requests_post.side_effect = requests.RequestException(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True

        # 4. Simulate ConnectionError
        mocked_requests_post.side_effect = requests.ConnectionError(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True

        # 5. Simulate HttpError
        mocked_requests_post.side_effect = requests.HTTPError(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True

        # 6. Simulate URLRequired
        mocked_requests_post.side_effect = requests.URLRequired(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True

        # 7. Simulate TooManyRedirects
        mocked_requests_post.side_effect = requests.TooManyRedirects(Mock(status=404), "Ambiguous excetpion.")
        try:
            splnk_utils.add_threat_intel_item(threat_type, {}, False)
        except splunk_utils.RequestError:
            assert True 
Example 13
Project: resilient-community-apps   Author: ibmresilient   File: splunk_utils.py    License: MIT License 4 votes vote down vote up
def update_notable(self, event_id, comment, status, cafile):
        """
        Update notable event
        :param event_id: event_id for notable event to be updated
        :param comment: comment to add to the notable event
        :param status: status of the notable event to change to
        :param cafile: Verify HTTPS cert or not
        :return:
        """

        headers = dict()
        headers["Authorization"] = "Splunk {}".format(self.session_key)

        args = dict()
        args["comment"] = comment
        args["status"] = status
        args["ruleUIDs"] = [event_id]

        ret = None
        url = self.base_url + "/services/notable_update"

        try:
            resp = requests.post(url,
                                 headers=headers,
                                 data=args,
                                 verify=cafile)

            #
            # We shall just return the response in json and let the post process
            # to make decision.
            #
            ret = {"status_code": resp.status_code,
                   "content": resp.json()}

        except requests.ConnectionError as e:
            raise RequestError(url, "Connection error. " + str(e))
        except requests.HTTPError as e:
            raise RequestError(url, "An HTTP error. " + str(e))
        except requests.URLRequired as e:
            raise RequestError(url, "An valid URL is required.")
        except requests.TooManyRedirects as e:
            raise RequestError(url, "Too many redirects")
        except requests.RequestException as e:
            raise RequestError(url, "Ambiguous exception when handling request. " + str(e))
        return ret 
Example 14
Project: resilient-community-apps   Author: ibmresilient   File: splunk_utils.py    License: MIT License 4 votes vote down vote up
def add_threat_intel_item(self, threat_type, threat_dict, cafile):
        """
        Add a new threat intel item to the ThreatIntelligence collections
        :param threat_type: ip_intel, file_intel, user_intel, http_intel, email_intel, service_intel
                         process_intel, registry_intel, or certificate_intel
        :param threat_dict:
        :param cafile:
        :return:
        """
        headers = dict()
        headers["Authorization"] = "Splunk {}".format(self.session_key)

        url = self.base_url + "/services/data/threat_intel/item/" + threat_type

        if threat_type not in self.SUPPORTED_THREAT_TYPE:
            raise RequestError(url, "{} is not supported")

        item = {"item": json.dumps(threat_dict)}

        try:
            resp = requests.post(url,
                                 headers=headers,
                                 data=item,
                                 verify=cafile)

            #
            # We shall just return the response in json and let the post process
            # to make decision.
            #
            ret = {"status_code": resp.status_code,
                   "content": resp.json()}

        except requests.ConnectionError as e:
            raise RequestError(url, "Connection error. " + str(e))
        except requests.HTTPError as e:
            raise RequestError(url, "An HTTP error. " + str(e))
        except requests.URLRequired as e:
            raise RequestError(url, "An valid URL is required.")
        except requests.TooManyRedirects as e:
            raise RequestError(url, "Too many redirects")
        except requests.RequestException as e:
            raise RequestError(url, "Ambiguous exception when handling request. " + str(e))
        return ret