Python requests.exceptions() Examples

def _set_response_ins_(self, pageurl):
        """
        Sets the response for the GET request of pageurl and stores it in self.resp

        :param pageurl: url for which we store the response.
        """
        try:
            s = requests.Session()
            a = requests.adapters.HTTPAdapter(max_retries=5)
            s.mount('http://', a)
            resp = s.get(pageurl, timeout=30)
            self.__resp_obj__ = resp
            resp.close()
        except requests.exceptions.Timeout:
            logging.error("\tVery Slow Internet Connection.")
        except requests.exceptions.ConnectionError:
            logging.error("\tNetwork Unavailable. Check your connection.")
        except requests.exceptions.MissingSchema:
            logging.error("\t503 Service Unavailable. Retrying download ... ") 

def entity_iter(ids, debug=False, attempts=5):
    for num, cur in enumerate(chunk(ids, page_size)):
        if debug:
            print('entity_iter: {}/{}'.format(num * page_size, len(ids)))
        ids = '|'.join(cur)
        for attempt in range(attempts):
            try:
                r = api_call({'action': 'wbgetentities', 'ids': ids})
                break
            except requests.exceptions.ChunkedEncodingError:
                if attempt == attempts - 1:
                    raise
                time.sleep(1)
        r.raise_for_status()
        json_data = r.json()
        if 'entities' not in json_data:
            mail.send_mail('error fetching wikidata entities', r.text)

        for qid, entity in json_data['entities'].items():
            yield qid, entity 

def http_put(self, path, data=None, **kwargs):
        """
        Make a PUT request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.put(url, data=data, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem putting data to " \
                      "the Kubernetes API server. URL: {}, " \
                      "data: {}".format(url, data)
            logger.error(message)
            raise KubeException(message) from err

        return response 

def upload_image(self, filename=None, filedata=None, **kwargs) -> str:
        if filedata is None:
            files = {"image": open(filename, 'rb')}
        else:
            files = {"image": filedata}

        try:
            r = requests.post(self.url, files=files, timeout=5)
        except requests.exceptions.Timeout:
            logger.error("Timeout uploading to VimCN")
            return None
        except:
            logger.exception("Unknown errror uploading to VimCN")
            return None
        if not r.ok:
            return None
        return r.text.strip() 

def checkout_branch(self, project: str, branch: str) -> None:
        """Checks out a new git branch. Only works in dev workspace.

        Args:
            project: Name of the Looker project to use.
            branch: Name of the Git branch to check out.
        """
        logger.debug(f"Setting Git branch to '{branch}'")
        url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"])
        body = {"name": branch}
        response = self.put(url=url, json=body, timeout=TIMEOUT_SEC)
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            raise LookerApiError(
                name="unable-to-checkout-branch",
                title="Couldn't checkout Git branch.",
                status=response.status_code,
                detail=(
                    f"Unable to checkout Git branch '{branch}'. "
                    "If you have uncommitted changes on the current branch, "
                    "please commit or revert them, then try again."
                ),
                response=response,
            ) 

def http_head(self, path, **kwargs):
        """
        Make a HEAD request to the k8s server.
        """
        try:

            url = urljoin(self.url, path)
            response = self.session.head(url, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem retrieving headers from " \
                "the Kubernetes API server. URL: {}".format(url)
            logger.error(message)
            raise KubeException(message) from err

        return response 

def http_post(self, path, data=None, json=None, **kwargs):
        """
        Make a POST request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.post(url, data=data, json=json, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem posting data to " \
                      "the Kubernetes API server. URL: {}, " \
                      "data: {}, json: {}".format(url, data, json)
            logger.error(message)
            raise KubeException(message) from err

        return response 

def get_lookml_models(self) -> List[JsonDict]:
        """Gets all models and explores from the LookmlModel endpoint.

        Returns:
            List[JsonDict]: JSON response containing LookML models and explores.

        """
        logger.debug(f"Getting all models and explores from {self.base_url}")
        url = utils.compose_url(self.api_url, path=["lookml_models"])
        response = self.get(url=url, timeout=TIMEOUT_SEC)
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            raise LookerApiError(
                name="unable-to-get-lookml",
                title="Couldn't retrieve models and explores.",
                status=response.status_code,
                detail="Unable to retrieve LookML details. Please try again.",
                response=response,
            )

        return response.json() 

def content_validation(self) -> JsonDict:
        logger.debug("Validating all content in Looker")
        url = utils.compose_url(self.api_url, path=["content_validation"])
        response = self.get(url=url, timeout=TIMEOUT_SEC)

        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            raise LookerApiError(
                name="unable-to-validate-content",
                title="Couldn't validate Looks and Dashboards.",
                status=response.status_code,
                detail=("Failed to run the content validator. Please try again."),
                response=response,
            )

        result = response.json()
        return result 

def all_folders(self, project: str) -> List[JsonDict]:
        logger.debug("Getting information about all folders")
        url = utils.compose_url(self.api_url, path=["folders"])
        response = self.get(url=url, timeout=TIMEOUT_SEC)

        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            raise LookerApiError(
                name="unable-to-get-folders",
                title="Couldn't obtain project folders.",
                status=response.status_code,
                detail=(f"Failed to get all folders for project '{project}'."),
                response=response,
            )

        result = response.json()
        return result 

def execute(self, method: str, url: str, payload: dict) -> dict:
        self.connection.create_jwt_token(url, method.upper(), json.dumps(payload))
        request_url = self.connection.url + url

        response = None
        try:
            response = requests.request(method, request_url,
                                        json=payload,
                                        headers=self.connection.header_dict)

            if response.status_code == 403:
                raise PluginException(preset=PluginException.Preset.API_KEY)
            if response.status_code >= 400:
                raise PluginException(preset=PluginException.Preset.UNKNOWN, data=response.json())
            if 200 <= response.status_code < 300:
                return komand.helper.clean(response.json())

            raise PluginException(preset=PluginException.Preset.UNKNOWN, data=response.text)
        except json.decoder.JSONDecodeError as e:
            self.logger.info(f"Invalid json: {e}")
            raise PluginException(preset=PluginException.Preset.INVALID_JSON, data=response.text)
        except requests.exceptions.HTTPError as e:
            self.logger.info(f"Call to Trend Micro Apex failed: {e}")
            raise PluginException(preset=PluginException.Preset.UNKNOWN, data=response.text) 

def get_cdas_url(starttime, endtime, vars, dataset, timeout=10):
    dataview = 'sp_phys'
    if vars is None:
        try:
            var_info = get_variables(dataset, timeout=timeout)
        except requests.exceptions.ReadTimeout:
            raise util.NoDataError(
                'Connection to CDAweb timed out when getting CDAS URL for '
                f'{dataset} data for interval {starttime} - {endtime}.')

        if not len(var_info):
            raise util.NoDataError(
                f'No {dataset} data available for {starttime} - {endtime}')

        vars = [v['Name'] for v in var_info['VariableDescription']]

    uri = '/'.join(['dataviews', dataview,
                    'datasets', dataset,
                    'data',
                    ','.join([starttime.strftime('%Y%m%dT%H%M%SZ'),
                              endtime.strftime('%Y%m%dT%H%M%SZ')]),
                    ','.join(vars)
                    ])
    url = '/'.join([CDAS_BASEURL, uri])
    return url 

def iter_lines(self):
        kwargs = {
            # OpenShift does not respond with any encoding value.
            # This causes requests module to guess it as ISO-8859-1.
            # Likely, the encoding is actually UTF-8, but we can't
            # guarantee it. Therefore, we take the approach of simply
            # passing through the encoded data with no effort to
            # attempt decoding it.
            'decode_unicode': False
        }
        if requests.__version__.startswith('2.6.'):
            kwargs['chunk_size'] = 1
        # if this fails for any reason other than ChunkedEncodingError
        # or IncompleteRead (either of which may happen when no bytes
        # are received), let someone else handle the exception
        try:
            for line in self.req.iter_lines(**kwargs):
                yield line
        except (requests.exceptions.ChunkedEncodingError,
                http_client.IncompleteRead):
            return 

def __call__(self, f):
        """
        A decorator function to retry a function (ie API call, web query) a
        number of times, with optional exceptions under which to retry.

        Returns results of a cleanup function if all retries fail.
        :return: decorator function.
        """
        @wraps(f)
        def wrapped_f(*args, **kwargs):
            for i in range(self.times):
                # Exponential backoff if required and limit to a max pause time
                pause = min(self.pause * self.retreat ** i, self.max_pause)
                try:
                    return f(*args, **kwargs)
                except self.exceptions:
                    if self.pause is not None:
                        time.sleep(pause)
                    else:
                        pass
            if self.cleanup is not None:
                return self.cleanup(*args, **kwargs)
        return wrapped_f 

def test_get_request_with_port(self, mock_requests, request_mock, mock_session):
        from requests.exceptions import MissingSchema

        with mock.patch(
            'airflow.hooks.base_hook.BaseHook.get_connection',
            side_effect=get_airflow_connection_with_port
        ):
            expected_url = 'http://test.com:1234/some/endpoint'
            for endpoint in ['some/endpoint', '/some/endpoint']:

                try:
                    self.get_hook.run(endpoint)
                except MissingSchema:
                    pass

                request_mock.assert_called_once_with(
                    mock.ANY,
                    expected_url,
                    headers=mock.ANY,
                    params=mock.ANY
                )

                request_mock.reset_mock() 

def test_hook_with_method_in_lowercase(self, mock_requests, request_mock):
        from requests.exceptions import MissingSchema, InvalidURL
        with mock.patch(
            'airflow.hooks.base_hook.BaseHook.get_connection',
            side_effect=get_airflow_connection_with_port
        ):
            data = "test params"
            try:
                self.get_lowercase_hook.run('v1/test', data=data)
            except (MissingSchema, InvalidURL):
                pass
            request_mock.assert_called_once_with(
                mock.ANY,
                mock.ANY,
                headers=mock.ANY,
                params=data
            ) 

def _must_post(self, api, data=None, json=None, timeout=10, **kwargs):
        if data is not None:
            kwargs['data'] = data
        elif json is not None:
            kwargs['json'] = json
        else:
            kwargs['data'] = {}
        kwargs['timeout'] = timeout

        try:
            r = requests.post(api, **kwargs)
            return r
        except requests.exceptions.Timeout:
            logger.error("Timeout requesting Gitter")
        except KeyboardInterrupt:
            raise
        except:
            logger.exception("Unknown error requesting Gitter")
        return None 

def test_retry_on_conn_error(self, mocked_session):

        retry_args = dict(
            wait=tenacity.wait_none(),
            stop=tenacity.stop_after_attempt(7),
            retry=tenacity.retry_if_exception_type(
                requests.exceptions.ConnectionError
            )
        )

        def send_and_raise(unused_request, **kwargs):
            raise requests.exceptions.ConnectionError

        mocked_session().send.side_effect = send_and_raise
        # The job failed for some reason
        with self.assertRaises(tenacity.RetryError):
            self.get_hook.run_with_advanced_retry(
                endpoint='v1/test',
                _retry_args=retry_args
            )
        self.assertEqual(
            self.get_hook._retry_obj.stop.max_attempt_number + 1,
            mocked_session.call_count
        ) 

def get_exception_code(exception_type):

    service_code = SERVICES['System']
    exception_code = EXCEPTIONS_MAP.get(exception_type.__name__, '0000')    # '0000' is default exception code

    # Exception inside a docker container
    if docker.errors.ContainerError.__name__ in EXCEPTIONS_MAP and \
       exception_code == EXCEPTIONS_MAP[docker.errors.ContainerError.__name__]:

        exception_codes = get_exception_codes_from_docker_trace()

        if len(exception_codes) > 0:
            # Take the first code in the list (may have more if multiple exceptions are raised)
            service_code = SERVICES['Docker']
            exception_code = exception_codes.pop()

    return exception_code, service_code 

def find_exception(module):
    # Exception classes in module
    exceptions = [ename for ename, eclass in inspect.getmembers(module, inspect.isclass)
                  if issubclass(eclass, BaseException)]

    # Exception classes in submodule

    try:
        submodules = inspect.getmembers(module, inspect.ismodule)
    except Exception:
        submodules = []

    for submodule_name, submodule in submodules:
        try:
            classes = inspect.getmembers(submodule, inspect.isclass)
        except Exception:
            classes = []

        exceptions += [ename for ename, eclass in classes
                       if issubclass(eclass, BaseException)]

    return set(exceptions) 

def api_patch(namespace, kind, name, entity_name, body):
    api_url = '/'.join([KUBE_API_URL, namespace, kind, name])
    while True:
        try:
            token = read_token()
            if token:
                r = requests.patch(api_url, data=body, verify=KUBE_CA_CERT,
                                   headers={'Content-Type': 'application/strategic-merge-patch+json',
                                            'Authorization': 'Bearer {0}'.format(token)})
                if r.status_code >= 300:
                    logger.warning('Unable to change %s: %s', entity_name, r.text)
                else:
                    break
            else:
                logger.warning('Unable to read Kubernetes authorization token')
        except requests.exceptions.RequestException as e:
            logger.warning('Exception when executing PATCH on %s: %s', api_url, e)
        time.sleep(1) 

def is_registry_running():
    """
    is docker registry running (at {docker0,lo}:5000)?
    """
    try:
        lo_response = requests.get(LOCALHOST_REGISTRY_HTTP)
    except requests.exceptions.ConnectionError:
        return False
    if not lo_response.ok:
        return False
    try:
        lo_response = requests.get(DOCKER0_REGISTRY_HTTP)  # leap of faith
    except requests.exceptions.ConnectionError:
        return False
    if not lo_response.ok:
        return False
    return True 

def _getJsonWithRetry(self, url):
        """Private method to wrap getting json data with retries. Only gets single page, nothing fancy. See getJson
        :param url: url to get
        :return: json data returned from server
        """
        retry_count = 0
        req = None
        while retry_count < self._max_retries:
            try:
                #val = 'Token ' + self._token
                headers = {'Authorization': 'Token ' + self._token} 
                req = requests.get(url, timeout=self._req_timeout, headers=headers)
                if req.status_code == requests.codes.ok:
                    break
                logging.warning('Failed to get %s, status %d, retry %d' % (url, req.status_code, retry_count))
            except requests.exceptions.RequestException as e:
                logging.warning('Failed to get request for %s, RequestException: %s' % (url, e))
                pass        # Just pass it, we will include it as a retry ahead
            finally:
                retry_count += 1

        if retry_count >= self._max_retries:
            logging.error("Exceeded max connection retries for %s" % url)
            if req is not None:
                logging.error("Request failure reason: %s" % req.reason)
                raise ConnectionError(req.reason)
            else:
                logging.error("No request, no reason!")
                raise ConnectionError

        return req.json()

    # TODO check documentation 

def get_url_data(url, max_tries=10):
        remaining_tries = int(max_tries)
        while remaining_tries > 0:
            try:
                return requests.get(url)
            except requests.exceptions:
                sleep(60)
            remaining_tries = remaining_tries - 1
        raise ResponseTimeout() 

def webfacade(testSet, configinstance, webcaller, config):
    """
    Perform the web request for a check.
    (Called upon by check())

    :param testSet: Name of testset to pull values
    :param configinstance: config class object
    :return requests output:
    """

    # config getters
    tmout = configinstance.get_request_timeout(testSet)
    vfyssl = configinstance.get_verify_ssl(testSet)
    testset = configinstance.get_test_set(testSet)

    # dispatch request
    out = webcaller.run(config,
                        testset['data']['uri'],
                        verify=vfyssl,
                        expected_http_status=str(
                            testset['data']['ok_http_code']),
                        identity_provider=testset[
                            'data']['identity_provider'],
                        timeout=tmout)

    if out == False:  # webcaller.run has requests.exceptions
        logging.error("Spawn request failed, skipping."
                      " url={0}".format(testset['data']['uri']))
        return False
    else:  # it works!
        logging.debug("Spawn request OK (at webfacade)")
        return out 

def http_delete(self, path, **kwargs):
        """
        Make a DELETE request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.delete(url, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem deleting data from " \
                      "the Kubernetes API server. URL: {}".format(url)
            logger.error(message)
            raise KubeException(message) from err

        return response 

def retry_on_exception(exceptions):
    """Decorator function to attempt retry based on passed exceptions"""
    def real_decorator(func):
        """Actual decorator to retry on exceptions"""
        @backoff.on_exception(backoff.expo,
                              exceptions, # This is a tuple with exceptions
                              max_tries=OutputDispatcher.MAX_RETRY_ATTEMPTS,
                              jitter=backoff.full_jitter,
                              on_backoff=backoff_handler(),
                              on_success=success_handler(),
                              on_giveup=giveup_handler())
        def wrapper(*args, **kwargs):
            return func(*args, **kwargs)
        return wrapper
    return real_decorator 

def _catch_exceptions(cls):
        """Classmethod that returns a tuple of the exceptions to catch"""
        default_exceptions = (OutputRequestFailure, ReqTimeout)
        exceptions = cls._get_exceptions_to_catch()
        if not exceptions:
            return default_exceptions

        if isinstance(exceptions, tuple):
            return default_exceptions + exceptions

        return default_exceptions + (exceptions,) 

def _get_exceptions_to_catch(cls):
        """Classmethod that returns a tuple of the exceptions to catch""" 

def __init__(self, times, exceptions = (IndexError), pause = 1, retreat = 1,
                 max_pause = None, cleanup = None):
        """Initiliase all input params"""
        self.times = times
        self.exceptions = exceptions
        self.pause = pause
        self.retreat = retreat
        self.max_pause = max_pause or (pause * retreat ** times)
        self.cleanup = cleanup