Python requests.exceptions() Examples

def _set_response_ins_(self, pageurl):
        """
        Sets the response for the GET request of pageurl and stores it in self.resp

        :param pageurl: url for which we store the response.
        """
        try:
            s = requests.Session()
            a = requests.adapters.HTTPAdapter(max_retries=5)
            s.mount('http://', a)
            resp = s.get(pageurl, timeout=30)
            self.__resp_obj__ = resp
            resp.close()
        except requests.exceptions.Timeout:
            logging.error("\tVery Slow Internet Connection.")
        except requests.exceptions.ConnectionError:
            logging.error("\tNetwork Unavailable. Check your connection.")
        except requests.exceptions.MissingSchema:
            logging.error("\t503 Service Unavailable. Retrying download ... ") 

def _must_post(self, api, data=None, json=None, timeout=10, **kwargs):
        if data is not None:
            kwargs['data'] = data
        elif json is not None:
            kwargs['json'] = json
        else:
            kwargs['data'] = {}
        kwargs['timeout'] = timeout

        try:
            r = requests.post(api, **kwargs)
            return r
        except requests.exceptions.Timeout:
            logger.error("Timeout requesting Gitter")
        except KeyboardInterrupt:
            raise
        except:
            logger.exception("Unknown error requesting Gitter")
        return None 

def upload_image(self, filename=None, filedata=None, **kwargs) -> str:
        if filedata is None:
            files = {"image": open(filename, 'rb')}
        else:
            files = {"image": filedata}

        try:
            r = requests.post(self.url, files=files, timeout=5)
        except requests.exceptions.Timeout:
            logger.error("Timeout uploading to VimCN")
            return None
        except:
            logger.exception("Unknown errror uploading to VimCN")
            return None
        if not r.ok:
            return None
        return r.text.strip() 

def http_head(self, path, **kwargs):
        """
        Make a HEAD request to the k8s server.
        """
        try:

            url = urljoin(self.url, path)
            response = self.session.head(url, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem retrieving headers from " \
                "the Kubernetes API server. URL: {}".format(url)
            logger.error(message)
            raise KubeException(message) from err

        return response 

def http_post(self, path, data=None, json=None, **kwargs):
        """
        Make a POST request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.post(url, data=data, json=json, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem posting data to " \
                      "the Kubernetes API server. URL: {}, " \
                      "data: {}, json: {}".format(url, data, json)
            logger.error(message)
            raise KubeException(message) from err

        return response 

def http_put(self, path, data=None, **kwargs):
        """
        Make a PUT request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.put(url, data=data, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem putting data to " \
                      "the Kubernetes API server. URL: {}, " \
                      "data: {}".format(url, data)
            logger.error(message)
            raise KubeException(message) from err

        return response 

def checkout_branch(self, project: str, branch: str) -> None:
        """Checks out a new git branch. Only works in dev workspace.

        Args:
            project: Name of the Looker project to use.
            branch: Name of the Git branch to check out.
        """
        logger.debug(f"Setting Git branch to '{branch}'")
        url = utils.compose_url(self.api_url, path=["projects", project, "git_branch"])
        body = {"name": branch}
        response = self.put(url=url, json=body, timeout=TIMEOUT_SEC)
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            raise LookerApiError(
                name="unable-to-checkout-branch",
                title="Couldn't checkout Git branch.",
                status=response.status_code,
                detail=(
                    f"Unable to checkout Git branch '{branch}'. "
                    "If you have uncommitted changes on the current branch, "
                    "please commit or revert them, then try again."
                ),
                response=response,
            ) 

def get_lookml_models(self) -> List[JsonDict]:
        """Gets all models and explores from the LookmlModel endpoint.

        Returns:
            List[JsonDict]: JSON response containing LookML models and explores.

        """
        logger.debug(f"Getting all models and explores from {self.base_url}")
        url = utils.compose_url(self.api_url, path=["lookml_models"])
        response = self.get(url=url, timeout=TIMEOUT_SEC)
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            raise LookerApiError(
                name="unable-to-get-lookml",
                title="Couldn't retrieve models and explores.",
                status=response.status_code,
                detail="Unable to retrieve LookML details. Please try again.",
                response=response,
            )

        return response.json() 

def content_validation(self) -> JsonDict:
        logger.debug("Validating all content in Looker")
        url = utils.compose_url(self.api_url, path=["content_validation"])
        response = self.get(url=url, timeout=TIMEOUT_SEC)

        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            raise LookerApiError(
                name="unable-to-validate-content",
                title="Couldn't validate Looks and Dashboards.",
                status=response.status_code,
                detail=("Failed to run the content validator. Please try again."),
                response=response,
            )

        result = response.json()
        return result 

def all_folders(self, project: str) -> List[JsonDict]:
        logger.debug("Getting information about all folders")
        url = utils.compose_url(self.api_url, path=["folders"])
        response = self.get(url=url, timeout=TIMEOUT_SEC)

        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            raise LookerApiError(
                name="unable-to-get-folders",
                title="Couldn't obtain project folders.",
                status=response.status_code,
                detail=(f"Failed to get all folders for project '{project}'."),
                response=response,
            )

        result = response.json()
        return result 

def execute(self, method: str, url: str, payload: dict) -> dict:
        self.connection.create_jwt_token(url, method.upper(), json.dumps(payload))
        request_url = self.connection.url + url

        response = None
        try:
            response = requests.request(method, request_url,
                                        json=payload,
                                        headers=self.connection.header_dict)

            if response.status_code == 403:
                raise PluginException(preset=PluginException.Preset.API_KEY)
            if response.status_code >= 400:
                raise PluginException(preset=PluginException.Preset.UNKNOWN, data=response.json())
            if 200 <= response.status_code < 300:
                return komand.helper.clean(response.json())

            raise PluginException(preset=PluginException.Preset.UNKNOWN, data=response.text)
        except json.decoder.JSONDecodeError as e:
            self.logger.info(f"Invalid json: {e}")
            raise PluginException(preset=PluginException.Preset.INVALID_JSON, data=response.text)
        except requests.exceptions.HTTPError as e:
            self.logger.info(f"Call to Trend Micro Apex failed: {e}")
            raise PluginException(preset=PluginException.Preset.UNKNOWN, data=response.text) 

def is_registry_running():
    """
    is docker registry running (at {docker0,lo}:5000)?
    """
    try:
        lo_response = requests.get(LOCALHOST_REGISTRY_HTTP)
    except requests.exceptions.ConnectionError:
        return False
    if not lo_response.ok:
        return False
    try:
        lo_response = requests.get(DOCKER0_REGISTRY_HTTP)  # leap of faith
    except requests.exceptions.ConnectionError:
        return False
    if not lo_response.ok:
        return False
    return True 

def get_cdas_url(starttime, endtime, vars, dataset, timeout=10):
    dataview = 'sp_phys'
    if vars is None:
        try:
            var_info = get_variables(dataset, timeout=timeout)
        except requests.exceptions.ReadTimeout:
            raise util.NoDataError(
                'Connection to CDAweb timed out when getting CDAS URL for '
                f'{dataset} data for interval {starttime} - {endtime}.')

        if not len(var_info):
            raise util.NoDataError(
                f'No {dataset} data available for {starttime} - {endtime}')

        vars = [v['Name'] for v in var_info['VariableDescription']]

    uri = '/'.join(['dataviews', dataview,
                    'datasets', dataset,
                    'data',
                    ','.join([starttime.strftime('%Y%m%dT%H%M%SZ'),
                              endtime.strftime('%Y%m%dT%H%M%SZ')]),
                    ','.join(vars)
                    ])
    url = '/'.join([CDAS_BASEURL, uri])
    return url 

def iter_lines(self):
        kwargs = {
            # OpenShift does not respond with any encoding value.
            # This causes requests module to guess it as ISO-8859-1.
            # Likely, the encoding is actually UTF-8, but we can't
            # guarantee it. Therefore, we take the approach of simply
            # passing through the encoded data with no effort to
            # attempt decoding it.
            'decode_unicode': False
        }
        if requests.__version__.startswith('2.6.'):
            kwargs['chunk_size'] = 1
        # if this fails for any reason other than ChunkedEncodingError
        # or IncompleteRead (either of which may happen when no bytes
        # are received), let someone else handle the exception
        try:
            for line in self.req.iter_lines(**kwargs):
                yield line
        except (requests.exceptions.ChunkedEncodingError,
                http_client.IncompleteRead):
            return 

def __call__(self, f):
        """
        A decorator function to retry a function (ie API call, web query) a
        number of times, with optional exceptions under which to retry.

        Returns results of a cleanup function if all retries fail.
        :return: decorator function.
        """
        @wraps(f)
        def wrapped_f(*args, **kwargs):
            for i in range(self.times):
                # Exponential backoff if required and limit to a max pause time
                pause = min(self.pause * self.retreat ** i, self.max_pause)
                try:
                    return f(*args, **kwargs)
                except self.exceptions:
                    if self.pause is not None:
                        time.sleep(pause)
                    else:
                        pass
            if self.cleanup is not None:
                return self.cleanup(*args, **kwargs)
        return wrapped_f 

def test_get_request_with_port(self, mock_requests, request_mock, mock_session):
        from requests.exceptions import MissingSchema

        with mock.patch(
            'airflow.hooks.base_hook.BaseHook.get_connection',
            side_effect=get_airflow_connection_with_port
        ):
            expected_url = 'http://test.com:1234/some/endpoint'
            for endpoint in ['some/endpoint', '/some/endpoint']:

                try:
                    self.get_hook.run(endpoint)
                except MissingSchema:
                    pass

                request_mock.assert_called_once_with(
                    mock.ANY,
                    expected_url,
                    headers=mock.ANY,
                    params=mock.ANY
                )

                request_mock.reset_mock() 

def test_hook_with_method_in_lowercase(self, mock_requests, request_mock):
        from requests.exceptions import MissingSchema, InvalidURL
        with mock.patch(
            'airflow.hooks.base_hook.BaseHook.get_connection',
            side_effect=get_airflow_connection_with_port
        ):
            data = "test params"
            try:
                self.get_lowercase_hook.run('v1/test', data=data)
            except (MissingSchema, InvalidURL):
                pass
            request_mock.assert_called_once_with(
                mock.ANY,
                mock.ANY,
                headers=mock.ANY,
                params=data
            ) 

def test_retry_on_conn_error(self, mocked_session):

        retry_args = dict(
            wait=tenacity.wait_none(),
            stop=tenacity.stop_after_attempt(7),
            retry=tenacity.retry_if_exception_type(
                requests.exceptions.ConnectionError
            )
        )

        def send_and_raise(unused_request, **kwargs):
            raise requests.exceptions.ConnectionError

        mocked_session().send.side_effect = send_and_raise
        # The job failed for some reason
        with self.assertRaises(tenacity.RetryError):
            self.get_hook.run_with_advanced_retry(
                endpoint='v1/test',
                _retry_args=retry_args
            )
        self.assertEqual(
            self.get_hook._retry_obj.stop.max_attempt_number + 1,
            mocked_session.call_count
        ) 

def get_exception_code(exception_type):

    service_code = SERVICES['System']
    exception_code = EXCEPTIONS_MAP.get(exception_type.__name__, '0000')    # '0000' is default exception code

    # Exception inside a docker container
    if docker.errors.ContainerError.__name__ in EXCEPTIONS_MAP and \
       exception_code == EXCEPTIONS_MAP[docker.errors.ContainerError.__name__]:

        exception_codes = get_exception_codes_from_docker_trace()

        if len(exception_codes) > 0:
            # Take the first code in the list (may have more if multiple exceptions are raised)
            service_code = SERVICES['Docker']
            exception_code = exception_codes.pop()

    return exception_code, service_code 

def find_exception(module):
    # Exception classes in module
    exceptions = [ename for ename, eclass in inspect.getmembers(module, inspect.isclass)
                  if issubclass(eclass, BaseException)]

    # Exception classes in submodule

    try:
        submodules = inspect.getmembers(module, inspect.ismodule)
    except Exception:
        submodules = []

    for submodule_name, submodule in submodules:
        try:
            classes = inspect.getmembers(submodule, inspect.isclass)
        except Exception:
            classes = []

        exceptions += [ename for ename, eclass in classes
                       if issubclass(eclass, BaseException)]

    return set(exceptions) 

def api_patch(namespace, kind, name, entity_name, body):
    api_url = '/'.join([KUBE_API_URL, namespace, kind, name])
    while True:
        try:
            token = read_token()
            if token:
                r = requests.patch(api_url, data=body, verify=KUBE_CA_CERT,
                                   headers={'Content-Type': 'application/strategic-merge-patch+json',
                                            'Authorization': 'Bearer {0}'.format(token)})
                if r.status_code >= 300:
                    logger.warning('Unable to change %s: %s', entity_name, r.text)
                else:
                    break
            else:
                logger.warning('Unable to read Kubernetes authorization token')
        except requests.exceptions.RequestException as e:
            logger.warning('Exception when executing PATCH on %s: %s', api_url, e)
        time.sleep(1) 

def entity_iter(ids, debug=False, attempts=5):
    for num, cur in enumerate(chunk(ids, page_size)):
        if debug:
            print('entity_iter: {}/{}'.format(num * page_size, len(ids)))
        ids = '|'.join(cur)
        for attempt in range(attempts):
            try:
                r = api_call({'action': 'wbgetentities', 'ids': ids})
                break
            except requests.exceptions.ChunkedEncodingError:
                if attempt == attempts - 1:
                    raise
                time.sleep(1)
        r.raise_for_status()
        json_data = r.json()
        if 'entities' not in json_data:
            mail.send_mail('error fetching wikidata entities', r.text)

        for qid, entity in json_data['entities'].items():
            yield qid, entity 

def new_paste(self, text, sender, **kwargs) -> str:

        ts = kwargs["date"] + kwargs["time"] \
            if "date" in kwargs and "time" in kwargs \
            else get_now().strftime("%Y%m%d%H%M")

        filename = "{sender}.{ts}.txt".format(
            sender=sender,
            ts=ts
        )
        data = {
            'api_option': "paste",
            'api_dev_key': self.api_dev_key,
            'api_paste_code': text,
            'api_paste_name': filename,
        }
        try:
            r = requests.post(self.api_url, data=data, timeout=5)
        except requests.exceptions.Timeout:
            logger.error("Timeout uploading to Pastebin")
            return None

        if r.text.startswith("http"):
            return r.text.strip()

        return None 

def new_paste(self, text, sender, **kwargs) -> str:
        data = {
            'vimcn': text,
        }
        try:
            r = requests.post(self.api_url, data=data, timeout=5)
        except requests.exceptions.Timeout:
            logger.error("Timeout uploading to Vinergy")
            return None

        if r.text.startswith("http"):
            return r.text.strip()

        return None 

def upload_image(self, filename=None, filedata=None, **kwargs):
        if filedata is None:
            with open(filename, 'rb') as f:
                b64img = b64encode(f.read())
        else:
            b64img = b64encode(filedata)

        headers = {"Authorization": "Client-ID %s" % self.client_id}
        try:
            r = requests.post(
                self.url,
                headers=headers,
                data={
                    'image': b64img,
                    'type': 'base64',
                },
                timeout=5,
            )
        except requests.exceptions.Timeout:
            logger.error("Timeout uploading to Imgur")
            return None
        except:
            logger.exception("Unknown errror uploading to Imgur")
            return None

        try:
            ret = json.loads(r.text)
        except:
            return None
        if ret.get('status', None) != 200 or ret.get('success', False) != True:
            logger.error(
                "Error: Imgur returned error, {}".format(ret.get('data', ''))
            )
            return None

        link = ret.get('data', {}).get('link', None)
        return link if link is None else re.sub(r'^http:', 'https:', link) 

def http_get(self, path, params=None, **kwargs):
        """
        Make a GET request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.get(url, params=params, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem retrieving data from " \
                      "the Kubernetes API server. URL: {}, params: {}".format(url, params)
            logger.error(message)
            raise KubeException(message) from err

        return response 

def http_delete(self, path, **kwargs):
        """
        Make a DELETE request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.delete(url, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem deleting data from " \
                      "the Kubernetes API server. URL: {}".format(url)
            logger.error(message)
            raise KubeException(message) from err

        return response 

def has_permissions(self):
        try:
            self.check_bulk_quota_usage()
        except requests.exceptions.HTTPError as err:
            if err.response is not None:
                for error_response_item in err.response.json():
                    if error_response_item.get('errorCode') == 'API_DISABLED_FOR_ORG':
                        return False
        return True 

def get_looker_release_version(self) -> str:
        """Gets the version number of connected Looker instance.

        Returns:
            str: Looker instance release version number (e.g. 6.22.12)

        """
        logger.debug("Checking Looker instance release version")

        url = utils.compose_url(self.api_url, path=["versions"])

        response = self.get(url=url, timeout=TIMEOUT_SEC)
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            raise LookerApiError(
                name="unable-to-get-version",
                title="Couldn't get Looker's release version.",
                status=response.status_code,
                detail=(
                    "Unable to get the release version of your Looker instance. "
                    "Please try again."
                ),
                response=response,
            )

        return response.json()["looker_release_version"] 

def update_workspace(self, project: str, workspace: str) -> None:
        """Updates the session workspace.

        Args:
            project: Name of the Looker project to use.
            workspace: The workspace to switch to, either 'production' or 'dev'
        """
        logger.debug(f"Updating session to use the {workspace} workspace")
        url = utils.compose_url(self.api_url, path=["session"])
        body = {"workspace_id": workspace}
        response = self.patch(url=url, json=body, timeout=TIMEOUT_SEC)
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError:
            raise LookerApiError(
                name="unable-to-update-workspace",
                title="Couldn't update the session's workspace.",
                status=response.status_code,
                detail=(
                    f"Unable to update workspace to '{workspace}'. "
                    "If you have any unsaved work on the branch "
                    "checked out by the user whose API credentials "
                    "Spectacles is using, please save it and try again."
                ),
                response=response,
            )