Python requests.exceptions() Examples

The following are code examples for showing how to use requests.exceptions(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: MangaScrapper   Author: AnimeshShaw   File: mangascrapper.py    Apache License 2.0 7 votes vote down vote up
def _set_response_ins_(self, pageurl):
        """
        Sets the response for the GET request of pageurl and stores it in self.resp

        :param pageurl: url for which we store the response.
        """
        try:
            s = requests.Session()
            a = requests.adapters.HTTPAdapter(max_retries=5)
            s.mount('http://', a)
            resp = s.get(pageurl, timeout=30)
            self.__resp_obj__ = resp
            resp.close()
        except requests.exceptions.Timeout:
            logging.error("\tVery Slow Internet Connection.")
        except requests.exceptions.ConnectionError:
            logging.error("\tNetwork Unavailable. Check your connection.")
        except requests.exceptions.MissingSchema:
            logging.error("\t503 Service Unavailable. Retrying download ... ") 
Example 2
Project: fishroom   Author: tuna   File: gitter.py    GNU General Public License v3.0 6 votes vote down vote up
def _must_post(self, api, data=None, json=None, timeout=10, **kwargs):
        if data is not None:
            kwargs['data'] = data
        elif json is not None:
            kwargs['json'] = json
        else:
            kwargs['data'] = {}
        kwargs['timeout'] = timeout

        try:
            r = requests.post(api, **kwargs)
            return r
        except requests.exceptions.Timeout:
            logger.error("Timeout requesting Gitter")
        except KeyboardInterrupt:
            raise
        except:
            logger.exception("Unknown error requesting Gitter")
        return None 
Example 3
Project: fishroom   Author: tuna   File: photostore.py    GNU General Public License v3.0 6 votes vote down vote up
def upload_image(self, filename=None, filedata=None, **kwargs) -> str:
        if filedata is None:
            files = {"image": open(filename, 'rb')}
        else:
            files = {"image": filedata}

        try:
            r = requests.post(self.url, files=files, timeout=5)
        except requests.exceptions.Timeout:
            logger.error("Timeout uploading to VimCN")
            return None
        except:
            logger.exception("Unknown errror uploading to VimCN")
            return None
        if not r.ok:
            return None
        return r.text.strip() 
Example 4
Project: controller   Author: deis   File: __init__.py    MIT License 6 votes vote down vote up
def http_head(self, path, **kwargs):
        """
        Make a HEAD request to the k8s server.
        """
        try:

            url = urljoin(self.url, path)
            response = self.session.head(url, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem retrieving headers from " \
                "the Kubernetes API server. URL: {}".format(url)
            logger.error(message)
            raise KubeException(message) from err

        return response 
Example 5
Project: controller   Author: deis   File: __init__.py    MIT License 6 votes vote down vote up
def http_post(self, path, data=None, json=None, **kwargs):
        """
        Make a POST request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.post(url, data=data, json=json, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem posting data to " \
                      "the Kubernetes API server. URL: {}, " \
                      "data: {}, json: {}".format(url, data, json)
            logger.error(message)
            raise KubeException(message) from err

        return response 
Example 6
Project: controller   Author: deis   File: __init__.py    MIT License 6 votes vote down vote up
def http_put(self, path, data=None, **kwargs):
        """
        Make a PUT request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.put(url, data=data, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem putting data to " \
                      "the Kubernetes API server. URL: {}, " \
                      "data: {}".format(url, data)
            logger.error(message)
            raise KubeException(message) from err

        return response 
Example 7
Project: cis   Author: mozilla-iam   File: common.py    Mozilla Public License 2.0 6 votes vote down vote up
def _load_publisher_rules(self, rules_url):
        """
        Get the CIS integration rules
        """
        rules = None
        if not self.always_use_local_file:
            if rules_url is not None:
                try:
                    r = requests.get(rules_url)
                    rules = r.json()
                except (json.JSONDecodeError, requests.exceptions.ConnectionError) as e:
                    logger.debug("Failed to load rules data from rules_url {} ({})".format(rules_url, e))
        # Fall-back to built-in copy
        if self.always_use_local_file or rules is None:
            rules_file = "data/well-known/mozilla-iam-publisher-rules"
            if not os.path.isfile(rules_file):
                dirname = os.path.dirname(os.path.realpath(__file__))
                path = dirname + "/" + rules_file
            else:
                path = rules_file

            rules = json.load(open(path))
        return rules 
Example 8
Project: adh6   Author: bonnetn   File: auth.py    GNU General Public License v3.0 6 votes vote down vote up
def get_sso_groups(token):
    try:
        verify_cert = True
        if is_development_environment():
            verify_cert = False

        headers = {"Authorization": "Bearer " + token}
        r = requests.get(
            current_app.config["AUTH_SERVER_ADDRESS"],
            headers=headers,
            timeout=1,
            verify=verify_cert
        )
    except requests.exceptions.ReadTimeout:
        return None
    if r.status_code != 200 or "id" not in r.json():
        return None

    result = r.json()
    if is_development_environment():
        result["groups"] = [ADH6_USER, ADH6_ADMIN]  # If we are testing, consider the user asg.admin
    return result 
Example 9
Project: pyfgt   Author: p4r4n0y1ng   File: fortigate.py    Apache License 2.0 6 votes vote down vote up
def __init__(self, host, user=None, passwd=None, debug=False, use_ssl=True, verify_ssl=False, timeout=300,
                 disable_request_warnings=False, apikey=None):
        super(FortiGate, self).__init__()
        self._host = host
        self._user = user
        self._req_id = 0
        self._url = None
        self._session = None
        self._sid = None
        self._timeout = timeout
        self._debug = debug
        self._use_ssl = use_ssl
        self._verify_ssl = verify_ssl
        self._apikeyused = True if passwd is None and apikey is not None else False
        self._passwd = passwd if passwd is not None else apikey
        self._req_resp_object = RequestResponse()
        self._logger = None
        if disable_request_warnings:
            requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning) 
Example 10
Project: pydkron   Author: oldmantaiter   File: client.py    MIT License 6 votes vote down vote up
def _call(self, method, endpoint, payload=None):
        """
        Call the endpoint and return the response
        """
        resp = None
        shuffle(self.hosts)
        for index in xrange(len(self.hosts)):
            url = "http://%s/v1%s" % (self.hosts[index], endpoint)
            req_args = {'timeout': 10, 'headers': {'Content-Type': 'application/json'}}
            try:
                if method == _GET:
                    resp = requests.get(url, **req_args)
                elif method == _POST:
                    resp = requests.post(
                        url, data=payload, **req_args)
                elif method == _DELETE:
                    resp = requests.delete(url, **req_args)
                break
            except requests.exceptions.RequestException:
                continue
        if resp is None:
            raise DkronClientException("No valid host found")
        return resp 
Example 11
Project: aiolocust   Author: kpidata   File: clients.py    MIT License 6 votes vote down vote up
def __exit__(self, exc, value, traceback):
        if self._is_reported:
            # if the user has already manually marked this response as failure or success
            # we can ignore the default haviour of letting the response code determine the outcome
            return exc is None
        
        if exc:
            if isinstance(value, ResponseError):
                self.failure(value)
            else:
                return False
        else:
            try:
                self.raise_for_status()
            except requests.exceptions.RequestException as e:
                self.failure(e)
            else:
                self.success()
        return True 
Example 12
Project: urllib-requests-adapter   Author: Matrixcoffee   File: __init__.py    Apache License 2.0 6 votes vote down vote up
def execute(self):
		timeout = self.timeout_seconds
		if timeout is None: timeout = GLOBAL_TIMEOUT_SECONDS
		try:
			if timeout is None:
				r = urllib.request.urlopen(self.rq, **_get_tls_parms())
			else:
				r = urllib.request.urlopen(self.rq, None, timeout, **_get_tls_parms())
		except urllib.error.HTTPError as e:
			self.status_code = e.code
			self.text = e.msg
			return self
		except (http.client.HTTPException, urllib.error.URLError, socket.timeout) as e:
			e2 = exceptions.RequestException("{}.{!r}".format(type(e).__module__, e))
			raise e2
		self._rdata = r.read()
		if DEBUG: print("Response._rdata set to:", repr(self._rdata), file=sys.stderr)
		self.map_request(r)
		r.close()
		return self 
Example 13
Project: codechef-cli   Author: sk364   File: helpers.py    GNU General Public License v3.0 6 votes vote down vote up
def request(session, method, url, **kwargs):
    """
    :desc: Custom wrapper method to add a timeout message
           when there is a `requests.exceptions.ConnectionError`
           or `requests.ReadTimeout` exception.
    :param: `session` requests.Session object
            `method` HTTP method to use
            `url` name of the URL
    :return: requests.Response object.
    """

    try:
        return session.request(method=method, url=url, timeout=(5, 5), **kwargs)
    except (ConnectionError, ReadTimeout):
        print(INTERNET_DOWN_MSG)
        sys.exit(1) 
Example 14
Project: python-wsmanclient   Author: intelsdi-x   File: wsman.py    Apache License 2.0 6 votes vote down vote up
def __init__(self, endpoint, resource_uri, optimization=True,
                 max_elems=100, filter_query=None, filter_dialect=None):
        self.endpoint = endpoint
        self.resource_uri = resource_uri
        self.filter_dialect = None
        self.filter_query = None
        self.optimization = optimization
        self.max_elems = max_elems

        if filter_query is not None:
            try:
                self.filter_dialect = FILTER_DIALECT_MAP[filter_dialect]
            except KeyError:
                valid_opts = ', '.join(FILTER_DIALECT_MAP)
                raise exceptions.WSManInvalidFilterDialect(
                    invalid_filter=filter_dialect, supported=valid_opts)

            self.filter_query = filter_query 
Example 15
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilder.py    MIT License 6 votes vote down vote up
def patch(self, restApi, data={}, silentMode=False):
        """
        Description
           A HTTP PATCH function to modify configurations.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           silentMode: True or False.  To display URL, data and header info.
        """

        if silentMode == False:
            print('\nPATCH:', restApi)
            print('DATA:', data)
            print('HEADERS:', self.jsonHeader)
        try:
            response = requests.patch(restApi, data=json.dumps(data), headers=self.jsonHeader)
            if silentMode == False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                print('\nPatch error:')
                raise IxNetRestApiException('http PATCH error: {0}\n'.format(response.text))
            return response
        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http PATCH error: {0}\n'.format(errMsg)) 
Example 16
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilderRestApi.py    MIT License 6 votes vote down vote up
def patch(self, restApi, data={}, silentMode=False):
        """
        Description
           A HTTP PATCH function to modify configurations.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           silentMode: True or False.  To display URL, data and header info.
        """

        if silentMode == False:
            print('\nPATCH:', restApi)
            print('DATA:', data)
            print('HEADERS:', self.jsonHeader)
        try:
            response = requests.patch(restApi, data=json.dumps(data), headers=self.jsonHeader)
            if silentMode == False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                print('\nPatch error:')
                raise IxNetRestApiException('http PATCH error: {0}\n'.format(response.text))
            return response
        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http PATCH error: {0}\n'.format(errMsg)) 
Example 17
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilderRestApi.py    MIT License 6 votes vote down vote up
def patch(self, restApi, data={}, silentMode=False):
        """
        Description
           A HTTP PATCH function to modify configurations.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           silentMode: True or False.  To display URL, data and header info.
        """

        if silentMode == False:
            print('\nPATCH:', restApi)
            print('DATA:', data)
            print('HEADERS:', self.jsonHeader)
        try:
            response = requests.patch(restApi, data=json.dumps(data), headers=self.jsonHeader)
            if silentMode == False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                print('\nPatch error:')
                raise IxNetRestApiException('http PATCH error: {0}\n'.format(response.text))
            return response
        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http PATCH error: {0}\n'.format(errMsg)) 
Example 18
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilderRestApi.py    MIT License 6 votes vote down vote up
def patch(self, restApi, data={}, silentMode=False):
        """
        Description
           A HTTP PATCH function to modify configurations.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           silentMode: True or False.  To display URL, data and header info.
        """

        if silentMode == False:
            print('\nPATCH:', restApi)
            print('DATA:', data)
            print('HEADERS:', self.jsonHeader)
        try:
            response = requests.patch(restApi, data=json.dumps(data), headers=self.jsonHeader)
            if silentMode == False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                print('\nPatch error:')
                raise IxNetRestApiException('http PATCH error: {0}\n'.format(response.text))
            return response
        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http PATCH error: {0}\n'.format(errMsg)) 
Example 19
Project: PTBot   Author: whocares-openscene   File: pagecheck.py    GNU General Public License v3.0 6 votes vote down vote up
def down(text):
    """<url> - checks if <url> is online or offline
    :type text: str
    """

    if "://" not in text:
        text = 'http://' + text

    text = 'http://' + urllib.parse.urlparse(text).netloc

    try:
        requests.get(text)
    except requests.exceptions.ConnectionError:
        return '{} seems to be down'.format(text)
    else:
        return '{} seems to be up'.format(text) 
Example 20
Project: smbproxy   Author: Seekscale   File: fs_cache.py    GNU General Public License v2.0 6 votes vote down vote up
def register_operation_failure(self, v):
        self.incr_counter('operation.failure')
        return v

    #
    # The raw HTTP requests
    #

    # def _http_req(self, endpoint, **kwargs):
    #     url = 'http://%s:%d/%s' % (self.http_service_host, self.http_service_port, endpoint)
    #
    #     # self.log.msg("[HTTP] %s %r" % (url, kwargs), level=logger.INFO)
    #
    #     try:
    #         r = requests.post(url, **kwargs)
    #     except requests.exceptions.RequestException:
    #         self.log.msg("ERROR in HTTP request: %s" % get_traceback(), level=logger.INFO)
    #     else:
    #         if r.status_code != 200:
    #             self.log.msg("Bad status code in HTTP request: %d" % r.status_code, level=logger.INFO)
    #             return None
    #         else:
    #             return r 
Example 21
Project: esst   Author: theendsofinvention   File: __init__.py    MIT License 6 votes vote down vote up
def wan_available(retry: int = 0):
    """

    Returns: True if connected to WAN

    """
    try:
        response = requests.get('http://google.com', timeout=2)
        commands.DCS.unblock_start('no WAN connection available')
        commands.DISCORD.can_start()
        return bool(response.ok)
    except requests.exceptions.RequestException:
        if retry < 5:
            LOGGER.debug('Internet connection loss detected, retry %s', retry)
            await asyncio.sleep(2)
            result = await wan_available(retry + 1)
            return result
        LOGGER.debug(f'Internet connection loss detected, no more retry')
        commands.DISCORD.cannot_start()
        commands.DCS.block_start('no WAN connection available')
        return False 
Example 22
Project: heliopy   Author: heliopython   File: cdasrest.py    GNU General Public License v3.0 6 votes vote down vote up
def get_cdas_url(starttime, endtime, vars, dataset, timeout=10):
    dataview = 'sp_phys'
    if vars is None:
        try:
            var_info = get_variables(dataset, timeout=timeout)
        except requests.exceptions.ReadTimeout:
            raise util.NoDataError(
                'Connection to CDAweb timed out when getting CDAS URL for '
                f'{dataset} data for interval {starttime} - {endtime}.')

        if not len(var_info):
            raise util.NoDataError(
                f'No {dataset} data available for date {date}')

        vars = [v['Name'] for v in var_info['VariableDescription']]

    uri = '/'.join(['dataviews', dataview,
                    'datasets', dataset,
                    'data',
                    ','.join([starttime.strftime('%Y%m%dT%H%M%SZ'),
                              endtime.strftime('%Y%m%dT%H%M%SZ')]),
                    ','.join(vars)
                    ])
    url = '/'.join([CDAS_BASEURL, uri])
    return url 
Example 23
Project: SparkBot   Author: UniversalSuperBox   File: test_all.py    Apache License 2.0 6 votes vote down vote up
def test_full_internal_error_text(self, full_bot_setup):
        """Tests that unhandled exceptions in commands with error text are handled safely"""

        bot = full_bot_setup["bot"]
        aux_api = full_bot_setup["aux_api"]
        emulator = full_bot_setup["emulator"]

        @bot.command("exception")
        def cause_exception():
            raise ValueError("Whoops", "Hey, an exception")

        self.start_receiver(full_bot_setup["receiver_process"], full_bot_setup["receiver_webhook_url"])

        bot_reply = self.invoke_bot(aux_api, emulator.bot_id, emulator.bot_displayname, "exception", room_name="test1")

        assert bot_reply.text == "⚠️ Error: Hey, an exception" 
Example 24
Project: SparkBot   Author: UniversalSuperBox   File: test_all.py    Apache License 2.0 6 votes vote down vote up
def test_full_internal_error(self, full_bot_setup):
        """Tests that unhandled exceptions in commands without error text are handled safely"""

        bot = full_bot_setup["bot"]
        aux_api = full_bot_setup["aux_api"]
        emulator = full_bot_setup["emulator"]

        @bot.command("exception")
        def cause_exception():
            raise ValueError("Whoops")

        self.start_receiver(full_bot_setup["receiver_process"], full_bot_setup["receiver_webhook_url"])

        bot_reply = self.invoke_bot(aux_api, emulator.bot_id, emulator.bot_displayname, "exception", room_name="test1")

        assert bot_reply.text == "⚠️ Error: Something happened internally. For more information, contact the bot author." 
Example 25
Project: ff-net-worth-calculator   Author: blocktrron   File: __init__.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def download(url, timeout=5):
    try:
        response = requests.get(url, timeout=timeout)
    except requests.exceptions.RequestException as ex:
        error(f"Exception caught while fetching {url}\n{ex}")
        return None

    if response.status_code != 200:
        error(f"Unexpected status code {response.status_code} while fetching {url}")
        return None

    return response 
Example 26
Project: fishroom   Author: tuna   File: textstore.py    GNU General Public License v3.0 5 votes vote down vote up
def new_paste(self, text, sender, **kwargs) -> str:

        ts = kwargs["date"] + kwargs["time"] \
            if "date" in kwargs and "time" in kwargs \
            else get_now().strftime("%Y%m%d%H%M")

        filename = "{sender}.{ts}.txt".format(
            sender=sender,
            ts=ts
        )
        data = {
            'api_option': "paste",
            'api_dev_key': self.api_dev_key,
            'api_paste_code': text,
            'api_paste_name': filename,
        }
        try:
            r = requests.post(self.api_url, data=data, timeout=5)
        except requests.exceptions.Timeout:
            logger.error("Timeout uploading to Pastebin")
            return None

        if r.text.startswith("http"):
            return r.text.strip()

        return None 
Example 27
Project: fishroom   Author: tuna   File: textstore.py    GNU General Public License v3.0 5 votes vote down vote up
def new_paste(self, text, sender, **kwargs) -> str:
        data = {
            'vimcn': text,
        }
        try:
            r = requests.post(self.api_url, data=data, timeout=5)
        except requests.exceptions.Timeout:
            logger.error("Timeout uploading to Vinergy")
            return None

        if r.text.startswith("http"):
            return r.text.strip()

        return None 
Example 28
Project: fishroom   Author: tuna   File: photostore.py    GNU General Public License v3.0 5 votes vote down vote up
def upload_image(self, filename=None, filedata=None, **kwargs):
        if filedata is None:
            with open(filename, 'rb') as f:
                b64img = b64encode(f.read())
        else:
            b64img = b64encode(filedata)

        headers = {"Authorization": "Client-ID %s" % self.client_id}
        try:
            r = requests.post(
                self.url,
                headers=headers,
                data={
                    'image': b64img,
                    'type': 'base64',
                },
                timeout=5,
            )
        except requests.exceptions.Timeout:
            logger.error("Timeout uploading to Imgur")
            return None
        except:
            logger.exception("Unknown errror uploading to Imgur")
            return None

        try:
            ret = json.loads(r.text)
        except:
            return None
        if ret.get('status', None) != 200 or ret.get('success', False) != True:
            logger.error(
                "Error: Imgur returned error, {}".format(ret.get('data', ''))
            )
            return None

        link = ret.get('data', {}).get('link', None)
        return link if link is None else re.sub(r'^http:', 'https:', link) 
Example 29
Project: bigquerylayers   Author: smandaric   File: requests.py    GNU General Public License v3.0 5 votes vote down vote up
def __call__(self, url, method='GET', body=None, headers=None,
                 timeout=None, **kwargs):
        """Make an HTTP request using requests.

        Args:
            url (str): The URI to be requested.
            method (str): The HTTP method to use for the request. Defaults
                to 'GET'.
            body (bytes): The payload / body in HTTP request.
            headers (Mapping[str, str]): Request headers.
            timeout (Optional[int]): The number of seconds to wait for a
                response from the server. If not specified or if None, the
                requests default timeout will be used.
            kwargs: Additional arguments passed through to the underlying
                requests :meth:`~requests.Session.request` method.

        Returns:
            google.auth.transport.Response: The HTTP response.

        Raises:
            google.auth.exceptions.TransportError: If any exception occurred.
        """
        try:
            _LOGGER.debug('Making request: %s %s', method, url)
            response = self.session.request(
                method, url, data=body, headers=headers, timeout=timeout,
                **kwargs)
            return _Response(response)
        except requests.exceptions.RequestException as caught_exc:
            new_exc = exceptions.TransportError(caught_exc)
            six.raise_from(new_exc, caught_exc) 
Example 30
Project: controller   Author: deis   File: __init__.py    MIT License 5 votes vote down vote up
def http_get(self, path, params=None, **kwargs):
        """
        Make a GET request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.get(url, params=params, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem retrieving data from " \
                      "the Kubernetes API server. URL: {}, params: {}".format(url, params)
            logger.error(message)
            raise KubeException(message) from err

        return response 
Example 31
Project: controller   Author: deis   File: __init__.py    MIT License 5 votes vote down vote up
def http_delete(self, path, **kwargs):
        """
        Make a DELETE request to the k8s server.
        """
        try:
            url = urljoin(self.url, path)
            response = self.session.delete(url, **kwargs)
        except requests.exceptions.ConnectionError as err:
            # reraise as KubeException, but log stacktrace.
            message = "There was a problem deleting data from " \
                      "the Kubernetes API server. URL: {}".format(url)
            logger.error(message)
            raise KubeException(message) from err

        return response 
Example 32
Project: ansible_collections_google   Author: ansible-collections   File: gcp_utils.py    GNU General Public License v3.0 5 votes vote down vote up
def full_get(self, url, params=None, **kwargs):
        kwargs['headers'] = self._set_headers(kwargs.get('headers'))
        try:
            return self.session().get(url, params=params, **kwargs)
        except getattr(requests.exceptions, 'RequestException') as inst:
            # Only log the message to avoid logging any sensitive info.
            self.module.fail_json(msg=inst.message) 
Example 33
Project: ansible_collections_google   Author: ansible-collections   File: gcp_utils.py    GNU General Public License v3.0 5 votes vote down vote up
def full_post(self, url, data=None, json=None, **kwargs):
        kwargs['headers'] = self._set_headers(kwargs.get('headers'))

        try:
            return self.session().post(url, data=data, json=json, **kwargs)
        except getattr(requests.exceptions, 'RequestException') as inst:
            self.module.fail_json(msg=inst.message) 
Example 34
Project: ansible_collections_google   Author: ansible-collections   File: gcp_utils.py    GNU General Public License v3.0 5 votes vote down vote up
def full_put(self, url, data=None, **kwargs):
        kwargs['headers'] = self._set_headers(kwargs.get('headers'))

        try:
            return self.session().put(url, data=data, **kwargs)
        except getattr(requests.exceptions, 'RequestException') as inst:
            self.module.fail_json(msg=inst.message) 
Example 35
Project: ansible_collections_google   Author: ansible-collections   File: gcp_utils.py    GNU General Public License v3.0 5 votes vote down vote up
def full_patch(self, url, data=None, **kwargs):
        kwargs['headers'] = self._set_headers(kwargs.get('headers'))

        try:
            return self.session().patch(url, data=data, **kwargs)
        except getattr(requests.exceptions, 'RequestException') as inst:
            self.module.fail_json(msg=inst.message) 
Example 36
Project: ansible_collections_google   Author: ansible-collections   File: gcp_utils.py    GNU General Public License v3.0 5 votes vote down vote up
def full_delete(self, url, **kwargs):
        kwargs['headers'] = self._set_headers(kwargs.get('headers'))

        try:
            return self.session().delete(url, **kwargs)
        except getattr(requests.exceptions, 'RequestException') as inst:
            self.module.fail_json(msg=inst.message) 
Example 37
Project: cis   Author: mozilla-iam   File: common.py    Mozilla Public License 2.0 5 votes vote down vote up
def _load_well_known(self):
        """
        Gets the discovery url's data ("well-known")
        Return dict,None the well-known JSON data copy
        """
        # Memory cache
        if self._well_known_json is not None:
            return self._well_known_json

        if not self.always_use_local_file:
            try:
                r = requests.get(self.discovery_url)
                self._well_known_json = r.json()
            except (json.JSONDecodeError, requests.exceptions.ConnectionError) as e:
                logger.debug("Failed to fetch schema url from discovery {} ({})".format(self.discovery_url, e))
                logger.debug("Using builtin copy")

        if self._well_known_json is None or self.always_use_local_file:
            well_known_file = "data/well-known/mozilla-iam"  # Local fall-back
            if not os.path.isfile(well_known_file):
                dirname = os.path.dirname(os.path.realpath(__file__))
                path = dirname + "/" + well_known_file
            else:
                path = well_known_file
            self._well_known_json = json.load(open(path))

        return self._well_known_json 
Example 38
Project: cis   Author: mozilla-iam   File: common.py    Mozilla Public License 2.0 5 votes vote down vote up
def _load_schema(self, schema_url, stype="data/profile.schema"):
        """
        Loads JSON Schema from an URL
        @schema_url: str,None the schema URL
        @stype: str, type of schema to load. This is also the name of the library builtin, local copy.
        Return dict JSON object which is the CIS Profile Schema
        """
        schema = None
        if not self.always_use_local_file:
            if schema_url is not None:
                try:
                    r = requests.get(schema_url)
                    schema = r.json()
                except (json.JSONDecodeError, requests.exceptions.ConnectionError) as e:
                    logger.debug("Failed to load schema from schema_url {} ({})".format(schema_url, e))

        # That did not work, fall-back to local, built-in copy
        if schema is None or self.always_use_local_file:
            # Builtin, hardcoded schema from library
            schema_file = stype
            if not os.path.isfile(schema_file):
                dirname = os.path.dirname(os.path.realpath(__file__))
                path = dirname + "/" + schema_file
            else:
                path = schema_file

            schema = json.load(open(path))
        return schema 
Example 39
Project: RTX   Author: RTXteam   File: cache_control_helper.py    MIT License 5 votes vote down vote up
def __init__(self):
        self.sess = CacheControl(requests.session(), heuristic=CustomHeuristic(days=30), cache=FileCache('.web_cache'))
        self.exceptions = requests.exceptions 
Example 40
Project: tap-salesforce   Author: singer-io   File: bulk.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def has_permissions(self):
        try:
            self.check_bulk_quota_usage()
        except requests.exceptions.HTTPError as err:
            if err.response is not None:
                for error_response_item in err.response.json():
                    if error_response_item.get('errorCode') == 'API_DISABLED_FOR_ORG':
                        return False
        return True 
Example 41
Project: hyperledger-py   Author: yeasy   File: client.py    Apache License 2.0 5 votes vote down vote up
def _raise_for_status(self, response, explanation=None):
        """Raises stored :class:`APIError`, if one occurred."""
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 404:
                raise errors.NotFound(e, response, explanation=explanation)
            raise errors.APIError(e, response, explanation=explanation) 
Example 42
Project: python-wsmanclient   Author: intelsdi-x   File: wsman.py    Apache License 2.0 5 votes vote down vote up
def _do_request(self, payload):
        payload = payload.build()
        LOG.debug('Sending request to %(endpoint)s: %(payload)s',
                  {'endpoint': self.endpoint, 'payload': payload})
        try:
            resp = requests.post(
                self.endpoint,
                auth=requests.auth.HTTPBasicAuth(self.username, self.password),
                data=payload,
                # TODO(ifarkas): enable cert verification
                verify=False)

        except Exception as e:
            # This is a hack for handling 'No route to host' ConnectionError,
            # so that the Traceback would not be shown
            if e.__class__ == 'requests.exceptions.ConnectionError':
                LOG.exception('Request failed (ConnectionError)')
                raise exceptions.WSManRequestFailure()
            if e.__class__ == 'requests.exceptions.RequestException':
                LOG.exception('Request failed')
                raise exceptions.WSManRequestFailure()
            else:
                raise

        LOG.debug('Received response from %(endpoint)s: %(payload)s',
                  {'endpoint': self.endpoint, 'payload': resp.content})
        if not resp.ok:
            raise exceptions.WSManInvalidResponse(
                status_code=resp.status_code,
                reason=resp.reason)
        else:
            return resp 
Example 43
Project: python-wsmanclient   Author: intelsdi-x   File: wsman.py    Apache License 2.0 5 votes vote down vote up
def invoke(self, resource_uri, method, selectors=None, properties=None,
               expected_return_value=None):
        """Invokes a remote WS-Man method

        :param resource_uri: URI of the resource
        :param method: name of the method to invoke
        :param selectors: dictionary of selectors
        :param properties: dictionary of properties
        :param expected_return_value: expected return value reported back by
            the DRAC card. For return value codes check the profile
            documentation of the resource used in the method call. If not set,
            return value checking is skipped.
        :returns: an lxml.etree.Element object of the response received
        :raises: WSManRequestFailure on request failures
        :raises: WSManInvalidResponse when receiving invalid response
        :raises: DRACOperationFailed on error reported back by the DRAC
                 interface
        :raises: DRACUnexpectedReturnValue on return value mismatch
        """
        if selectors is None:
            selectors = {}

        if properties is None:
            properties = {}

        resp = super(WSManClient, self).invoke(resource_uri, method, selectors,
                                               properties)

        return_value = utils.find_xml(resp, 'ReturnValue', resource_uri).text
        if return_value == utils.RET_ERROR:
            message_elems = utils.find_xml(resp, 'Message', resource_uri, True)
            messages = [message_elem.text for message_elem in message_elems]
            raise exceptions.DRACOperationFailed(drac_messages=messages)

        if (expected_return_value is not None and
                return_value != expected_return_value):
            raise exceptions.DRACUnexpectedReturnValue(
                expected_return_value=expected_return_value,
                actual_return_value=return_value)

        return resp 
Example 44
Project: nucypher   Author: nucypher   File: nodes.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def block_until_number_of_known_nodes_is(self,
                                             number_of_nodes_to_know: int,
                                             timeout: int = 10,
                                             learn_on_this_thread: bool = False):
        start = maya.now()
        starting_round = self._learning_round

        while True:
            rounds_undertaken = self._learning_round - starting_round
            if len(self.__known_nodes) >= number_of_nodes_to_know:
                if rounds_undertaken:
                    self.log.info("Learned about enough nodes after {} rounds.".format(rounds_undertaken))
                return True

            if not self._learning_task.running:
                self.log.warn("Blocking to learn about nodes, but learning loop isn't running.")
            if learn_on_this_thread:
                try:
                    self.learn_from_teacher_node(eager=True)
                except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout):
                    # TODO: Even this "same thread" logic can be done off the main thread.
                    self.log.warn("Teacher was unreachable.  No good way to handle this on the main thread.")

            # The rest of the fucking owl
            if (maya.now() - start).seconds > timeout:
                if not self._learning_task.running:
                    raise RuntimeError("Learning loop is not running.  Start it with start_learning().")
                else:
                    raise self.NotEnoughNodes("After {} seconds and {} rounds, didn't find {} nodes".format(
                        timeout, rounds_undertaken, number_of_nodes_to_know))
            else:
                time.sleep(.1) 
Example 45
Project: tts-ros1   Author: aws-robotics   File: amazonpolly.py    Apache License 2.0 5 votes vote down vote up
def _node_request_handler(self, request):
        """The callback function for processing service request.

        It never raises. If anything unexpected happens, it will return a PollyResponse with details of the exception.

        :param request: an instance of PollyRequest
        :return: a PollyResponse
        """
        rospy.loginfo('Amazon Polly Request: {}'.format(request))

        try:
            response = self._dispatch(request)
            rospy.loginfo('will return {}'.format(response))
            return PollyResponse(result=response)
        except Exception as e:
            current_dir = os.path.dirname(os.path.abspath(__file__))
            exc_type = sys.exc_info()[0]

            # not using `issubclass(exc_type, ConnectionError)` for the condition below because some versions
            # of urllib3 raises exception when doing `from requests.exceptions import ConnectionError`
            error_ogg_filename = 'connerror.ogg' if 'ConnectionError' in exc_type.__name__ else 'error.ogg'

            error_details = {
                'Audio File': os.path.join(current_dir, 'data', error_ogg_filename),
                'Audio Type': 'ogg',
                'Exception': {
                    'Type': str(exc_type),
                    'Module': exc_type.__module__,
                    'Name': exc_type.__name__,
                    'Value': str(e),
                },
                'Traceback': traceback.format_exc()
            }

            error_str = json.dumps(error_details)
            rospy.logerr(error_str)
            return PollyResponse(result=error_str) 
Example 46
Project: hypernova   Author: hyperhq   File: client.py    Apache License 2.0 5 votes vote down vote up
def _raise_for_status(self, response, explanation=None):
        """Raises stored :class:`APIError`, if one occurred."""
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 404:
                raise errors.NotFound(e, response, explanation=explanation)
            raise errors.APIError(e, response, explanation=explanation) 
Example 47
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilder.py    MIT License 5 votes vote down vote up
def get(self, restApi, data={}, stream=False, silentMode=False, ignoreError=False):
        """
        Description
           A HTTP GET function to send REST APIs.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           silentMode: True or False.  To display URL, data and header info.
           ignoreError: True or False.  If False, the response will be returned.
        """
        if silentMode is False:
            print('\nGET:', restApi)
            print('HEADERS:', self.jsonHeader)

        try:
            response = requests.get(restApi, headers=self.jsonHeader)

            if silentMode is False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                if ignoreError == False:
                    raise IxNetRestApiException('http GET error:{0}\n'.format(response.text))
            return response

        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http GET error: {0}\n'.format(errMsg)) 
Example 48
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilder.py    MIT License 5 votes vote down vote up
def post(self, restApi, data={}, headers=None, silentMode=False, ignoreError=False):
        """
        Description
           A HTTP POST function to mainly used to create or start operations.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           headers: The special header to use for the URL.
           silentMode: True or False.  To display URL, data and header info.
           noDataJsonDumps: True or False. If True, use json dumps. Else, accept the data as-is. 
           ignoreError: True or False.  If False, the response will be returned. No exception will be raised.
        """

        if headers != None:
            originalJsonHeader = self.jsonHeader
            self.jsonHeader = headers

        data = json.dumps(data)

        print('\nPOST:', restApi)
        if silentMode == False:
            print('DATA:', data)
            print('HEADERS:', self.jsonHeader)

        try:
            response = requests.post(restApi, data=data, headers=self.jsonHeader)
            # 200 or 201
            if silentMode == False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                if ignoreError == False:
                    raise IxNetRestApiException('http POST error: {0}\n'.format(response.text))

            return response
        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http POST error: {0}\n'.format(errMsg)) 
Example 49
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilder.py    MIT License 5 votes vote down vote up
def delete(self, restApi, data={}, headers=None):
        """
        Description
           A HTTP DELETE function to delete the session.
           For Linux API server only.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           headers: The header to use for the URL.
        """

        if headers != None:
            self.jsonHeader = headers

        print('\nDELETE:', restApi)
        print('DATA:', data)
        print('HEADERS:', self.jsonHeader)
        try:
            response = requests.delete(restApi, data=json.dumps(data), headers=self.jsonHeader)
            print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                raise IxNetRestApiException('http DELETE error: {0}\n'.format(response.text))
            return response
        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http DELETE error: {0}\n'.format(errMsg)) 
Example 50
Project: IxNetwork   Author: OpenIxia   File: IxNetRestApi.py    MIT License 5 votes vote down vote up
def get(self, restApi, data={}, stream=False, silentMode=False, ignoreError=False):
        """
        Description
           A HTTP GET function to send REST APIs.

        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           silentMode: True or False.  To display URL, data and header info.
           ignoreError: True or False.  If False, the response will be returned.
        """
        if silentMode is False or self.generateRestLogFile is True:
            #print('\nGET:', restApi)
            #print('HEADERS:', self.jsonHeader)
            self.logInfo('\nGET: {0}'.format(restApi))
            self.logInfo('HEADERS: {0}'.format(self.jsonHeader))

        try:
            # For binary file
            if stream:
                response = requests.get(restApi, stream=True, headers=self.jsonHeader, verify=self.verifySslCert)
            if stream == False:
                response = requests.get(restApi, headers=self.jsonHeader, verify=self.verifySslCert)

            if silentMode is False:
                self.logInfo('STATUS CODE: {0}'.format(response.status_code))
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                if ignoreError == False:
                    if 'message' in response.json() and response.json()['messsage'] != None:
                        self.logWarning('\n%s' % response.json()['message'])
                    raise IxNetRestApiException('GET error:{0}\n'.format(response.text))
            return response

        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('GET error: {0}\n'.format(errMsg)) 
Example 51
Project: IxNetwork   Author: OpenIxia   File: IxNetRestApi.py    MIT License 5 votes vote down vote up
def patch(self, restApi, data={}, silentMode=False):
        """
        Description
           A HTTP PATCH function to modify configurations.

        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           silentMode: True or False.  To display URL, data and header info.
        """

        if silentMode == False:
            self.logInfo('\nPATCH: %s' % restApi)
            self.logInfo('DATA: %s' % data)
            self.logInfo('HEADERS: %s' % self.jsonHeader)

        try:
            response = requests.patch(restApi, data=json.dumps(data), headers=self.jsonHeader, verify=self.verifySslCert)
            if silentMode == False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                if 'message' in response.json() and response.json()['messsage'] != None:
                    self.logWarning('\n%s' % response.json()['message'])
                self.showErrorMessage()
                raise IxNetRestApiException('PATCH error: {0}\n'.format(response.text))
            return response
        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('PATCH error: {0}\n'.format(errMsg)) 
Example 52
Project: IxNetwork   Author: OpenIxia   File: IxNetRestApi.py    MIT License 5 votes vote down vote up
def delete(self, restApi, data={}, headers=None):
        """
        Description
           A HTTP DELETE function to delete the session.
           For Linux API server only.

        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           headers: The header to use for the URL.
        """

        if headers != None:
            self.jsonHeader = headers

        self.logInfo('\nDELETE: %s' % restApi)
        self.logInfo('DATA: %s' % data)
        self.logInfo('HEADERS: %s' % self.jsonHeader)
        try:
            response = requests.delete(restApi, data=json.dumps(data), headers=self.jsonHeader, verify=self.verifySslCert)
            print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                self.showErrorMessage()
                raise IxNetRestApiException('DELETE error: {0}\n'.format(response.text))
            return response
        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('DELETE error: {0}\n'.format(errMsg)) 
Example 53
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilderRestApi.py    MIT License 5 votes vote down vote up
def get(self, restApi, data={}, stream=False, silentMode=False, ignoreError=False):
        """
        Description
           A HTTP GET function to send REST APIs.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           silentMode: True or False.  To display URL, data and header info.
           ignoreError: True or False.  If False, the response will be returned.
        """
        if silentMode is False:
            print('\nGET:', restApi)
            print('HEADERS:', self.jsonHeader)

        try:
            response = requests.get(restApi, headers=self.jsonHeader)

            if silentMode is False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                if ignoreError == False:
                    raise IxNetRestApiException('http GET error:{0}\n'.format(response.text))
            return response

        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http GET error: {0}\n'.format(errMsg)) 
Example 54
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilderRestApi.py    MIT License 5 votes vote down vote up
def post(self, restApi, data={}, headers=None, silentMode=False, ignoreError=False):
        """
        Description
           A HTTP POST function to mainly used to create or start operations.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           headers: The special header to use for the URL.
           silentMode: True or False.  To display URL, data and header info.
           noDataJsonDumps: True or False. If True, use json dumps. Else, accept the data as-is. 
           ignoreError: True or False.  If False, the response will be returned. No exception will be raised.
        """

        if headers != None:
            originalJsonHeader = self.jsonHeader
            self.jsonHeader = headers

        data = json.dumps(data)

        print('\nPOST:', restApi)
        if silentMode == False:
            print('DATA:', data)
            print('HEADERS:', self.jsonHeader)

        try:
            response = requests.post(restApi, data=data, headers=self.jsonHeader)
            # 200 or 201
            if silentMode == False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                if ignoreError == False:
                    raise IxNetRestApiException('http POST error: {0}\n'.format(response.text))

            return response
        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http POST error: {0}\n'.format(errMsg)) 
Example 55
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilderRestApi.py    MIT License 5 votes vote down vote up
def get(self, restApi, data={}, stream=False, silentMode=False, ignoreError=False):
        """
        Description
           A HTTP GET function to send REST APIs.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           silentMode: True or False.  To display URL, data and header info.
           ignoreError: True or False.  If False, the response will be returned.
        """
        if silentMode is False:
            print('\nGET:', restApi)
            print('HEADERS:', self.jsonHeader)

        try:
            response = requests.get(restApi, headers=self.jsonHeader)

            if silentMode is False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                if ignoreError == False:
                    raise IxNetRestApiException('http GET error:{0}\n'.format(response.text))
            return response

        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http GET error: {0}\n'.format(errMsg)) 
Example 56
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilderRestApi.py    MIT License 5 votes vote down vote up
def post(self, restApi, data={}, headers=None, silentMode=False, ignoreError=False):
        """
        Description
           A HTTP POST function to mainly used to create or start operations.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           headers: The special header to use for the URL.
           silentMode: True or False.  To display URL, data and header info.
           noDataJsonDumps: True or False. If True, use json dumps. Else, accept the data as-is. 
           ignoreError: True or False.  If False, the response will be returned. No exception will be raised.
        """

        if headers != None:
            originalJsonHeader = self.jsonHeader
            self.jsonHeader = headers

        data = json.dumps(data)

        print('\nPOST:', restApi)
        if silentMode == False:
            print('DATA:', data)
            print('HEADERS:', self.jsonHeader)

        try:
            response = requests.post(restApi, data=data, headers=self.jsonHeader)
            # 200 or 201
            if silentMode == False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                if ignoreError == False:
                    raise IxNetRestApiException('http POST error: {0}\n'.format(response.text))

            return response
        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http POST error: {0}\n'.format(errMsg)) 
Example 57
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilderRestApi.py    MIT License 5 votes vote down vote up
def delete(self, restApi, data={}, headers=None):
        """
        Description
           A HTTP DELETE function to delete the session.
           For Linux API server only.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           headers: The header to use for the URL.
        """

        if headers != None:
            self.jsonHeader = headers

        print('\nDELETE:', restApi)
        print('DATA:', data)
        print('HEADERS:', self.jsonHeader)
        try:
            response = requests.delete(restApi, data=json.dumps(data), headers=self.jsonHeader)
            print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                raise IxNetRestApiException('http DELETE error: {0}\n'.format(response.text))
            return response
        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http DELETE error: {0}\n'.format(errMsg)) 
Example 58
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilderRestApi.py    MIT License 5 votes vote down vote up
def get(self, restApi, data={}, stream=False, silentMode=False, ignoreError=False):
        """
        Description
           A HTTP GET function to send REST APIs.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           silentMode: True or False.  To display URL, data and header info.
           ignoreError: True or False.  If False, the response will be returned.
        """
        if silentMode is False:
            print('\nGET:', restApi)
            print('HEADERS:', self.jsonHeader)

        try:
            response = requests.get(restApi, headers=self.jsonHeader)

            if silentMode is False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                if ignoreError == False:
                    raise IxNetRestApiException('http GET error:{0}\n'.format(response.text))
            return response

        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http GET error: {0}\n'.format(errMsg)) 
Example 59
Project: IxNetwork   Author: OpenIxia   File: ixVmChassisBuilderRestApi.py    MIT License 5 votes vote down vote up
def post(self, restApi, data={}, headers=None, silentMode=False, ignoreError=False):
        """
        Description
           A HTTP POST function to mainly used to create or start operations.
        
        Parameters
           restApi: The REST API URL.
           data: The data payload for the URL.
           headers: The special header to use for the URL.
           silentMode: True or False.  To display URL, data and header info.
           noDataJsonDumps: True or False. If True, use json dumps. Else, accept the data as-is. 
           ignoreError: True or False.  If False, the response will be returned. No exception will be raised.
        """

        if headers != None:
            originalJsonHeader = self.jsonHeader
            self.jsonHeader = headers

        data = json.dumps(data)

        print('\nPOST:', restApi)
        if silentMode == False:
            print('DATA:', data)
            print('HEADERS:', self.jsonHeader)

        try:
            response = requests.post(restApi, data=data, headers=self.jsonHeader)
            # 200 or 201
            if silentMode == False:
                print('STATUS CODE:', response.status_code)
            if not re.match('2[0-9][0-9]', str(response.status_code)):
                if ignoreError == False:
                    raise IxNetRestApiException('http POST error: {0}\n'.format(response.text))

            return response
        except requests.exceptions.RequestException as errMsg:
            raise IxNetRestApiException('http POST error: {0}\n'.format(errMsg)) 
Example 60
Project: openeo-python-client   Author: Open-EO   File: connection.py    Apache License 2.0 5 votes vote down vote up
def _handle_error_response(self, response):
        # TODO replace this with `_raise_api_error`
        if response.status_code == 502:
            from requests.exceptions import ProxyError
            raise ProxyError("The proxy returned an error, this could be due to a timeout.")
        else:
            message = None
            if response.headers['Content-Type'] == 'application/json':
                message = response.json().get('message', None)
            if message:
                message = response.text

            raise ConnectionAbortedError(message) 
Example 61
Project: PTBot   Author: whocares-openscene   File: pagecheck.py    GNU General Public License v3.0 5 votes vote down vote up
def isup(text):
    """<url> - uses isup.me to check if <url> is online or offline
    :type text: str
    """
    url = text.strip()

    # slightly overcomplicated, esoteric URL parsing
    scheme, auth, path, query, fragment = urllib.parse.urlsplit(url)

    domain = auth or path

    try:
        response = requests.get('http://isup.me/' + domain)
    except requests.exceptions.ConnectionError:
        return "Failed to get status."
    if response.status_code != requests.codes.ok:
        return "Failed to get status."

    soup = BeautifulSoup(response.text, 'lxml')

    content = soup.find('div').text.strip()

    if "not just you" in content:
        return "It's not just you. {} looks \x02\x034down\x02\x0f from here!".format(url)
    elif "is up" in content:
        return "It's just you. {} is \x02\x033up\x02\x0f.".format(url)
    else:
        return "Huh? That doesn't look like a site on the interweb." 
Example 62
Project: PTBot   Author: whocares-openscene   File: fishbans.py    GNU General Public License v3.0 5 votes vote down vote up
def fishbans(text, bot):
    """<user> - gets information on <user>'s minecraft bans from fishbans"""
    user = text.strip()
    headers = {'User-Agent': bot.user_agent}

    try:
        request = requests.get(api_url.format(quote_plus(user)), headers=headers)
        request.raise_for_status()
    except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
        return "Could not fetch ban data from the Fishbans API: {}".format(e)

    try:
        json = request.json()
    except ValueError:
        return "Could not fetch ban data from the Fishbans API: Invalid Response"

    if not json["success"]:
        return "Could not fetch ban data for {}.".format(user)

    user_url = "http://fishbans.com/u/{}/".format(user)
    ban_count = json["stats"]["totalbans"]

    if ban_count == 1:
        return "The user \x02{}\x02 has \x021\x02 ban - {}".format(user, user_url)
    elif ban_count > 1:
        return "The user \x02{}\x02 has \x02{}\x02 bans - {}".format(user, ban_count, user_url)
    else:
        return "The user \x02{}\x02 has no bans - {}".format(user, user_url) 
Example 63
Project: PTBot   Author: whocares-openscene   File: fishbans.py    GNU General Public License v3.0 5 votes vote down vote up
def bancount(text, bot):
    """<user> - gets a count of <user>'s minecraft bans from fishbans"""
    user = text.strip()
    headers = {'User-Agent': bot.user_agent}

    try:
        request = requests.get(api_url.format(quote_plus(user)), headers=headers)
        request.raise_for_status()
    except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
        return "Could not fetch ban data from the Fishbans API: {}".format(e)

    try:
        json = request.json()
    except ValueError:
        return "Could not fetch ban data from the Fishbans API: Invalid Response"

    if not json["success"]:
        return "Could not fetch ban data for {}.".format(user)

    user_url = "http://fishbans.com/u/{}/".format(user)
    services = json["stats"]["service"]

    out = []
    for service, ban_count in list(services.items()):
        if ban_count != 0:
            out.append("{}: \x02{}\x02".format(service, ban_count))
        else:
            pass

    if not out:
        return "The user \x02{}\x02 has no bans - {}".format(user, user_url)
    else:
        return "Bans for \x02{}\x02: {} - {}".format(user, formatting.get_text_list(out, "and"), user_url) 
Example 64
Project: pycosio   Author: Accelize   File: test_storage_http.py    Apache License 2.0 5 votes vote down vote up
def test_handle_http_errors():
    """Test pycosio.http._handle_http_errors"""
    from pycosio.storage.http import _handle_http_errors
    from pycosio._core.exceptions import (
        ObjectNotFoundError, ObjectPermissionError)

    # Mocks response
    class Response:
        """Dummy response"""
        status_code = 200
        reason = 'reason'
        raised = False

        def raise_for_status(self):
            """Do nothing"""
            self.raised = True

    response = Response()

    # No error
    assert _handle_http_errors(response) is response

    # 403 error
    response.status_code = 403
    with pytest.raises(ObjectPermissionError):
        _handle_http_errors(response)

    # 404 error
    response.status_code = 404
    with pytest.raises(ObjectNotFoundError):
        _handle_http_errors(response)

    # Any error
    response.status_code = 500
    assert not response.raised
    _handle_http_errors(response)
    assert response.raised 
Example 65
Project: open-recipe   Author: dspray95   File: requests.py    The Unlicense 5 votes vote down vote up
def __call__(self, url, method='GET', body=None, headers=None,
                 timeout=None, **kwargs):
        """Make an HTTP request using requests.

        Args:
            url (str): The URI to be requested.
            method (str): The HTTP method to use for the request. Defaults
                to 'GET'.
            body (bytes): The payload / body in HTTP request.
            headers (Mapping[str, str]): Request headers.
            timeout (Optional[int]): The number of seconds to wait for a
                response from the server. If not specified or if None, the
                requests default timeout will be used.
            kwargs: Additional arguments passed through to the underlying
                requests :meth:`~requests.Session.request` method.

        Returns:
            google.auth.transport.Response: The HTTP response.

        Raises:
            google.auth.exceptions.TransportError: If any exception occurred.
        """
        try:
            _LOGGER.debug('Making request: %s %s', method, url)
            response = self.session.request(
                method, url, data=body, headers=headers, timeout=timeout,
                **kwargs)
            return _Response(response)
        except requests.exceptions.RequestException as caught_exc:
            new_exc = exceptions.TransportError(caught_exc)
            six.raise_from(new_exc, caught_exc) 
Example 66
Project: tcf   Author: intel   File: tc.py    Apache License 2.0 5 votes vote down vote up
def wait(self, regex_or_str, timeout = None, console = None):
        """
        Wait for a particular regex/string to be received on a given
        console of this target before a given timeout.

        See :py:meth:`expect` for a version that just raises
        exceptions when the output is not received.

        :param int timeout: Seconds to wait for regex_or_str to be
          received, raise :py:exc:`tcfl.tc.error_e`
          otherwise. If *False*, no timeout check is done; if *None*,
          it is taken from the default timeout set by the testcase.

        :param str console: (optional) name of console from which to
           receive the data

        :returns: *True* if the output was received before the
          timeout, *False* otherwise.
        """
        if timeout:
            assert isinstance(timeout, int)
        console = self.console._console_get(console)

        try:
            with self.on_console_rx_cm(regex_or_str, timeout = timeout,
                                       console = console, result = "pass"):
                # Run the expect loop
                self.testcase.tls.expecter.run()
        except pass_e:
            return True
        except error_e:
            return False 
Example 67
Project: tcf   Author: intel   File: tc.py    Apache License 2.0 5 votes vote down vote up
def from_exception(fn):
        """
        Call a phase function to translate exceptions into
        :class:`tcfl.tc.result_c` return codes.


        Passes through the return code, unless it is None, in which
        case we just return result_c(1, 0, 0, 0, 0)

        Note this function prints some more extra detail in case of
        fail/block/skip.
        it.
        """
        def _decorated_fn(*args, **kwargs):

            _tc = args[0] # The `self`  argument to the test case
            try:
                r = fn(*args, **kwargs)
                if r == None:
                    return result_c(1, 0, 0, 0, 0)
                else:
                    return r
            # Some exceptions that are common and we know about, so we
            # can print some more info that will be helpful
            except subprocess.CalledProcessError as e:
                return result_c.from_exception_cpe(_tc, e)
            except OSError as e:
                attachments = dict(
                    errno = e.errno,
                    strerror = e.strerror
                )
                if e.filename:
                    attachments['filename'] = e.filename
                return result_c.report_from_exception(_tc, e, attachments)
            except Exception as e:
                return result_c.report_from_exception(_tc, e)

        return _decorated_fn 
Example 68
Project: tcf   Author: intel   File: tc.py    Apache License 2.0 5 votes vote down vote up
def wait(self, regex_or_str, timeout = None, console = None):
        """
        Wait for a particular regex/string to be received on a given
        console of this target before a given timeout.

        See :py:meth:`expect` for a version that just raises
        exceptions when the output is not received.

        :param int timeout: Seconds to wait for regex_or_str to be
          received, raise :py:exc:`tcfl.tc.error_e`
          otherwise. If *False*, no timeout check is done; if *None*,
          it is taken from the default timeout set by the testcase.

        :param str console: (optional) name of console from which to
           receive the data

        :returns: *True* if the output was received before the
          timeout, *False* otherwise.
        """
        if timeout:
            assert isinstance(timeout, int)
        console = self.console._console_get(console)

        try:
            with self.on_console_rx_cm(regex_or_str, timeout = timeout,
                                       console = console, result = "pass"):
                # Run the expect loop
                self.testcase.tls.expecter.run()
        except pass_e:
            return True
        except error_e:
            return False 
Example 69
Project: datadog-unix-agent   Author: DataDog   File: agent.py    Apache License 2.0 5 votes vote down vote up
def status(cls, config, to_screen=True):
        status = {}
        rendered = None
        api_addr = config['api']['bind_host']
        api_port = config['api']['port']

        target = 'http://{host}:{port}/status'.format(host=api_addr, port=api_port)
        try:
            r = requests.get(target, timeout=cls.STATUS_TIMEOUT)
            r.raise_for_status()

            status = r.json()
        except requests.exceptions.HTTPError as e:
            log.error("HTTP error collecting agent status: %s", e)
        except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
            log.error("Problem connecting or connection timed out, is the agent up? Error: %s", e)
        except ValueError as e:
            log.error("There was a problem unmarshaling JSON response: %s", e)

        if status:
            here = os.path.dirname(os.path.realpath(__file__))
            templates = os.path.join(here, 'templates')
            template_env = Environment(loader=FileSystemLoader(templates))
            template = template_env.get_template('status.jinja')
            rendered = template.render(version=AGENT_VERSION, status=status)
            if to_screen:
                print(rendered)

        return rendered 
Example 70
Project: gro-controller   Author: OpenAgricultureFoundation   File: server.py    GNU General Public License v2.0 5 votes vote down vote up
def _getJsonWithRetry(self, url):
        """Private method to wrap getting json data with retries. Only gets single page, nothing fancy. See getJson
        :param url: url to get
        :return: json data returned from server
        """
        retry_count = 0
        req = None
        while retry_count < self._max_retries:
            try:
                #val = 'Token ' + self._token
                headers = {'Authorization': 'Token ' + self._token} 
                req = requests.get(url, timeout=self._req_timeout, headers=headers)
                if req.status_code == requests.codes.ok:
                    break
                logging.warning('Failed to get %s, status %d, retry %d' % (url, req.status_code, retry_count))
            except requests.exceptions.RequestException as e:
                logging.warning('Failed to get request for %s, RequestException: %s' % (url, e))
                pass        # Just pass it, we will include it as a retry ahead
            finally:
                retry_count += 1

        if retry_count >= self._max_retries:
            logging.error("Exceeded max connection retries for %s" % url)
            if req is not None:
                logging.error("Request failure reason: %s" % req.reason)
                raise ConnectionError(req.reason)
            else:
                logging.error("No request, no reason!")
                raise ConnectionError

        return req.json()

    # TODO check documentation 
Example 71
Project: rowgenerators   Author: Metatab   File: download.py    MIT License 4 votes vote down vote up
def download(self, url):
        from os.path import abspath, join
        from genericpath import exists

        from rowgenerators.appurl.url import Url
        from rowgenerators.exceptions import DownloadError, AccessError

        # logger.debug(f"Download {url}")

        working_dir = self.working_dir if self.working_dir else ''

        r = Resource()

        # For local files, don't download, just reference in place.
        if url.scheme == 'file':
            r.cache_path = Url(url.resource_url).path
            r.download_time = None

            # Many places the file may exist
            locations = {  # What a mess ...
                abspath(r.cache_path),
                abspath(r.cache_path.lstrip('/')),
                abspath(join(working_dir, r.cache_path)),
                abspath(r.cache_path.lstrip('/'))
            }

            for l in locations:
                if exists(l):
                    r.sys_path = l
                    logger.debug("Found '{}'as local file '{}'".format(str(url), l))
                    break
            else:
                raise DownloadError(("File resource does not exist. Found none of:"
                                     "\n{}\n\nWorking dir = {}\ncache_path={}\nspec_path={}")
                                    .format('\n'.join(locations), working_dir, r.cache_path, url.path))

        else:
            # Not a local file, so actually need to download it.
            try:
                r.cache_path, r.download_time = self._download_with_lock(url.resource_url)
            except AccessError as e:
                # Try again, using a URL that we may have configured an account for. This is
                # primarily S3 urls, with Boto or AWS credential
                try:
                    r.cache_path, r.download_time = self._download_with_lock(url.auth_resource_url)
                except AttributeError as e:
                    raise e
                except DownloadError as e:
                    raise DownloadError("Access error for url '{}'; also tried accessing as S3 url '{}'".format(url, url.auth_resource_url ))

            r.sys_path = self.cache.getsyspath(r.cache_path)

        return r 
Example 72
Project: django-oidc-rp   Author: impak-finance   File: middleware.py    MIT License 4 votes vote down vote up
def refresh_token(self, request):
        """ Refreshes the token of the current user. """
        # NOTE: no refresh token in the session means that the user wasn't authentified using the
        # OpenID Connect provider (OP).
        refresh_token = request.session.get('oidc_auth_refresh_token')
        if refresh_token is None:
            return

        id_token_exp_timestamp = request.session.get('oidc_auth_id_token_exp_timestamp', None)
        now_timestamp = time.time()
        # Returns immediatelly if the token is still valid.
        if id_token_exp_timestamp is not None and id_token_exp_timestamp > now_timestamp:
            return

        # Prepares the token payload that will be used to request a new token from the token
        # endpoint.
        refresh_token = request.session.pop('oidc_auth_refresh_token')
        token_payload = {
            'client_id': oidc_rp_settings.CLIENT_ID,
            'client_secret': oidc_rp_settings.CLIENT_SECRET,
            'grant_type': 'refresh_token',
            'refresh_token': refresh_token,
        }

        # Calls the token endpoint.
        token_response = requests.post(oidc_rp_settings.PROVIDER_TOKEN_ENDPOINT, data=token_payload)
        try:
            token_response.raise_for_status()
        except requests.exceptions.HTTPError:
            auth.logout(request)
            return
        token_response_data = token_response.json()

        # Validates the token.
        raw_id_token = token_response_data.get('id_token')
        id_token = validate_and_return_id_token(raw_id_token, validate_nonce=False)

        # If the token cannot be validated we have to log out the current user.
        if id_token is None:
            auth.logout(request)
            return

        # Retrieves the access token and refresh token.
        access_token = token_response_data.get('access_token')
        refresh_token = token_response_data.get('refresh_token')

        # Stores the ID token, the related access token and the refresh token in the session.
        request.session['oidc_auth_id_token'] = raw_id_token
        request.session['oidc_auth_access_token'] = access_token
        request.session['oidc_auth_refresh_token'] = refresh_token

        # Saves the new expiration timestamp.
        request.session['oidc_auth_id_token_exp_timestamp'] = \
            time.time() + oidc_rp_settings.ID_TOKEN_MAX_AGE 
Example 73
Project: IxNetwork   Author: OpenIxia   File: IxNetRestApi.py    MIT License 4 votes vote down vote up
def get(self, restApi, data={}, stream=False, silentMode=False, ignoreError=False, maxRetries=5):
        """
        Description
            A HTTP GET function to send REST APIs.

        Parameters
           restApi: (str): The REST API URL.
           data: (dict): The data payload for the URL.
           silentMode: (bool):  To display on stdout: URL, data and header info.
           ignoreError: (bool): True: Don't raise an exception.  False: The response will be returned.
           maxRetries: <int>: The maximum amount of GET retries before declaring as server connection failure.

        Syntax
            /api/v1/sessions/1/ixnetwork/operations
        """
        retryInterval = 3
        restExecutionFailures = 0
        while True:
            if silentMode is False:
                self.logInfo('\n\tGET: {0}'.format(restApi))

            try:
                # For binary file
                if stream:
                    response = self._session.request('GET', restApi, stream=True, headers=self.jsonHeader, allow_redirects=True, verify=self.verifySslCert)

                if stream == False:
                    response = self._session.request('GET', restApi, headers=self.jsonHeader, allow_redirects=True, verify=self.verifySslCert)

                if silentMode is False:
                    for redirectStatus in response.history:
                        if '307' in str(response.history):
                            self.logInfo('\t{0}: {1}'.format(redirectStatus, response.url), timestamp=False)

                    self.logInfo('\tSTATUS CODE: {0}'.format(response.status_code), timestamp=False)

                if not str(response.status_code).startswith('2'):
                    if ignoreError == False:
                        if 'message' in response.json() and response.json()['messsage'] != None:
                            self.logWarning('\n%s' % response.json()['message'])

                        errMsg = 'GET Exception error: {0}'.format(response.text)
                        raise IxNetRestApiException(errMsg)

                return response

            except (requests.exceptions.RequestException, Exception) as errMsg:
                errMsg = 'GET Exception error {]/{} retries: {}'.format(restExecutionFailures, maxRetries, errMsg)

                if restExecutionFailures < maxRetries:
                    self.logError(errMsg)
                    restExecutionFailures += 1
                    time.sleep(retryInterval)
                    continue
                
                if restExecutionFailures == maxRetries:
                    raise IxNetRestApiException(errMsg) 
Example 74
Project: IxNetwork   Author: OpenIxia   File: IxNetRestApi.py    MIT License 4 votes vote down vote up
def options(self, restApi, data={}, silentMode=False, ignoreError=False, maxRetries=5):
        """
        Description
            A HTTP OPTIONS function to send REST APIs.

        Parameters
           restApi: (str): The REST API URL.
           silentMode: (bool):  To display on stdout: URL, data and header info.
           ignoreError: (bool): True: Don't raise an exception.  False: The response will be returned.
           maxRetries: <int>: The maximum amount of GET retries before declaring as server connection failu
        """
        retryInterval = 3
        restExecutionFailures = 0
        while True:
            if silentMode is False:
                self.logInfo('\n\tOPTIONS: {0}'.format(restApi))

            try:
                # For binary file
                response = self._session.request('OPTIONS', restApi, headers=self.jsonHeader, allow_redirects=True, verify=self.verifySslCert)

                if silentMode is False:
                    for redirectStatus in response.history:
                        if '307' in str(response.history):
                            self.logInfo('\t{0}: {1}'.format(redirectStatus, response.url), timestamp=False)

                    self.logInfo('\tSTATUS CODE: {0}'.format(response.status_code), timestamp=False)

                if not str(response.status_code).startswith('2'):
                    if ignoreError == False:
                        if 'message' in response.json() and response.json()['messsage'] != None:
                            self.logWarning('\n%s' % response.json()['message'])

                        errMsg = 'OPTIONS Exception error: {0}'.format(response.text)
                        raise IxNetRestApiException(errMsg)
                return response

            except (requests.exceptions.RequestException, Exception) as errMsg:
                errMsg = 'OPTIONS Exception error {}/{} retries: {}'.format(restExecutionFailures, maxRetries, errMsg)

                if restExecutionFailures < maxRetries:
                    self.logError(errMsg)
                    restExecutionFailures += 1
                    time.sleep(retryInterval)
                    continue
                
                if restExecutionFailures == maxRetries:
                    raise IxNetRestApiException(errMsg) 
Example 75
Project: IxNetwork   Author: OpenIxia   File: IxNetRestApi.py    MIT License 4 votes vote down vote up
def delete(self, restApi, data={}, headers=None, maxRetries=5):
        """
        Description
           A HTTP DELETE function to delete the session.
           For Linux and Windows Connection Mgr API server only.

        Paramters
           restApi: (str): The REST API URL.
           data: (dict): The data payload for the URL.
           headers: (str): The headers to use for the URL.
           maxRetries: <int>: The maximum amount of GET retries before declaring as server connection failure.
        """
        if headers != None:
            self.jsonHeader = headers
            
        retryInterval = 3
        restExecutionFailures = 0
        while True:
            self.logInfo('\n\tDELETE: {0}\n\tDATA: {1}'.format(restApi, data))

            try:
                response = self._session.request('DELETE', restApi, data=json.dumps(data), headers=self.jsonHeader, allow_redirects=True, 
                                                 verify=self.verifySslCert)

                for redirectStatus in response.history:
                    if '307' in str(response.history):
                        self.logInfo('\t{0}: {1}'.format(redirectStatus, response.url), timestamp=False)

                self.logInfo('\tSTATUS CODE: %s' % response.status_code, timestamp=False)
                if not str(response.status_code).startswith('2'):
                    self.showErrorMessage()
                    errMsg = 'DELETE Exception error: {0}\n'.format(response.text)
                    self.logError(errMsg)
                    raise IxNetRestApiException(errMsg)
                return response

            except (requests.exceptions.RequestException, Exception) as errMsg:
                errMsg = 'DELETE Exception error {}/{} retries: {}\n'.format(restExecutionFailures, maxRetries, errMsg)

                if restExecutionFailures < maxRetries:
                    self.logError(errMsg)
                    restExecutionFailures += 1
                    time.sleep(retryInterval)
                    continue
                
                if restExecutionFailures == maxRetries:
                    raise IxNetRestApiException(errMsg) 
Example 76
Project: tcf   Author: intel   File: tc.py    Apache License 2.0 4 votes vote down vote up
def __method_trampoline_call(self, fname, fn, _type, targets):
        # runs a class function and return a return_c
        #
        # convers the return value to return_c from bool, or None, or
        # passes a return_c. Passes exceptions.
        #
        # We use fname instead of fn.__name__ because if we have made
        # an alias out of the method in the class (with setattr(), we
        # want to see the alias name, not the original function name).
        if fname == fn.__name__:
            self.report_info("running %s.%s()"
                             % (type(self).__name__, fn.__name__),
                             dlevel = 3)
        else:
            self.report_info("running %s.%s() [App builder's alias for "
                             "%s() which calls tc_c._*_50_for_target()]"
                             % (type(self).__name__, fname, fn.__name__),
                             dlevel = 3)
        if type(fn) == types.MethodType:	# instance needed
            if fn.im_self == None:
                # This was added from target_wanted_add(), binding to an
                # specific object; we forced it unbound, so we have to
                # call it with 'self'
                r = getattr(self, fname)(self, *targets)
            else:
                # This was added from the @target() decorator, binding
                # to a class, it doesn't need a self as Python will
                # add it for us, current self.
                r = getattr(self, fname)(*targets)
        else:	# static/classmethod
            r = fn(*targets)

        # Now, let's see what did it return
        if isinstance(r, result_c):
            return r
        elif r == None or r == True:
            return result_c(1, 0, 0, 0, 0)
        elif r == False:
            return result_c(0, 1, 0, 0, 0)
        else:
            raise blocked_e(
                "%s.%s(): don't know what to do with "
                "return value of type %s; return nothing or "
                "True for success, False for failure, raise "
                "any exception for blockage, or "
                "tcfl.{passed|blocked|error|failed|skip}_e "
                % (type(self).__name__, fn.__name__,
                   type(r).__name)) 
Example 77
Project: tcf   Author: intel   File: tc.py    Apache License 2.0 4 votes vote down vote up
def _methods_run(self, do_serially, serial_list, parallel_list):
        """
        Implement the guts of _methods_call()

        Runs and aggregates results of serial methods, then launches a
        threadpool for the parallel methods; aggregates their results
        and raise exception if any happened.

        :returns result_c: aggregated result of all the methods.
        """
        result = result_c(0, 0, 0, 0, 0)
        if do_serially:
            for fname, fn, _type, args in sorted(serial_list + parallel_list):
                retval = self._method_run(fname, fn, _type, args)
                result += retval
                if retval.errors or retval.failed \
                   or retval.blocked or retval.skipped:
                    return result
        else:
            for fname, fn, _type, args in serial_list:
                retval = self._method_run(fname, fn, _type, args)
                result += retval
                if retval.errors or retval.failed \
                   or retval.blocked or retval.skipped:
                    return result
            # FIXME: set from from config
            # All these are supposedly I/O bound jobs as either they
            # are chitchatting with the network or spawning jobs to do
            # stuff
            thread_pool = _multiprocessing_method_pool_c(processes = 10)
            threads = {}
            for fname, fn, _type, args in parallel_list:
                targets = self._mk_target_args_for_fn(fn, args)
                threads[fname] = thread_pool.apply_async(
                    self.__method_trampoline_thread,
                    (msgid_c(l = 2), fname, fn, _type, targets,
                     dict(expecter_parent = self.tls.expecter)))
                # so we can Ctrl-C right away--the system is designed
                # to cleanup top bottom, with everything being
                # expendable
                threads[fname].daemon = True
            thread_pool.close()
            thread_pool.join()
            for thread in threads.values():
                r = thread.get()
                if r[1] != None:	# re-raise thrown exceptions
                    raise r[1][0], r[1][1], r[1][2]
                result += r[0]
            del thread_pool
        return result 
Example 78
Project: tcf   Author: intel   File: tc.py    Apache License 2.0 4 votes vote down vote up
def __method_trampoline_call(self, fname, fn, _type, targets):
        # runs a class function and return a return_c
        #
        # convers the return value to return_c from bool, or None, or
        # passes a return_c. Passes exceptions.
        #
        # We use fname instead of fn.__name__ because if we have made
        # an alias out of the method in the class (with setattr(), we
        # want to see the alias name, not the original function name).
        if fname == fn.__name__:
            self.report_info("running %s.%s()"
                             % (type(self).__name__, fn.__name__),
                             dlevel = 3)
        else:
            self.report_info("running %s.%s() [App builder's alias for "
                             "%s() which calls tc_c._*_50_for_target()]"
                             % (type(self).__name__, fname, fn.__name__),
                             dlevel = 3)
        if type(fn) == types.MethodType:	# instance needed
            if fn.im_self == None:
                # This was added from target_wanted_add(), binding to an
                # specific object; we forced it unbound, so we have to
                # call it with 'self'
                r = getattr(self, fname)(self, *targets)
            else:
                # This was added from the @target() decorator, binding
                # to a class, it doesn't need a self as Python will
                # add it for us, current self.
                r = getattr(self, fname)(*targets)
        else:	# static/classmethod
            r = fn(*targets)

        # Now, let's see what did it return
        if isinstance(r, result_c):
            return r
        elif r == None or r == True:
            return result_c(1, 0, 0, 0, 0)
        elif r == False:
            return result_c(0, 1, 0, 0, 0)
        else:
            raise blocked_e(
                "%s.%s(): don't know what to do with "
                "return value of type %s; return nothing or "
                "True for success, False for failure, raise "
                "any exception for blockage, or "
                "tcfl.{passed|blocked|error|failed|skip}_e "
                % (type(self).__name__, fn.__name__,
                   type(r).__name)) 
Example 79
Project: tcf   Author: intel   File: tc.py    Apache License 2.0 4 votes vote down vote up
def _methods_run(self, do_serially, serial_list, parallel_list):
        """
        Implement the guts of _methods_call()

        Runs and aggregates results of serial methods, then launches a
        threadpool for the parallel methods; aggregates their results
        and raise exception if any happened.

        :returns result_c: aggregated result of all the methods.
        """
        result = result_c(0, 0, 0, 0, 0)
        if do_serially:
            for fname, fn, _type, args in sorted(serial_list + parallel_list):
                retval = self._method_run(fname, fn, _type, args)
                result += retval
                if retval.errors or retval.failed \
                   or retval.blocked or retval.skipped:
                    return result
        else:
            for fname, fn, _type, args in serial_list:
                retval = self._method_run(fname, fn, _type, args)
                result += retval
                if retval.errors or retval.failed \
                   or retval.blocked or retval.skipped:
                    return result
            # FIXME: set from from config
            # All these are supposedly I/O bound jobs as either they
            # are chitchatting with the network or spawning jobs to do
            # stuff
            thread_pool = _multiprocessing_method_pool_c(processes = 10)
            threads = {}
            for fname, fn, _type, args in parallel_list:
                targets = self._mk_target_args_for_fn(fn, args)
                threads[fname] = thread_pool.apply_async(
                    self.__method_trampoline_thread,
                    (msgid_c(l = 2), fname, fn, _type, targets,
                     dict(expecter_parent = self.tls.expecter)))
                # so we can Ctrl-C right away--the system is designed
                # to cleanup top bottom, with everything being
                # expendable
                threads[fname].daemon = True
            thread_pool.close()
            thread_pool.join()
            for thread in threads.values():
                r = thread.get()
                if r[1] != None:	# re-raise thrown exceptions
                    raise r[1][0], r[1][1], r[1][2]
                result += r[0]
            del thread_pool
        return result 
Example 80
Project: dpa-pipe   Author: Clemson-DPA   File: client.py    MIT License 4 votes vote down vote up
def execute_request_url(self, http_method, url, data=None, params=None,
        headers=None):

        data_format = self.data_format

        if params:
            params = self._sanitize_params(params)

        if data:
            data = json.dumps(data)

        if data and not headers:
            if data_format == 'json':
                headers = {'content-type': 'application/json'}
            else:
                raise RestfulClientError("Unknown data format: " + data_format)

        # requests method based on the supplied http method name.
        # it *should* be a 1 to 1 lc mapping. (GET==get, PUT==put, etc.)
        requests_method_name = http_method.lower()

        # see if the requests api has a method that matches
        try:
            requests_method = getattr(requests, requests_method_name)
        except AttributeError:
            raise RestfulClientError(
                "Unknown method for requests: " + str(requests_method_name))

        # execute the request
        response = self._try_request(requests_method, url, params=params,
            data=data, headers=headers)

        # raise a custom exception if 400/500 error
        try:
            response.raise_for_status()
        except requests.exceptions.HTTPError as e:
            raise RestfulClientError(e.response.text)

        # deserialize the returned data. may want to flesh this out at some
        # point to allow for additional data formats
        if data_format == 'json':
            # json.loads(response.content) results in unicode instead of
            # str. using yaml results in string values. so using that for now.
            return yaml.load(response.content)
        else:
            raise RestfulClientError("Unknown data format: " + data_format)

    # -------------------------------------------------------------------------
    # Private class methods:
    # -------------------------------------------------------------------------