Python requests.exceptions() Examples
The following are code examples for showing how to use requests.exceptions(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.
Example 1
| Project: MangaScrapper Author: AnimeshShaw File: mangascrapper.py Apache License 2.0 | 7 votes |
|
def _set_response_ins_(self, pageurl):
"""
Sets the response for the GET request of pageurl and stores it in self.resp
:param pageurl: url for which we store the response.
"""
try:
s = requests.Session()
a = requests.adapters.HTTPAdapter(max_retries=5)
s.mount('http://', a)
resp = s.get(pageurl, timeout=30)
self.__resp_obj__ = resp
resp.close()
except requests.exceptions.Timeout:
logging.error("\tVery Slow Internet Connection.")
except requests.exceptions.ConnectionError:
logging.error("\tNetwork Unavailable. Check your connection.")
except requests.exceptions.MissingSchema:
logging.error("\t503 Service Unavailable. Retrying download ... ")
Example 2
| Project: fishroom Author: tuna File: gitter.py GNU General Public License v3.0 | 6 votes |
|
def _must_post(self, api, data=None, json=None, timeout=10, **kwargs):
if data is not None:
kwargs['data'] = data
elif json is not None:
kwargs['json'] = json
else:
kwargs['data'] = {}
kwargs['timeout'] = timeout
try:
r = requests.post(api, **kwargs)
return r
except requests.exceptions.Timeout:
logger.error("Timeout requesting Gitter")
except KeyboardInterrupt:
raise
except:
logger.exception("Unknown error requesting Gitter")
return None
Example 3
| Project: fishroom Author: tuna File: photostore.py GNU General Public License v3.0 | 6 votes |
|
def upload_image(self, filename=None, filedata=None, **kwargs) -> str:
if filedata is None:
files = {"image": open(filename, 'rb')}
else:
files = {"image": filedata}
try:
r = requests.post(self.url, files=files, timeout=5)
except requests.exceptions.Timeout:
logger.error("Timeout uploading to VimCN")
return None
except:
logger.exception("Unknown errror uploading to VimCN")
return None
if not r.ok:
return None
return r.text.strip()
Example 4
| Project: controller Author: deis File: __init__.py MIT License | 6 votes |
|
def http_head(self, path, **kwargs):
"""
Make a HEAD request to the k8s server.
"""
try:
url = urljoin(self.url, path)
response = self.session.head(url, **kwargs)
except requests.exceptions.ConnectionError as err:
# reraise as KubeException, but log stacktrace.
message = "There was a problem retrieving headers from " \
"the Kubernetes API server. URL: {}".format(url)
logger.error(message)
raise KubeException(message) from err
return response
Example 5
| Project: controller Author: deis File: __init__.py MIT License | 6 votes |
|
def http_post(self, path, data=None, json=None, **kwargs):
"""
Make a POST request to the k8s server.
"""
try:
url = urljoin(self.url, path)
response = self.session.post(url, data=data, json=json, **kwargs)
except requests.exceptions.ConnectionError as err:
# reraise as KubeException, but log stacktrace.
message = "There was a problem posting data to " \
"the Kubernetes API server. URL: {}, " \
"data: {}, json: {}".format(url, data, json)
logger.error(message)
raise KubeException(message) from err
return response
Example 6
| Project: controller Author: deis File: __init__.py MIT License | 6 votes |
|
def http_put(self, path, data=None, **kwargs):
"""
Make a PUT request to the k8s server.
"""
try:
url = urljoin(self.url, path)
response = self.session.put(url, data=data, **kwargs)
except requests.exceptions.ConnectionError as err:
# reraise as KubeException, but log stacktrace.
message = "There was a problem putting data to " \
"the Kubernetes API server. URL: {}, " \
"data: {}".format(url, data)
logger.error(message)
raise KubeException(message) from err
return response
Example 7
| Project: cis Author: mozilla-iam File: common.py Mozilla Public License 2.0 | 6 votes |
|
def _load_publisher_rules(self, rules_url):
"""
Get the CIS integration rules
"""
rules = None
if not self.always_use_local_file:
if rules_url is not None:
try:
r = requests.get(rules_url)
rules = r.json()
except (json.JSONDecodeError, requests.exceptions.ConnectionError) as e:
logger.debug("Failed to load rules data from rules_url {} ({})".format(rules_url, e))
# Fall-back to built-in copy
if self.always_use_local_file or rules is None:
rules_file = "data/well-known/mozilla-iam-publisher-rules"
if not os.path.isfile(rules_file):
dirname = os.path.dirname(os.path.realpath(__file__))
path = dirname + "/" + rules_file
else:
path = rules_file
rules = json.load(open(path))
return rules
Example 8
| Project: adh6 Author: bonnetn File: auth.py GNU General Public License v3.0 | 6 votes |
|
def get_sso_groups(token):
try:
verify_cert = True
if is_development_environment():
verify_cert = False
headers = {"Authorization": "Bearer " + token}
r = requests.get(
current_app.config["AUTH_SERVER_ADDRESS"],
headers=headers,
timeout=1,
verify=verify_cert
)
except requests.exceptions.ReadTimeout:
return None
if r.status_code != 200 or "id" not in r.json():
return None
result = r.json()
if is_development_environment():
result["groups"] = [ADH6_USER, ADH6_ADMIN] # If we are testing, consider the user asg.admin
return result
Example 9
| Project: pyfgt Author: p4r4n0y1ng File: fortigate.py Apache License 2.0 | 6 votes |
|
def __init__(self, host, user=None, passwd=None, debug=False, use_ssl=True, verify_ssl=False, timeout=300,
disable_request_warnings=False, apikey=None):
super(FortiGate, self).__init__()
self._host = host
self._user = user
self._req_id = 0
self._url = None
self._session = None
self._sid = None
self._timeout = timeout
self._debug = debug
self._use_ssl = use_ssl
self._verify_ssl = verify_ssl
self._apikeyused = True if passwd is None and apikey is not None else False
self._passwd = passwd if passwd is not None else apikey
self._req_resp_object = RequestResponse()
self._logger = None
if disable_request_warnings:
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
Example 10
| Project: pydkron Author: oldmantaiter File: client.py MIT License | 6 votes |
|
def _call(self, method, endpoint, payload=None):
"""
Call the endpoint and return the response
"""
resp = None
shuffle(self.hosts)
for index in xrange(len(self.hosts)):
url = "http://%s/v1%s" % (self.hosts[index], endpoint)
req_args = {'timeout': 10, 'headers': {'Content-Type': 'application/json'}}
try:
if method == _GET:
resp = requests.get(url, **req_args)
elif method == _POST:
resp = requests.post(
url, data=payload, **req_args)
elif method == _DELETE:
resp = requests.delete(url, **req_args)
break
except requests.exceptions.RequestException:
continue
if resp is None:
raise DkronClientException("No valid host found")
return resp
Example 11
| Project: aiolocust Author: kpidata File: clients.py MIT License | 6 votes |
|
def __exit__(self, exc, value, traceback):
if self._is_reported:
# if the user has already manually marked this response as failure or success
# we can ignore the default haviour of letting the response code determine the outcome
return exc is None
if exc:
if isinstance(value, ResponseError):
self.failure(value)
else:
return False
else:
try:
self.raise_for_status()
except requests.exceptions.RequestException as e:
self.failure(e)
else:
self.success()
return True
Example 12
| Project: urllib-requests-adapter Author: Matrixcoffee File: __init__.py Apache License 2.0 | 6 votes |
|
def execute(self):
timeout = self.timeout_seconds
if timeout is None: timeout = GLOBAL_TIMEOUT_SECONDS
try:
if timeout is None:
r = urllib.request.urlopen(self.rq, **_get_tls_parms())
else:
r = urllib.request.urlopen(self.rq, None, timeout, **_get_tls_parms())
except urllib.error.HTTPError as e:
self.status_code = e.code
self.text = e.msg
return self
except (http.client.HTTPException, urllib.error.URLError, socket.timeout) as e:
e2 = exceptions.RequestException("{}.{!r}".format(type(e).__module__, e))
raise e2
self._rdata = r.read()
if DEBUG: print("Response._rdata set to:", repr(self._rdata), file=sys.stderr)
self.map_request(r)
r.close()
return self
Example 13
| Project: codechef-cli Author: sk364 File: helpers.py GNU General Public License v3.0 | 6 votes |
|
def request(session, method, url, **kwargs):
"""
:desc: Custom wrapper method to add a timeout message
when there is a `requests.exceptions.ConnectionError`
or `requests.ReadTimeout` exception.
:param: `session` requests.Session object
`method` HTTP method to use
`url` name of the URL
:return: requests.Response object.
"""
try:
return session.request(method=method, url=url, timeout=(5, 5), **kwargs)
except (ConnectionError, ReadTimeout):
print(INTERNET_DOWN_MSG)
sys.exit(1)
Example 14
| Project: python-wsmanclient Author: intelsdi-x File: wsman.py Apache License 2.0 | 6 votes |
|
def __init__(self, endpoint, resource_uri, optimization=True,
max_elems=100, filter_query=None, filter_dialect=None):
self.endpoint = endpoint
self.resource_uri = resource_uri
self.filter_dialect = None
self.filter_query = None
self.optimization = optimization
self.max_elems = max_elems
if filter_query is not None:
try:
self.filter_dialect = FILTER_DIALECT_MAP[filter_dialect]
except KeyError:
valid_opts = ', '.join(FILTER_DIALECT_MAP)
raise exceptions.WSManInvalidFilterDialect(
invalid_filter=filter_dialect, supported=valid_opts)
self.filter_query = filter_query
Example 15
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilder.py MIT License | 6 votes |
|
def patch(self, restApi, data={}, silentMode=False):
"""
Description
A HTTP PATCH function to modify configurations.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
silentMode: True or False. To display URL, data and header info.
"""
if silentMode == False:
print('\nPATCH:', restApi)
print('DATA:', data)
print('HEADERS:', self.jsonHeader)
try:
response = requests.patch(restApi, data=json.dumps(data), headers=self.jsonHeader)
if silentMode == False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
print('\nPatch error:')
raise IxNetRestApiException('http PATCH error: {0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http PATCH error: {0}\n'.format(errMsg))
Example 16
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilderRestApi.py MIT License | 6 votes |
|
def patch(self, restApi, data={}, silentMode=False):
"""
Description
A HTTP PATCH function to modify configurations.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
silentMode: True or False. To display URL, data and header info.
"""
if silentMode == False:
print('\nPATCH:', restApi)
print('DATA:', data)
print('HEADERS:', self.jsonHeader)
try:
response = requests.patch(restApi, data=json.dumps(data), headers=self.jsonHeader)
if silentMode == False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
print('\nPatch error:')
raise IxNetRestApiException('http PATCH error: {0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http PATCH error: {0}\n'.format(errMsg))
Example 17
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilderRestApi.py MIT License | 6 votes |
|
def patch(self, restApi, data={}, silentMode=False):
"""
Description
A HTTP PATCH function to modify configurations.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
silentMode: True or False. To display URL, data and header info.
"""
if silentMode == False:
print('\nPATCH:', restApi)
print('DATA:', data)
print('HEADERS:', self.jsonHeader)
try:
response = requests.patch(restApi, data=json.dumps(data), headers=self.jsonHeader)
if silentMode == False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
print('\nPatch error:')
raise IxNetRestApiException('http PATCH error: {0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http PATCH error: {0}\n'.format(errMsg))
Example 18
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilderRestApi.py MIT License | 6 votes |
|
def patch(self, restApi, data={}, silentMode=False):
"""
Description
A HTTP PATCH function to modify configurations.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
silentMode: True or False. To display URL, data and header info.
"""
if silentMode == False:
print('\nPATCH:', restApi)
print('DATA:', data)
print('HEADERS:', self.jsonHeader)
try:
response = requests.patch(restApi, data=json.dumps(data), headers=self.jsonHeader)
if silentMode == False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
print('\nPatch error:')
raise IxNetRestApiException('http PATCH error: {0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http PATCH error: {0}\n'.format(errMsg))
Example 19
| Project: PTBot Author: whocares-openscene File: pagecheck.py GNU General Public License v3.0 | 6 votes |
|
def down(text):
"""<url> - checks if <url> is online or offline
:type text: str
"""
if "://" not in text:
text = 'http://' + text
text = 'http://' + urllib.parse.urlparse(text).netloc
try:
requests.get(text)
except requests.exceptions.ConnectionError:
return '{} seems to be down'.format(text)
else:
return '{} seems to be up'.format(text)
Example 20
| Project: smbproxy Author: Seekscale File: fs_cache.py GNU General Public License v2.0 | 6 votes |
|
def register_operation_failure(self, v):
self.incr_counter('operation.failure')
return v
#
# The raw HTTP requests
#
# def _http_req(self, endpoint, **kwargs):
# url = 'http://%s:%d/%s' % (self.http_service_host, self.http_service_port, endpoint)
#
# # self.log.msg("[HTTP] %s %r" % (url, kwargs), level=logger.INFO)
#
# try:
# r = requests.post(url, **kwargs)
# except requests.exceptions.RequestException:
# self.log.msg("ERROR in HTTP request: %s" % get_traceback(), level=logger.INFO)
# else:
# if r.status_code != 200:
# self.log.msg("Bad status code in HTTP request: %d" % r.status_code, level=logger.INFO)
# return None
# else:
# return r
Example 21
| Project: esst Author: theendsofinvention File: __init__.py MIT License | 6 votes |
|
def wan_available(retry: int = 0):
"""
Returns: True if connected to WAN
"""
try:
response = requests.get('http://google.com', timeout=2)
commands.DCS.unblock_start('no WAN connection available')
commands.DISCORD.can_start()
return bool(response.ok)
except requests.exceptions.RequestException:
if retry < 5:
LOGGER.debug('Internet connection loss detected, retry %s', retry)
await asyncio.sleep(2)
result = await wan_available(retry + 1)
return result
LOGGER.debug(f'Internet connection loss detected, no more retry')
commands.DISCORD.cannot_start()
commands.DCS.block_start('no WAN connection available')
return False
Example 22
| Project: heliopy Author: heliopython File: cdasrest.py GNU General Public License v3.0 | 6 votes |
|
def get_cdas_url(starttime, endtime, vars, dataset, timeout=10):
dataview = 'sp_phys'
if vars is None:
try:
var_info = get_variables(dataset, timeout=timeout)
except requests.exceptions.ReadTimeout:
raise util.NoDataError(
'Connection to CDAweb timed out when getting CDAS URL for '
f'{dataset} data for interval {starttime} - {endtime}.')
if not len(var_info):
raise util.NoDataError(
f'No {dataset} data available for date {date}')
vars = [v['Name'] for v in var_info['VariableDescription']]
uri = '/'.join(['dataviews', dataview,
'datasets', dataset,
'data',
','.join([starttime.strftime('%Y%m%dT%H%M%SZ'),
endtime.strftime('%Y%m%dT%H%M%SZ')]),
','.join(vars)
])
url = '/'.join([CDAS_BASEURL, uri])
return url
Example 23
| Project: SparkBot Author: UniversalSuperBox File: test_all.py Apache License 2.0 | 6 votes |
|
def test_full_internal_error_text(self, full_bot_setup):
"""Tests that unhandled exceptions in commands with error text are handled safely"""
bot = full_bot_setup["bot"]
aux_api = full_bot_setup["aux_api"]
emulator = full_bot_setup["emulator"]
@bot.command("exception")
def cause_exception():
raise ValueError("Whoops", "Hey, an exception")
self.start_receiver(full_bot_setup["receiver_process"], full_bot_setup["receiver_webhook_url"])
bot_reply = self.invoke_bot(aux_api, emulator.bot_id, emulator.bot_displayname, "exception", room_name="test1")
assert bot_reply.text == "⚠️ Error: Hey, an exception"
Example 24
| Project: SparkBot Author: UniversalSuperBox File: test_all.py Apache License 2.0 | 6 votes |
|
def test_full_internal_error(self, full_bot_setup):
"""Tests that unhandled exceptions in commands without error text are handled safely"""
bot = full_bot_setup["bot"]
aux_api = full_bot_setup["aux_api"]
emulator = full_bot_setup["emulator"]
@bot.command("exception")
def cause_exception():
raise ValueError("Whoops")
self.start_receiver(full_bot_setup["receiver_process"], full_bot_setup["receiver_webhook_url"])
bot_reply = self.invoke_bot(aux_api, emulator.bot_id, emulator.bot_displayname, "exception", room_name="test1")
assert bot_reply.text == "⚠️ Error: Something happened internally. For more information, contact the bot author."
Example 25
| Project: ff-net-worth-calculator Author: blocktrron File: __init__.py GNU Affero General Public License v3.0 | 5 votes |
|
def download(url, timeout=5):
try:
response = requests.get(url, timeout=timeout)
except requests.exceptions.RequestException as ex:
error(f"Exception caught while fetching {url}\n{ex}")
return None
if response.status_code != 200:
error(f"Unexpected status code {response.status_code} while fetching {url}")
return None
return response
Example 26
| Project: fishroom Author: tuna File: textstore.py GNU General Public License v3.0 | 5 votes |
|
def new_paste(self, text, sender, **kwargs) -> str:
ts = kwargs["date"] + kwargs["time"] \
if "date" in kwargs and "time" in kwargs \
else get_now().strftime("%Y%m%d%H%M")
filename = "{sender}.{ts}.txt".format(
sender=sender,
ts=ts
)
data = {
'api_option': "paste",
'api_dev_key': self.api_dev_key,
'api_paste_code': text,
'api_paste_name': filename,
}
try:
r = requests.post(self.api_url, data=data, timeout=5)
except requests.exceptions.Timeout:
logger.error("Timeout uploading to Pastebin")
return None
if r.text.startswith("http"):
return r.text.strip()
return None
Example 27
| Project: fishroom Author: tuna File: textstore.py GNU General Public License v3.0 | 5 votes |
|
def new_paste(self, text, sender, **kwargs) -> str:
data = {
'vimcn': text,
}
try:
r = requests.post(self.api_url, data=data, timeout=5)
except requests.exceptions.Timeout:
logger.error("Timeout uploading to Vinergy")
return None
if r.text.startswith("http"):
return r.text.strip()
return None
Example 28
| Project: fishroom Author: tuna File: photostore.py GNU General Public License v3.0 | 5 votes |
|
def upload_image(self, filename=None, filedata=None, **kwargs):
if filedata is None:
with open(filename, 'rb') as f:
b64img = b64encode(f.read())
else:
b64img = b64encode(filedata)
headers = {"Authorization": "Client-ID %s" % self.client_id}
try:
r = requests.post(
self.url,
headers=headers,
data={
'image': b64img,
'type': 'base64',
},
timeout=5,
)
except requests.exceptions.Timeout:
logger.error("Timeout uploading to Imgur")
return None
except:
logger.exception("Unknown errror uploading to Imgur")
return None
try:
ret = json.loads(r.text)
except:
return None
if ret.get('status', None) != 200 or ret.get('success', False) != True:
logger.error(
"Error: Imgur returned error, {}".format(ret.get('data', ''))
)
return None
link = ret.get('data', {}).get('link', None)
return link if link is None else re.sub(r'^http:', 'https:', link)
Example 29
| Project: bigquerylayers Author: smandaric File: requests.py GNU General Public License v3.0 | 5 votes |
|
def __call__(self, url, method='GET', body=None, headers=None,
timeout=None, **kwargs):
"""Make an HTTP request using requests.
Args:
url (str): The URI to be requested.
method (str): The HTTP method to use for the request. Defaults
to 'GET'.
body (bytes): The payload / body in HTTP request.
headers (Mapping[str, str]): Request headers.
timeout (Optional[int]): The number of seconds to wait for a
response from the server. If not specified or if None, the
requests default timeout will be used.
kwargs: Additional arguments passed through to the underlying
requests :meth:`~requests.Session.request` method.
Returns:
google.auth.transport.Response: The HTTP response.
Raises:
google.auth.exceptions.TransportError: If any exception occurred.
"""
try:
_LOGGER.debug('Making request: %s %s', method, url)
response = self.session.request(
method, url, data=body, headers=headers, timeout=timeout,
**kwargs)
return _Response(response)
except requests.exceptions.RequestException as caught_exc:
new_exc = exceptions.TransportError(caught_exc)
six.raise_from(new_exc, caught_exc)
Example 30
| Project: controller Author: deis File: __init__.py MIT License | 5 votes |
|
def http_get(self, path, params=None, **kwargs):
"""
Make a GET request to the k8s server.
"""
try:
url = urljoin(self.url, path)
response = self.session.get(url, params=params, **kwargs)
except requests.exceptions.ConnectionError as err:
# reraise as KubeException, but log stacktrace.
message = "There was a problem retrieving data from " \
"the Kubernetes API server. URL: {}, params: {}".format(url, params)
logger.error(message)
raise KubeException(message) from err
return response
Example 31
| Project: controller Author: deis File: __init__.py MIT License | 5 votes |
|
def http_delete(self, path, **kwargs):
"""
Make a DELETE request to the k8s server.
"""
try:
url = urljoin(self.url, path)
response = self.session.delete(url, **kwargs)
except requests.exceptions.ConnectionError as err:
# reraise as KubeException, but log stacktrace.
message = "There was a problem deleting data from " \
"the Kubernetes API server. URL: {}".format(url)
logger.error(message)
raise KubeException(message) from err
return response
Example 32
| Project: ansible_collections_google Author: ansible-collections File: gcp_utils.py GNU General Public License v3.0 | 5 votes |
|
def full_get(self, url, params=None, **kwargs):
kwargs['headers'] = self._set_headers(kwargs.get('headers'))
try:
return self.session().get(url, params=params, **kwargs)
except getattr(requests.exceptions, 'RequestException') as inst:
# Only log the message to avoid logging any sensitive info.
self.module.fail_json(msg=inst.message)
Example 33
| Project: ansible_collections_google Author: ansible-collections File: gcp_utils.py GNU General Public License v3.0 | 5 votes |
|
def full_post(self, url, data=None, json=None, **kwargs):
kwargs['headers'] = self._set_headers(kwargs.get('headers'))
try:
return self.session().post(url, data=data, json=json, **kwargs)
except getattr(requests.exceptions, 'RequestException') as inst:
self.module.fail_json(msg=inst.message)
Example 34
| Project: ansible_collections_google Author: ansible-collections File: gcp_utils.py GNU General Public License v3.0 | 5 votes |
|
def full_put(self, url, data=None, **kwargs):
kwargs['headers'] = self._set_headers(kwargs.get('headers'))
try:
return self.session().put(url, data=data, **kwargs)
except getattr(requests.exceptions, 'RequestException') as inst:
self.module.fail_json(msg=inst.message)
Example 35
| Project: ansible_collections_google Author: ansible-collections File: gcp_utils.py GNU General Public License v3.0 | 5 votes |
|
def full_patch(self, url, data=None, **kwargs):
kwargs['headers'] = self._set_headers(kwargs.get('headers'))
try:
return self.session().patch(url, data=data, **kwargs)
except getattr(requests.exceptions, 'RequestException') as inst:
self.module.fail_json(msg=inst.message)
Example 36
| Project: ansible_collections_google Author: ansible-collections File: gcp_utils.py GNU General Public License v3.0 | 5 votes |
|
def full_delete(self, url, **kwargs):
kwargs['headers'] = self._set_headers(kwargs.get('headers'))
try:
return self.session().delete(url, **kwargs)
except getattr(requests.exceptions, 'RequestException') as inst:
self.module.fail_json(msg=inst.message)
Example 37
| Project: cis Author: mozilla-iam File: common.py Mozilla Public License 2.0 | 5 votes |
|
def _load_well_known(self):
"""
Gets the discovery url's data ("well-known")
Return dict,None the well-known JSON data copy
"""
# Memory cache
if self._well_known_json is not None:
return self._well_known_json
if not self.always_use_local_file:
try:
r = requests.get(self.discovery_url)
self._well_known_json = r.json()
except (json.JSONDecodeError, requests.exceptions.ConnectionError) as e:
logger.debug("Failed to fetch schema url from discovery {} ({})".format(self.discovery_url, e))
logger.debug("Using builtin copy")
if self._well_known_json is None or self.always_use_local_file:
well_known_file = "data/well-known/mozilla-iam" # Local fall-back
if not os.path.isfile(well_known_file):
dirname = os.path.dirname(os.path.realpath(__file__))
path = dirname + "/" + well_known_file
else:
path = well_known_file
self._well_known_json = json.load(open(path))
return self._well_known_json
Example 38
| Project: cis Author: mozilla-iam File: common.py Mozilla Public License 2.0 | 5 votes |
|
def _load_schema(self, schema_url, stype="data/profile.schema"):
"""
Loads JSON Schema from an URL
@schema_url: str,None the schema URL
@stype: str, type of schema to load. This is also the name of the library builtin, local copy.
Return dict JSON object which is the CIS Profile Schema
"""
schema = None
if not self.always_use_local_file:
if schema_url is not None:
try:
r = requests.get(schema_url)
schema = r.json()
except (json.JSONDecodeError, requests.exceptions.ConnectionError) as e:
logger.debug("Failed to load schema from schema_url {} ({})".format(schema_url, e))
# That did not work, fall-back to local, built-in copy
if schema is None or self.always_use_local_file:
# Builtin, hardcoded schema from library
schema_file = stype
if not os.path.isfile(schema_file):
dirname = os.path.dirname(os.path.realpath(__file__))
path = dirname + "/" + schema_file
else:
path = schema_file
schema = json.load(open(path))
return schema
Example 39
| Project: RTX Author: RTXteam File: cache_control_helper.py MIT License | 5 votes |
|
def __init__(self):
self.sess = CacheControl(requests.session(), heuristic=CustomHeuristic(days=30), cache=FileCache('.web_cache'))
self.exceptions = requests.exceptions
Example 40
| Project: tap-salesforce Author: singer-io File: bulk.py GNU Affero General Public License v3.0 | 5 votes |
|
def has_permissions(self):
try:
self.check_bulk_quota_usage()
except requests.exceptions.HTTPError as err:
if err.response is not None:
for error_response_item in err.response.json():
if error_response_item.get('errorCode') == 'API_DISABLED_FOR_ORG':
return False
return True
Example 41
| Project: hyperledger-py Author: yeasy File: client.py Apache License 2.0 | 5 votes |
|
def _raise_for_status(self, response, explanation=None):
"""Raises stored :class:`APIError`, if one occurred."""
try:
response.raise_for_status()
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
raise errors.NotFound(e, response, explanation=explanation)
raise errors.APIError(e, response, explanation=explanation)
Example 42
| Project: python-wsmanclient Author: intelsdi-x File: wsman.py Apache License 2.0 | 5 votes |
|
def _do_request(self, payload):
payload = payload.build()
LOG.debug('Sending request to %(endpoint)s: %(payload)s',
{'endpoint': self.endpoint, 'payload': payload})
try:
resp = requests.post(
self.endpoint,
auth=requests.auth.HTTPBasicAuth(self.username, self.password),
data=payload,
# TODO(ifarkas): enable cert verification
verify=False)
except Exception as e:
# This is a hack for handling 'No route to host' ConnectionError,
# so that the Traceback would not be shown
if e.__class__ == 'requests.exceptions.ConnectionError':
LOG.exception('Request failed (ConnectionError)')
raise exceptions.WSManRequestFailure()
if e.__class__ == 'requests.exceptions.RequestException':
LOG.exception('Request failed')
raise exceptions.WSManRequestFailure()
else:
raise
LOG.debug('Received response from %(endpoint)s: %(payload)s',
{'endpoint': self.endpoint, 'payload': resp.content})
if not resp.ok:
raise exceptions.WSManInvalidResponse(
status_code=resp.status_code,
reason=resp.reason)
else:
return resp
Example 43
| Project: python-wsmanclient Author: intelsdi-x File: wsman.py Apache License 2.0 | 5 votes |
|
def invoke(self, resource_uri, method, selectors=None, properties=None,
expected_return_value=None):
"""Invokes a remote WS-Man method
:param resource_uri: URI of the resource
:param method: name of the method to invoke
:param selectors: dictionary of selectors
:param properties: dictionary of properties
:param expected_return_value: expected return value reported back by
the DRAC card. For return value codes check the profile
documentation of the resource used in the method call. If not set,
return value checking is skipped.
:returns: an lxml.etree.Element object of the response received
:raises: WSManRequestFailure on request failures
:raises: WSManInvalidResponse when receiving invalid response
:raises: DRACOperationFailed on error reported back by the DRAC
interface
:raises: DRACUnexpectedReturnValue on return value mismatch
"""
if selectors is None:
selectors = {}
if properties is None:
properties = {}
resp = super(WSManClient, self).invoke(resource_uri, method, selectors,
properties)
return_value = utils.find_xml(resp, 'ReturnValue', resource_uri).text
if return_value == utils.RET_ERROR:
message_elems = utils.find_xml(resp, 'Message', resource_uri, True)
messages = [message_elem.text for message_elem in message_elems]
raise exceptions.DRACOperationFailed(drac_messages=messages)
if (expected_return_value is not None and
return_value != expected_return_value):
raise exceptions.DRACUnexpectedReturnValue(
expected_return_value=expected_return_value,
actual_return_value=return_value)
return resp
Example 44
| Project: nucypher Author: nucypher File: nodes.py GNU Affero General Public License v3.0 | 5 votes |
|
def block_until_number_of_known_nodes_is(self,
number_of_nodes_to_know: int,
timeout: int = 10,
learn_on_this_thread: bool = False):
start = maya.now()
starting_round = self._learning_round
while True:
rounds_undertaken = self._learning_round - starting_round
if len(self.__known_nodes) >= number_of_nodes_to_know:
if rounds_undertaken:
self.log.info("Learned about enough nodes after {} rounds.".format(rounds_undertaken))
return True
if not self._learning_task.running:
self.log.warn("Blocking to learn about nodes, but learning loop isn't running.")
if learn_on_this_thread:
try:
self.learn_from_teacher_node(eager=True)
except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectTimeout):
# TODO: Even this "same thread" logic can be done off the main thread.
self.log.warn("Teacher was unreachable. No good way to handle this on the main thread.")
# The rest of the fucking owl
if (maya.now() - start).seconds > timeout:
if not self._learning_task.running:
raise RuntimeError("Learning loop is not running. Start it with start_learning().")
else:
raise self.NotEnoughNodes("After {} seconds and {} rounds, didn't find {} nodes".format(
timeout, rounds_undertaken, number_of_nodes_to_know))
else:
time.sleep(.1)
Example 45
| Project: tts-ros1 Author: aws-robotics File: amazonpolly.py Apache License 2.0 | 5 votes |
|
def _node_request_handler(self, request):
"""The callback function for processing service request.
It never raises. If anything unexpected happens, it will return a PollyResponse with details of the exception.
:param request: an instance of PollyRequest
:return: a PollyResponse
"""
rospy.loginfo('Amazon Polly Request: {}'.format(request))
try:
response = self._dispatch(request)
rospy.loginfo('will return {}'.format(response))
return PollyResponse(result=response)
except Exception as e:
current_dir = os.path.dirname(os.path.abspath(__file__))
exc_type = sys.exc_info()[0]
# not using `issubclass(exc_type, ConnectionError)` for the condition below because some versions
# of urllib3 raises exception when doing `from requests.exceptions import ConnectionError`
error_ogg_filename = 'connerror.ogg' if 'ConnectionError' in exc_type.__name__ else 'error.ogg'
error_details = {
'Audio File': os.path.join(current_dir, 'data', error_ogg_filename),
'Audio Type': 'ogg',
'Exception': {
'Type': str(exc_type),
'Module': exc_type.__module__,
'Name': exc_type.__name__,
'Value': str(e),
},
'Traceback': traceback.format_exc()
}
error_str = json.dumps(error_details)
rospy.logerr(error_str)
return PollyResponse(result=error_str)
Example 46
| Project: hypernova Author: hyperhq File: client.py Apache License 2.0 | 5 votes |
|
def _raise_for_status(self, response, explanation=None):
"""Raises stored :class:`APIError`, if one occurred."""
try:
response.raise_for_status()
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
raise errors.NotFound(e, response, explanation=explanation)
raise errors.APIError(e, response, explanation=explanation)
Example 47
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilder.py MIT License | 5 votes |
|
def get(self, restApi, data={}, stream=False, silentMode=False, ignoreError=False):
"""
Description
A HTTP GET function to send REST APIs.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
silentMode: True or False. To display URL, data and header info.
ignoreError: True or False. If False, the response will be returned.
"""
if silentMode is False:
print('\nGET:', restApi)
print('HEADERS:', self.jsonHeader)
try:
response = requests.get(restApi, headers=self.jsonHeader)
if silentMode is False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
if ignoreError == False:
raise IxNetRestApiException('http GET error:{0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http GET error: {0}\n'.format(errMsg))
Example 48
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilder.py MIT License | 5 votes |
|
def post(self, restApi, data={}, headers=None, silentMode=False, ignoreError=False):
"""
Description
A HTTP POST function to mainly used to create or start operations.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
headers: The special header to use for the URL.
silentMode: True or False. To display URL, data and header info.
noDataJsonDumps: True or False. If True, use json dumps. Else, accept the data as-is.
ignoreError: True or False. If False, the response will be returned. No exception will be raised.
"""
if headers != None:
originalJsonHeader = self.jsonHeader
self.jsonHeader = headers
data = json.dumps(data)
print('\nPOST:', restApi)
if silentMode == False:
print('DATA:', data)
print('HEADERS:', self.jsonHeader)
try:
response = requests.post(restApi, data=data, headers=self.jsonHeader)
# 200 or 201
if silentMode == False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
if ignoreError == False:
raise IxNetRestApiException('http POST error: {0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http POST error: {0}\n'.format(errMsg))
Example 49
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilder.py MIT License | 5 votes |
|
def delete(self, restApi, data={}, headers=None):
"""
Description
A HTTP DELETE function to delete the session.
For Linux API server only.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
headers: The header to use for the URL.
"""
if headers != None:
self.jsonHeader = headers
print('\nDELETE:', restApi)
print('DATA:', data)
print('HEADERS:', self.jsonHeader)
try:
response = requests.delete(restApi, data=json.dumps(data), headers=self.jsonHeader)
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
raise IxNetRestApiException('http DELETE error: {0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http DELETE error: {0}\n'.format(errMsg))
Example 50
| Project: IxNetwork Author: OpenIxia File: IxNetRestApi.py MIT License | 5 votes |
|
def get(self, restApi, data={}, stream=False, silentMode=False, ignoreError=False):
"""
Description
A HTTP GET function to send REST APIs.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
silentMode: True or False. To display URL, data and header info.
ignoreError: True or False. If False, the response will be returned.
"""
if silentMode is False or self.generateRestLogFile is True:
#print('\nGET:', restApi)
#print('HEADERS:', self.jsonHeader)
self.logInfo('\nGET: {0}'.format(restApi))
self.logInfo('HEADERS: {0}'.format(self.jsonHeader))
try:
# For binary file
if stream:
response = requests.get(restApi, stream=True, headers=self.jsonHeader, verify=self.verifySslCert)
if stream == False:
response = requests.get(restApi, headers=self.jsonHeader, verify=self.verifySslCert)
if silentMode is False:
self.logInfo('STATUS CODE: {0}'.format(response.status_code))
if not re.match('2[0-9][0-9]', str(response.status_code)):
if ignoreError == False:
if 'message' in response.json() and response.json()['messsage'] != None:
self.logWarning('\n%s' % response.json()['message'])
raise IxNetRestApiException('GET error:{0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('GET error: {0}\n'.format(errMsg))
Example 51
| Project: IxNetwork Author: OpenIxia File: IxNetRestApi.py MIT License | 5 votes |
|
def patch(self, restApi, data={}, silentMode=False):
"""
Description
A HTTP PATCH function to modify configurations.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
silentMode: True or False. To display URL, data and header info.
"""
if silentMode == False:
self.logInfo('\nPATCH: %s' % restApi)
self.logInfo('DATA: %s' % data)
self.logInfo('HEADERS: %s' % self.jsonHeader)
try:
response = requests.patch(restApi, data=json.dumps(data), headers=self.jsonHeader, verify=self.verifySslCert)
if silentMode == False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
if 'message' in response.json() and response.json()['messsage'] != None:
self.logWarning('\n%s' % response.json()['message'])
self.showErrorMessage()
raise IxNetRestApiException('PATCH error: {0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('PATCH error: {0}\n'.format(errMsg))
Example 52
| Project: IxNetwork Author: OpenIxia File: IxNetRestApi.py MIT License | 5 votes |
|
def delete(self, restApi, data={}, headers=None):
"""
Description
A HTTP DELETE function to delete the session.
For Linux API server only.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
headers: The header to use for the URL.
"""
if headers != None:
self.jsonHeader = headers
self.logInfo('\nDELETE: %s' % restApi)
self.logInfo('DATA: %s' % data)
self.logInfo('HEADERS: %s' % self.jsonHeader)
try:
response = requests.delete(restApi, data=json.dumps(data), headers=self.jsonHeader, verify=self.verifySslCert)
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
self.showErrorMessage()
raise IxNetRestApiException('DELETE error: {0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('DELETE error: {0}\n'.format(errMsg))
Example 53
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilderRestApi.py MIT License | 5 votes |
|
def get(self, restApi, data={}, stream=False, silentMode=False, ignoreError=False):
"""
Description
A HTTP GET function to send REST APIs.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
silentMode: True or False. To display URL, data and header info.
ignoreError: True or False. If False, the response will be returned.
"""
if silentMode is False:
print('\nGET:', restApi)
print('HEADERS:', self.jsonHeader)
try:
response = requests.get(restApi, headers=self.jsonHeader)
if silentMode is False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
if ignoreError == False:
raise IxNetRestApiException('http GET error:{0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http GET error: {0}\n'.format(errMsg))
Example 54
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilderRestApi.py MIT License | 5 votes |
|
def post(self, restApi, data={}, headers=None, silentMode=False, ignoreError=False):
"""
Description
A HTTP POST function to mainly used to create or start operations.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
headers: The special header to use for the URL.
silentMode: True or False. To display URL, data and header info.
noDataJsonDumps: True or False. If True, use json dumps. Else, accept the data as-is.
ignoreError: True or False. If False, the response will be returned. No exception will be raised.
"""
if headers != None:
originalJsonHeader = self.jsonHeader
self.jsonHeader = headers
data = json.dumps(data)
print('\nPOST:', restApi)
if silentMode == False:
print('DATA:', data)
print('HEADERS:', self.jsonHeader)
try:
response = requests.post(restApi, data=data, headers=self.jsonHeader)
# 200 or 201
if silentMode == False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
if ignoreError == False:
raise IxNetRestApiException('http POST error: {0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http POST error: {0}\n'.format(errMsg))
Example 55
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilderRestApi.py MIT License | 5 votes |
|
def get(self, restApi, data={}, stream=False, silentMode=False, ignoreError=False):
"""
Description
A HTTP GET function to send REST APIs.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
silentMode: True or False. To display URL, data and header info.
ignoreError: True or False. If False, the response will be returned.
"""
if silentMode is False:
print('\nGET:', restApi)
print('HEADERS:', self.jsonHeader)
try:
response = requests.get(restApi, headers=self.jsonHeader)
if silentMode is False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
if ignoreError == False:
raise IxNetRestApiException('http GET error:{0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http GET error: {0}\n'.format(errMsg))
Example 56
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilderRestApi.py MIT License | 5 votes |
|
def post(self, restApi, data={}, headers=None, silentMode=False, ignoreError=False):
"""
Description
A HTTP POST function to mainly used to create or start operations.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
headers: The special header to use for the URL.
silentMode: True or False. To display URL, data and header info.
noDataJsonDumps: True or False. If True, use json dumps. Else, accept the data as-is.
ignoreError: True or False. If False, the response will be returned. No exception will be raised.
"""
if headers != None:
originalJsonHeader = self.jsonHeader
self.jsonHeader = headers
data = json.dumps(data)
print('\nPOST:', restApi)
if silentMode == False:
print('DATA:', data)
print('HEADERS:', self.jsonHeader)
try:
response = requests.post(restApi, data=data, headers=self.jsonHeader)
# 200 or 201
if silentMode == False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
if ignoreError == False:
raise IxNetRestApiException('http POST error: {0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http POST error: {0}\n'.format(errMsg))
Example 57
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilderRestApi.py MIT License | 5 votes |
|
def delete(self, restApi, data={}, headers=None):
"""
Description
A HTTP DELETE function to delete the session.
For Linux API server only.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
headers: The header to use for the URL.
"""
if headers != None:
self.jsonHeader = headers
print('\nDELETE:', restApi)
print('DATA:', data)
print('HEADERS:', self.jsonHeader)
try:
response = requests.delete(restApi, data=json.dumps(data), headers=self.jsonHeader)
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
raise IxNetRestApiException('http DELETE error: {0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http DELETE error: {0}\n'.format(errMsg))
Example 58
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilderRestApi.py MIT License | 5 votes |
|
def get(self, restApi, data={}, stream=False, silentMode=False, ignoreError=False):
"""
Description
A HTTP GET function to send REST APIs.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
silentMode: True or False. To display URL, data and header info.
ignoreError: True or False. If False, the response will be returned.
"""
if silentMode is False:
print('\nGET:', restApi)
print('HEADERS:', self.jsonHeader)
try:
response = requests.get(restApi, headers=self.jsonHeader)
if silentMode is False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
if ignoreError == False:
raise IxNetRestApiException('http GET error:{0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http GET error: {0}\n'.format(errMsg))
Example 59
| Project: IxNetwork Author: OpenIxia File: ixVmChassisBuilderRestApi.py MIT License | 5 votes |
|
def post(self, restApi, data={}, headers=None, silentMode=False, ignoreError=False):
"""
Description
A HTTP POST function to mainly used to create or start operations.
Parameters
restApi: The REST API URL.
data: The data payload for the URL.
headers: The special header to use for the URL.
silentMode: True or False. To display URL, data and header info.
noDataJsonDumps: True or False. If True, use json dumps. Else, accept the data as-is.
ignoreError: True or False. If False, the response will be returned. No exception will be raised.
"""
if headers != None:
originalJsonHeader = self.jsonHeader
self.jsonHeader = headers
data = json.dumps(data)
print('\nPOST:', restApi)
if silentMode == False:
print('DATA:', data)
print('HEADERS:', self.jsonHeader)
try:
response = requests.post(restApi, data=data, headers=self.jsonHeader)
# 200 or 201
if silentMode == False:
print('STATUS CODE:', response.status_code)
if not re.match('2[0-9][0-9]', str(response.status_code)):
if ignoreError == False:
raise IxNetRestApiException('http POST error: {0}\n'.format(response.text))
return response
except requests.exceptions.RequestException as errMsg:
raise IxNetRestApiException('http POST error: {0}\n'.format(errMsg))
Example 60
| Project: openeo-python-client Author: Open-EO File: connection.py Apache License 2.0 | 5 votes |
|
def _handle_error_response(self, response):
# TODO replace this with `_raise_api_error`
if response.status_code == 502:
from requests.exceptions import ProxyError
raise ProxyError("The proxy returned an error, this could be due to a timeout.")
else:
message = None
if response.headers['Content-Type'] == 'application/json':
message = response.json().get('message', None)
if message:
message = response.text
raise ConnectionAbortedError(message)
Example 61
| Project: PTBot Author: whocares-openscene File: pagecheck.py GNU General Public License v3.0 | 5 votes |
|
def isup(text):
"""<url> - uses isup.me to check if <url> is online or offline
:type text: str
"""
url = text.strip()
# slightly overcomplicated, esoteric URL parsing
scheme, auth, path, query, fragment = urllib.parse.urlsplit(url)
domain = auth or path
try:
response = requests.get('http://isup.me/' + domain)
except requests.exceptions.ConnectionError:
return "Failed to get status."
if response.status_code != requests.codes.ok:
return "Failed to get status."
soup = BeautifulSoup(response.text, 'lxml')
content = soup.find('div').text.strip()
if "not just you" in content:
return "It's not just you. {} looks \x02\x034down\x02\x0f from here!".format(url)
elif "is up" in content:
return "It's just you. {} is \x02\x033up\x02\x0f.".format(url)
else:
return "Huh? That doesn't look like a site on the interweb."
Example 62
| Project: PTBot Author: whocares-openscene File: fishbans.py GNU General Public License v3.0 | 5 votes |
|
def fishbans(text, bot):
"""<user> - gets information on <user>'s minecraft bans from fishbans"""
user = text.strip()
headers = {'User-Agent': bot.user_agent}
try:
request = requests.get(api_url.format(quote_plus(user)), headers=headers)
request.raise_for_status()
except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
return "Could not fetch ban data from the Fishbans API: {}".format(e)
try:
json = request.json()
except ValueError:
return "Could not fetch ban data from the Fishbans API: Invalid Response"
if not json["success"]:
return "Could not fetch ban data for {}.".format(user)
user_url = "http://fishbans.com/u/{}/".format(user)
ban_count = json["stats"]["totalbans"]
if ban_count == 1:
return "The user \x02{}\x02 has \x021\x02 ban - {}".format(user, user_url)
elif ban_count > 1:
return "The user \x02{}\x02 has \x02{}\x02 bans - {}".format(user, ban_count, user_url)
else:
return "The user \x02{}\x02 has no bans - {}".format(user, user_url)
Example 63
| Project: PTBot Author: whocares-openscene File: fishbans.py GNU General Public License v3.0 | 5 votes |
|
def bancount(text, bot):
"""<user> - gets a count of <user>'s minecraft bans from fishbans"""
user = text.strip()
headers = {'User-Agent': bot.user_agent}
try:
request = requests.get(api_url.format(quote_plus(user)), headers=headers)
request.raise_for_status()
except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
return "Could not fetch ban data from the Fishbans API: {}".format(e)
try:
json = request.json()
except ValueError:
return "Could not fetch ban data from the Fishbans API: Invalid Response"
if not json["success"]:
return "Could not fetch ban data for {}.".format(user)
user_url = "http://fishbans.com/u/{}/".format(user)
services = json["stats"]["service"]
out = []
for service, ban_count in list(services.items()):
if ban_count != 0:
out.append("{}: \x02{}\x02".format(service, ban_count))
else:
pass
if not out:
return "The user \x02{}\x02 has no bans - {}".format(user, user_url)
else:
return "Bans for \x02{}\x02: {} - {}".format(user, formatting.get_text_list(out, "and"), user_url)
Example 64
| Project: pycosio Author: Accelize File: test_storage_http.py Apache License 2.0 | 5 votes |
|
def test_handle_http_errors():
"""Test pycosio.http._handle_http_errors"""
from pycosio.storage.http import _handle_http_errors
from pycosio._core.exceptions import (
ObjectNotFoundError, ObjectPermissionError)
# Mocks response
class Response:
"""Dummy response"""
status_code = 200
reason = 'reason'
raised = False
def raise_for_status(self):
"""Do nothing"""
self.raised = True
response = Response()
# No error
assert _handle_http_errors(response) is response
# 403 error
response.status_code = 403
with pytest.raises(ObjectPermissionError):
_handle_http_errors(response)
# 404 error
response.status_code = 404
with pytest.raises(ObjectNotFoundError):
_handle_http_errors(response)
# Any error
response.status_code = 500
assert not response.raised
_handle_http_errors(response)
assert response.raised
Example 65
| Project: open-recipe Author: dspray95 File: requests.py The Unlicense | 5 votes |
|
def __call__(self, url, method='GET', body=None, headers=None,
timeout=None, **kwargs):
"""Make an HTTP request using requests.
Args:
url (str): The URI to be requested.
method (str): The HTTP method to use for the request. Defaults
to 'GET'.
body (bytes): The payload / body in HTTP request.
headers (Mapping[str, str]): Request headers.
timeout (Optional[int]): The number of seconds to wait for a
response from the server. If not specified or if None, the
requests default timeout will be used.
kwargs: Additional arguments passed through to the underlying
requests :meth:`~requests.Session.request` method.
Returns:
google.auth.transport.Response: The HTTP response.
Raises:
google.auth.exceptions.TransportError: If any exception occurred.
"""
try:
_LOGGER.debug('Making request: %s %s', method, url)
response = self.session.request(
method, url, data=body, headers=headers, timeout=timeout,
**kwargs)
return _Response(response)
except requests.exceptions.RequestException as caught_exc:
new_exc = exceptions.TransportError(caught_exc)
six.raise_from(new_exc, caught_exc)
Example 66
| Project: tcf Author: intel File: tc.py Apache License 2.0 | 5 votes |
|
def wait(self, regex_or_str, timeout = None, console = None):
"""
Wait for a particular regex/string to be received on a given
console of this target before a given timeout.
See :py:meth:`expect` for a version that just raises
exceptions when the output is not received.
:param int timeout: Seconds to wait for regex_or_str to be
received, raise :py:exc:`tcfl.tc.error_e`
otherwise. If *False*, no timeout check is done; if *None*,
it is taken from the default timeout set by the testcase.
:param str console: (optional) name of console from which to
receive the data
:returns: *True* if the output was received before the
timeout, *False* otherwise.
"""
if timeout:
assert isinstance(timeout, int)
console = self.console._console_get(console)
try:
with self.on_console_rx_cm(regex_or_str, timeout = timeout,
console = console, result = "pass"):
# Run the expect loop
self.testcase.tls.expecter.run()
except pass_e:
return True
except error_e:
return False
Example 67
| Project: tcf Author: intel File: tc.py Apache License 2.0 | 5 votes |
|
def from_exception(fn):
"""
Call a phase function to translate exceptions into
:class:`tcfl.tc.result_c` return codes.
Passes through the return code, unless it is None, in which
case we just return result_c(1, 0, 0, 0, 0)
Note this function prints some more extra detail in case of
fail/block/skip.
it.
"""
def _decorated_fn(*args, **kwargs):
_tc = args[0] # The `self` argument to the test case
try:
r = fn(*args, **kwargs)
if r == None:
return result_c(1, 0, 0, 0, 0)
else:
return r
# Some exceptions that are common and we know about, so we
# can print some more info that will be helpful
except subprocess.CalledProcessError as e:
return result_c.from_exception_cpe(_tc, e)
except OSError as e:
attachments = dict(
errno = e.errno,
strerror = e.strerror
)
if e.filename:
attachments['filename'] = e.filename
return result_c.report_from_exception(_tc, e, attachments)
except Exception as e:
return result_c.report_from_exception(_tc, e)
return _decorated_fn
Example 68
| Project: tcf Author: intel File: tc.py Apache License 2.0 | 5 votes |
|
def wait(self, regex_or_str, timeout = None, console = None):
"""
Wait for a particular regex/string to be received on a given
console of this target before a given timeout.
See :py:meth:`expect` for a version that just raises
exceptions when the output is not received.
:param int timeout: Seconds to wait for regex_or_str to be
received, raise :py:exc:`tcfl.tc.error_e`
otherwise. If *False*, no timeout check is done; if *None*,
it is taken from the default timeout set by the testcase.
:param str console: (optional) name of console from which to
receive the data
:returns: *True* if the output was received before the
timeout, *False* otherwise.
"""
if timeout:
assert isinstance(timeout, int)
console = self.console._console_get(console)
try:
with self.on_console_rx_cm(regex_or_str, timeout = timeout,
console = console, result = "pass"):
# Run the expect loop
self.testcase.tls.expecter.run()
except pass_e:
return True
except error_e:
return False
Example 69
| Project: datadog-unix-agent Author: DataDog File: agent.py Apache License 2.0 | 5 votes |
|
def status(cls, config, to_screen=True):
status = {}
rendered = None
api_addr = config['api']['bind_host']
api_port = config['api']['port']
target = 'http://{host}:{port}/status'.format(host=api_addr, port=api_port)
try:
r = requests.get(target, timeout=cls.STATUS_TIMEOUT)
r.raise_for_status()
status = r.json()
except requests.exceptions.HTTPError as e:
log.error("HTTP error collecting agent status: %s", e)
except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:
log.error("Problem connecting or connection timed out, is the agent up? Error: %s", e)
except ValueError as e:
log.error("There was a problem unmarshaling JSON response: %s", e)
if status:
here = os.path.dirname(os.path.realpath(__file__))
templates = os.path.join(here, 'templates')
template_env = Environment(loader=FileSystemLoader(templates))
template = template_env.get_template('status.jinja')
rendered = template.render(version=AGENT_VERSION, status=status)
if to_screen:
print(rendered)
return rendered
Example 70
| Project: gro-controller Author: OpenAgricultureFoundation File: server.py GNU General Public License v2.0 | 5 votes |
|
def _getJsonWithRetry(self, url):
"""Private method to wrap getting json data with retries. Only gets single page, nothing fancy. See getJson
:param url: url to get
:return: json data returned from server
"""
retry_count = 0
req = None
while retry_count < self._max_retries:
try:
#val = 'Token ' + self._token
headers = {'Authorization': 'Token ' + self._token}
req = requests.get(url, timeout=self._req_timeout, headers=headers)
if req.status_code == requests.codes.ok:
break
logging.warning('Failed to get %s, status %d, retry %d' % (url, req.status_code, retry_count))
except requests.exceptions.RequestException as e:
logging.warning('Failed to get request for %s, RequestException: %s' % (url, e))
pass # Just pass it, we will include it as a retry ahead
finally:
retry_count += 1
if retry_count >= self._max_retries:
logging.error("Exceeded max connection retries for %s" % url)
if req is not None:
logging.error("Request failure reason: %s" % req.reason)
raise ConnectionError(req.reason)
else:
logging.error("No request, no reason!")
raise ConnectionError
return req.json()
# TODO check documentation
Example 71
| Project: rowgenerators Author: Metatab File: download.py MIT License | 4 votes |
|
def download(self, url):
from os.path import abspath, join
from genericpath import exists
from rowgenerators.appurl.url import Url
from rowgenerators.exceptions import DownloadError, AccessError
# logger.debug(f"Download {url}")
working_dir = self.working_dir if self.working_dir else ''
r = Resource()
# For local files, don't download, just reference in place.
if url.scheme == 'file':
r.cache_path = Url(url.resource_url).path
r.download_time = None
# Many places the file may exist
locations = { # What a mess ...
abspath(r.cache_path),
abspath(r.cache_path.lstrip('/')),
abspath(join(working_dir, r.cache_path)),
abspath(r.cache_path.lstrip('/'))
}
for l in locations:
if exists(l):
r.sys_path = l
logger.debug("Found '{}'as local file '{}'".format(str(url), l))
break
else:
raise DownloadError(("File resource does not exist. Found none of:"
"\n{}\n\nWorking dir = {}\ncache_path={}\nspec_path={}")
.format('\n'.join(locations), working_dir, r.cache_path, url.path))
else:
# Not a local file, so actually need to download it.
try:
r.cache_path, r.download_time = self._download_with_lock(url.resource_url)
except AccessError as e:
# Try again, using a URL that we may have configured an account for. This is
# primarily S3 urls, with Boto or AWS credential
try:
r.cache_path, r.download_time = self._download_with_lock(url.auth_resource_url)
except AttributeError as e:
raise e
except DownloadError as e:
raise DownloadError("Access error for url '{}'; also tried accessing as S3 url '{}'".format(url, url.auth_resource_url ))
r.sys_path = self.cache.getsyspath(r.cache_path)
return r
Example 72
| Project: django-oidc-rp Author: impak-finance File: middleware.py MIT License | 4 votes |
|
def refresh_token(self, request):
""" Refreshes the token of the current user. """
# NOTE: no refresh token in the session means that the user wasn't authentified using the
# OpenID Connect provider (OP).
refresh_token = request.session.get('oidc_auth_refresh_token')
if refresh_token is None:
return
id_token_exp_timestamp = request.session.get('oidc_auth_id_token_exp_timestamp', None)
now_timestamp = time.time()
# Returns immediatelly if the token is still valid.
if id_token_exp_timestamp is not None and id_token_exp_timestamp > now_timestamp:
return
# Prepares the token payload that will be used to request a new token from the token
# endpoint.
refresh_token = request.session.pop('oidc_auth_refresh_token')
token_payload = {
'client_id': oidc_rp_settings.CLIENT_ID,
'client_secret': oidc_rp_settings.CLIENT_SECRET,
'grant_type': 'refresh_token',
'refresh_token': refresh_token,
}
# Calls the token endpoint.
token_response = requests.post(oidc_rp_settings.PROVIDER_TOKEN_ENDPOINT, data=token_payload)
try:
token_response.raise_for_status()
except requests.exceptions.HTTPError:
auth.logout(request)
return
token_response_data = token_response.json()
# Validates the token.
raw_id_token = token_response_data.get('id_token')
id_token = validate_and_return_id_token(raw_id_token, validate_nonce=False)
# If the token cannot be validated we have to log out the current user.
if id_token is None:
auth.logout(request)
return
# Retrieves the access token and refresh token.
access_token = token_response_data.get('access_token')
refresh_token = token_response_data.get('refresh_token')
# Stores the ID token, the related access token and the refresh token in the session.
request.session['oidc_auth_id_token'] = raw_id_token
request.session['oidc_auth_access_token'] = access_token
request.session['oidc_auth_refresh_token'] = refresh_token
# Saves the new expiration timestamp.
request.session['oidc_auth_id_token_exp_timestamp'] = \
time.time() + oidc_rp_settings.ID_TOKEN_MAX_AGE
Example 73
| Project: IxNetwork Author: OpenIxia File: IxNetRestApi.py MIT License | 4 votes |
|
def get(self, restApi, data={}, stream=False, silentMode=False, ignoreError=False, maxRetries=5):
"""
Description
A HTTP GET function to send REST APIs.
Parameters
restApi: (str): The REST API URL.
data: (dict): The data payload for the URL.
silentMode: (bool): To display on stdout: URL, data and header info.
ignoreError: (bool): True: Don't raise an exception. False: The response will be returned.
maxRetries: <int>: The maximum amount of GET retries before declaring as server connection failure.
Syntax
/api/v1/sessions/1/ixnetwork/operations
"""
retryInterval = 3
restExecutionFailures = 0
while True:
if silentMode is False:
self.logInfo('\n\tGET: {0}'.format(restApi))
try:
# For binary file
if stream:
response = self._session.request('GET', restApi, stream=True, headers=self.jsonHeader, allow_redirects=True, verify=self.verifySslCert)
if stream == False:
response = self._session.request('GET', restApi, headers=self.jsonHeader, allow_redirects=True, verify=self.verifySslCert)
if silentMode is False:
for redirectStatus in response.history:
if '307' in str(response.history):
self.logInfo('\t{0}: {1}'.format(redirectStatus, response.url), timestamp=False)
self.logInfo('\tSTATUS CODE: {0}'.format(response.status_code), timestamp=False)
if not str(response.status_code).startswith('2'):
if ignoreError == False:
if 'message' in response.json() and response.json()['messsage'] != None:
self.logWarning('\n%s' % response.json()['message'])
errMsg = 'GET Exception error: {0}'.format(response.text)
raise IxNetRestApiException(errMsg)
return response
except (requests.exceptions.RequestException, Exception) as errMsg:
errMsg = 'GET Exception error {]/{} retries: {}'.format(restExecutionFailures, maxRetries, errMsg)
if restExecutionFailures < maxRetries:
self.logError(errMsg)
restExecutionFailures += 1
time.sleep(retryInterval)
continue
if restExecutionFailures == maxRetries:
raise IxNetRestApiException(errMsg)
Example 74
| Project: IxNetwork Author: OpenIxia File: IxNetRestApi.py MIT License | 4 votes |
|
def options(self, restApi, data={}, silentMode=False, ignoreError=False, maxRetries=5):
"""
Description
A HTTP OPTIONS function to send REST APIs.
Parameters
restApi: (str): The REST API URL.
silentMode: (bool): To display on stdout: URL, data and header info.
ignoreError: (bool): True: Don't raise an exception. False: The response will be returned.
maxRetries: <int>: The maximum amount of GET retries before declaring as server connection failu
"""
retryInterval = 3
restExecutionFailures = 0
while True:
if silentMode is False:
self.logInfo('\n\tOPTIONS: {0}'.format(restApi))
try:
# For binary file
response = self._session.request('OPTIONS', restApi, headers=self.jsonHeader, allow_redirects=True, verify=self.verifySslCert)
if silentMode is False:
for redirectStatus in response.history:
if '307' in str(response.history):
self.logInfo('\t{0}: {1}'.format(redirectStatus, response.url), timestamp=False)
self.logInfo('\tSTATUS CODE: {0}'.format(response.status_code), timestamp=False)
if not str(response.status_code).startswith('2'):
if ignoreError == False:
if 'message' in response.json() and response.json()['messsage'] != None:
self.logWarning('\n%s' % response.json()['message'])
errMsg = 'OPTIONS Exception error: {0}'.format(response.text)
raise IxNetRestApiException(errMsg)
return response
except (requests.exceptions.RequestException, Exception) as errMsg:
errMsg = 'OPTIONS Exception error {}/{} retries: {}'.format(restExecutionFailures, maxRetries, errMsg)
if restExecutionFailures < maxRetries:
self.logError(errMsg)
restExecutionFailures += 1
time.sleep(retryInterval)
continue
if restExecutionFailures == maxRetries:
raise IxNetRestApiException(errMsg)
Example 75
| Project: IxNetwork Author: OpenIxia File: IxNetRestApi.py MIT License | 4 votes |
|
def delete(self, restApi, data={}, headers=None, maxRetries=5):
"""
Description
A HTTP DELETE function to delete the session.
For Linux and Windows Connection Mgr API server only.
Paramters
restApi: (str): The REST API URL.
data: (dict): The data payload for the URL.
headers: (str): The headers to use for the URL.
maxRetries: <int>: The maximum amount of GET retries before declaring as server connection failure.
"""
if headers != None:
self.jsonHeader = headers
retryInterval = 3
restExecutionFailures = 0
while True:
self.logInfo('\n\tDELETE: {0}\n\tDATA: {1}'.format(restApi, data))
try:
response = self._session.request('DELETE', restApi, data=json.dumps(data), headers=self.jsonHeader, allow_redirects=True,
verify=self.verifySslCert)
for redirectStatus in response.history:
if '307' in str(response.history):
self.logInfo('\t{0}: {1}'.format(redirectStatus, response.url), timestamp=False)
self.logInfo('\tSTATUS CODE: %s' % response.status_code, timestamp=False)
if not str(response.status_code).startswith('2'):
self.showErrorMessage()
errMsg = 'DELETE Exception error: {0}\n'.format(response.text)
self.logError(errMsg)
raise IxNetRestApiException(errMsg)
return response
except (requests.exceptions.RequestException, Exception) as errMsg:
errMsg = 'DELETE Exception error {}/{} retries: {}\n'.format(restExecutionFailures, maxRetries, errMsg)
if restExecutionFailures < maxRetries:
self.logError(errMsg)
restExecutionFailures += 1
time.sleep(retryInterval)
continue
if restExecutionFailures == maxRetries:
raise IxNetRestApiException(errMsg)
Example 76
| Project: tcf Author: intel File: tc.py Apache License 2.0 | 4 votes |
|
def __method_trampoline_call(self, fname, fn, _type, targets):
# runs a class function and return a return_c
#
# convers the return value to return_c from bool, or None, or
# passes a return_c. Passes exceptions.
#
# We use fname instead of fn.__name__ because if we have made
# an alias out of the method in the class (with setattr(), we
# want to see the alias name, not the original function name).
if fname == fn.__name__:
self.report_info("running %s.%s()"
% (type(self).__name__, fn.__name__),
dlevel = 3)
else:
self.report_info("running %s.%s() [App builder's alias for "
"%s() which calls tc_c._*_50_for_target()]"
% (type(self).__name__, fname, fn.__name__),
dlevel = 3)
if type(fn) == types.MethodType: # instance needed
if fn.im_self == None:
# This was added from target_wanted_add(), binding to an
# specific object; we forced it unbound, so we have to
# call it with 'self'
r = getattr(self, fname)(self, *targets)
else:
# This was added from the @target() decorator, binding
# to a class, it doesn't need a self as Python will
# add it for us, current self.
r = getattr(self, fname)(*targets)
else: # static/classmethod
r = fn(*targets)
# Now, let's see what did it return
if isinstance(r, result_c):
return r
elif r == None or r == True:
return result_c(1, 0, 0, 0, 0)
elif r == False:
return result_c(0, 1, 0, 0, 0)
else:
raise blocked_e(
"%s.%s(): don't know what to do with "
"return value of type %s; return nothing or "
"True for success, False for failure, raise "
"any exception for blockage, or "
"tcfl.{passed|blocked|error|failed|skip}_e "
% (type(self).__name__, fn.__name__,
type(r).__name))
Example 77
| Project: tcf Author: intel File: tc.py Apache License 2.0 | 4 votes |
|
def _methods_run(self, do_serially, serial_list, parallel_list):
"""
Implement the guts of _methods_call()
Runs and aggregates results of serial methods, then launches a
threadpool for the parallel methods; aggregates their results
and raise exception if any happened.
:returns result_c: aggregated result of all the methods.
"""
result = result_c(0, 0, 0, 0, 0)
if do_serially:
for fname, fn, _type, args in sorted(serial_list + parallel_list):
retval = self._method_run(fname, fn, _type, args)
result += retval
if retval.errors or retval.failed \
or retval.blocked or retval.skipped:
return result
else:
for fname, fn, _type, args in serial_list:
retval = self._method_run(fname, fn, _type, args)
result += retval
if retval.errors or retval.failed \
or retval.blocked or retval.skipped:
return result
# FIXME: set from from config
# All these are supposedly I/O bound jobs as either they
# are chitchatting with the network or spawning jobs to do
# stuff
thread_pool = _multiprocessing_method_pool_c(processes = 10)
threads = {}
for fname, fn, _type, args in parallel_list:
targets = self._mk_target_args_for_fn(fn, args)
threads[fname] = thread_pool.apply_async(
self.__method_trampoline_thread,
(msgid_c(l = 2), fname, fn, _type, targets,
dict(expecter_parent = self.tls.expecter)))
# so we can Ctrl-C right away--the system is designed
# to cleanup top bottom, with everything being
# expendable
threads[fname].daemon = True
thread_pool.close()
thread_pool.join()
for thread in threads.values():
r = thread.get()
if r[1] != None: # re-raise thrown exceptions
raise r[1][0], r[1][1], r[1][2]
result += r[0]
del thread_pool
return result
Example 78
| Project: tcf Author: intel File: tc.py Apache License 2.0 | 4 votes |
|
def __method_trampoline_call(self, fname, fn, _type, targets):
# runs a class function and return a return_c
#
# convers the return value to return_c from bool, or None, or
# passes a return_c. Passes exceptions.
#
# We use fname instead of fn.__name__ because if we have made
# an alias out of the method in the class (with setattr(), we
# want to see the alias name, not the original function name).
if fname == fn.__name__:
self.report_info("running %s.%s()"
% (type(self).__name__, fn.__name__),
dlevel = 3)
else:
self.report_info("running %s.%s() [App builder's alias for "
"%s() which calls tc_c._*_50_for_target()]"
% (type(self).__name__, fname, fn.__name__),
dlevel = 3)
if type(fn) == types.MethodType: # instance needed
if fn.im_self == None:
# This was added from target_wanted_add(), binding to an
# specific object; we forced it unbound, so we have to
# call it with 'self'
r = getattr(self, fname)(self, *targets)
else:
# This was added from the @target() decorator, binding
# to a class, it doesn't need a self as Python will
# add it for us, current self.
r = getattr(self, fname)(*targets)
else: # static/classmethod
r = fn(*targets)
# Now, let's see what did it return
if isinstance(r, result_c):
return r
elif r == None or r == True:
return result_c(1, 0, 0, 0, 0)
elif r == False:
return result_c(0, 1, 0, 0, 0)
else:
raise blocked_e(
"%s.%s(): don't know what to do with "
"return value of type %s; return nothing or "
"True for success, False for failure, raise "
"any exception for blockage, or "
"tcfl.{passed|blocked|error|failed|skip}_e "
% (type(self).__name__, fn.__name__,
type(r).__name))
Example 79
| Project: tcf Author: intel File: tc.py Apache License 2.0 | 4 votes |
|
def _methods_run(self, do_serially, serial_list, parallel_list):
"""
Implement the guts of _methods_call()
Runs and aggregates results of serial methods, then launches a
threadpool for the parallel methods; aggregates their results
and raise exception if any happened.
:returns result_c: aggregated result of all the methods.
"""
result = result_c(0, 0, 0, 0, 0)
if do_serially:
for fname, fn, _type, args in sorted(serial_list + parallel_list):
retval = self._method_run(fname, fn, _type, args)
result += retval
if retval.errors or retval.failed \
or retval.blocked or retval.skipped:
return result
else:
for fname, fn, _type, args in serial_list:
retval = self._method_run(fname, fn, _type, args)
result += retval
if retval.errors or retval.failed \
or retval.blocked or retval.skipped:
return result
# FIXME: set from from config
# All these are supposedly I/O bound jobs as either they
# are chitchatting with the network or spawning jobs to do
# stuff
thread_pool = _multiprocessing_method_pool_c(processes = 10)
threads = {}
for fname, fn, _type, args in parallel_list:
targets = self._mk_target_args_for_fn(fn, args)
threads[fname] = thread_pool.apply_async(
self.__method_trampoline_thread,
(msgid_c(l = 2), fname, fn, _type, targets,
dict(expecter_parent = self.tls.expecter)))
# so we can Ctrl-C right away--the system is designed
# to cleanup top bottom, with everything being
# expendable
threads[fname].daemon = True
thread_pool.close()
thread_pool.join()
for thread in threads.values():
r = thread.get()
if r[1] != None: # re-raise thrown exceptions
raise r[1][0], r[1][1], r[1][2]
result += r[0]
del thread_pool
return result
Example 80
| Project: dpa-pipe Author: Clemson-DPA File: client.py MIT License | 4 votes |
|
def execute_request_url(self, http_method, url, data=None, params=None,
headers=None):
data_format = self.data_format
if params:
params = self._sanitize_params(params)
if data:
data = json.dumps(data)
if data and not headers:
if data_format == 'json':
headers = {'content-type': 'application/json'}
else:
raise RestfulClientError("Unknown data format: " + data_format)
# requests method based on the supplied http method name.
# it *should* be a 1 to 1 lc mapping. (GET==get, PUT==put, etc.)
requests_method_name = http_method.lower()
# see if the requests api has a method that matches
try:
requests_method = getattr(requests, requests_method_name)
except AttributeError:
raise RestfulClientError(
"Unknown method for requests: " + str(requests_method_name))
# execute the request
response = self._try_request(requests_method, url, params=params,
data=data, headers=headers)
# raise a custom exception if 400/500 error
try:
response.raise_for_status()
except requests.exceptions.HTTPError as e:
raise RestfulClientError(e.response.text)
# deserialize the returned data. may want to flesh this out at some
# point to allow for additional data formats
if data_format == 'json':
# json.loads(response.content) results in unicode instead of
# str. using yaml results in string values. so using that for now.
return yaml.load(response.content)
else:
raise RestfulClientError("Unknown data format: " + data_format)
# -------------------------------------------------------------------------
# Private class methods:
# -------------------------------------------------------------------------
