Python requests.ReadTimeout() Examples

The following are 26 code examples for showing how to use requests.ReadTimeout(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module requests , or try the search function .

Example 1
Project: kuryr-kubernetes   Author: openstack   File: test_k8s_client.py    License: Apache License 2.0 6 votes vote down vote up
def test_watch_restart(self, m_get):
        path = '/test'
        data = [{'object': {'metadata': {'name': 'obj%s' % i,
                                         'resourceVersion': i}}}
                for i in range(3)]
        lines = [jsonutils.dump_as_bytes(i) for i in data]

        m_resp = mock.MagicMock()
        m_resp.ok = True
        m_resp.iter_lines.side_effect = [lines, requests.ReadTimeout, lines]
        m_get.return_value = m_resp

        self.assertEqual(data * 2,
                         list(itertools.islice(self.client.watch(path),
                                               len(data) * 2)))
        self.assertEqual(3, m_get.call_count)
        self.assertEqual(3, m_resp.close.call_count)
        m_get.assert_any_call(
            self.base_url + path, headers={}, stream=True,
            params={"watch": "true"}, cert=(None, None), verify=False,
            timeout=(30, 60))
        m_get.assert_any_call(
            self.base_url + path, headers={}, stream=True,
            params={"watch": "true", "resourceVersion": 2}, cert=(None, None),
            verify=False, timeout=(30, 60)) 
Example 2
Project: AmbroBot   Author: Ambro17   File: utils.py    License: GNU General Public License v3.0 6 votes vote down vote up
def soupify_url(url, timeout=2, encoding='utf-8', **kwargs):
    """Given a url returns a BeautifulSoup object"""
    try:
        r = requests.get(url, timeout=timeout, **kwargs)
    except ReadTimeout:
        logger.info("[soupify_url] Request for %s timed out.", url)
        raise
    except Exception as e:
        logger.error(f"Request for {url} could not be resolved", exc_info=True)
        raise ConnectionError(repr(e))


    r.raise_for_status()
    r.encoding = encoding
    if r.status_code == 200:
        return BeautifulSoup(r.text, 'lxml')
    else:
        raise ConnectionError(
            f'{url} returned error status %s - ', r.status_code, r.reason
        ) 
Example 3
Project: CloudBot   Author: TotallyNotRobots   File: test_link_announcer.py    License: GNU General Public License v3.0 6 votes vote down vote up
def test_read_timeout(mock_requests):
    url = "http://example.com"

    def callback(resp):
        raise requests.ReadTimeout()

    mock_requests.add_callback('GET', url, callback)

    match = url_re.search(url)
    assert match
    mck = MagicMock()
    logger = MagicMock()

    assert print_url_title(match=match, message=mck, logger=logger) is None

    logger.debug.assert_called_with('Read timeout reached for %r', url) 
Example 4
Project: AdslProxy   Author: Python3WebSpider   File: checker.py    License: MIT License 6 votes vote down vote up
def check(self, proxy):
        """
        测试代理,返回测试结果
        :param proxy: 代理
        :return: 测试结果
        """
        try:
            response = requests.get(settings.TEST_URL, proxies={
                'http': 'http://' + proxy,
                'https': 'https://' + proxy
            }, timeout=settings.TEST_TIMEOUT)
            logger.debug(f'Using {proxy} to test {settings.TEST_URL}...')
            if response.status_code == 200:
                return True
        except (ConnectionError, ReadTimeout):
            return False 
Example 5
Project: GitPrey   Author: repoog   File: GitPrey.py    License: GNU General Public License v3.0 6 votes vote down vote up
def __get_page_html(self, url):
        """
        Get parse html page from requesting url
        :param url: Requesting url
        :returns: Parsed html page
        """
        try:
            page_html = requests.get(url, headers=self.headers, cookies=self.cookies, timeout=SCAN_DEEP[SEARCH_LEVEL - 1])
            if page_html.status_code == 429:
                time.sleep(SCAN_DEEP[SEARCH_LEVEL - 1])
                self.__get_page_html(url)
            return page_html.text
        except requests.ConnectionError as e:
            error_print("[!] Error: There is '%s' problem in requesting html page." % str(e))
            exit()
        except requests.ReadTimeout:
            return '' 
Example 6
Project: codechef-cli   Author: sk364   File: helpers.py    License: GNU General Public License v3.0 5 votes vote down vote up
def request(session=None, method="GET", url="", token=None, **kwargs):
    if not session:
        session = get_session()
    if token:
        session.headers = getattr(session, 'headers') or {}
        session.headers.update({'X-CSRF-Token': token})

    if BASE_URL not in url:
        url = f'{BASE_URL}{url}'

    try:
        return session.request(method=method, url=url, timeout=(15, 15), **kwargs)
    except (ConnectionError, ReadTimeout):
        print(INTERNET_DOWN_MSG)
        sys.exit(1) 
Example 7
Project: galaxy_blizzard_plugin   Author: bartok765   File: backend.py    License: MIT License 5 votes vote down vote up
def do_request(self, method, url, data=None, json=True, headers=None, ignore_failure=False):
        loop = asyncio.get_event_loop()
        if not headers:
            headers = self._authentication_client.session.headers
        try:
            if data is None:
                data = {}
            params = {
                "method": method,
                "url": url,
                "data": data,
                "timeout": self._authentication_client.timeout,
                "headers": headers
            }
            try:
                response = await loop.run_in_executor(None, functools.partial(self._authentication_client.session.request, **params))
            except (requests.Timeout, requests.ConnectTimeout, requests.ReadTimeout):
                raise BackendTimeout()
            except requests.ConnectionError:
                raise NetworkError

            if not ignore_failure:
                self.handle_status_code(response.status_code)

            if json:
                return response.json()
            else:
                return response

        except Exception as e:
            raise e 
Example 8
Project: oh-my-rss   Author: richshaw2015   File: wemp.py    License: MIT License 5 votes vote down vote up
def wemp_spider(url, site):
    """
    抓取微信内容
    :param url:
    :param site:
    :return:
    """
    if is_crawled_url(url):
        return

    try:
        rsp = requests.get(url, timeout=10)

        if rsp.ok:
            try:
                if get_host_name(rsp.url) == 'mp.weixin.qq.com':
                    title, author, content = parse_weixin_page(rsp)
                elif 'ershicimi.com' in get_host_name(rsp.url):
                    title, author, content = parse_ershicimi_page(rsp)
                else:
                    logger.warning(f'公众号域名解析异常:`{rsp.url}')
                    return
            except:
                logger.info(f'公众号内容解析异常:`{rsp.url}')
                return

            article = Article(title=title, author=author, site=site, uindex=current_ts(),
                              content=content, src_url=url)
            article.save()

            mark_crawled_url(url)
    except (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError):
        logger.warning(f'公众号爬取出现网络异常:`{url}')
    except:
        logger.warning(f'公众号爬取出现未知异常:`{url}') 
Example 9
Project: oh-my-rss   Author: richshaw2015   File: utils.py    License: MIT License 5 votes vote down vote up
def save_avatar(avatar, userid, size=100):
    """
    保存网络头像
    :param avatar:
    :param userid:
    :param size:
    :return: 保存后的头像地址
    """
    try:
        rsp = requests.get(avatar, timeout=10)

        if rsp.ok:
            img_obj = Image.open(BytesIO(rsp.content))
            img_obj.thumbnail((size, size))
            jpg = get_hash_name(userid) + '.jpg'

            if img_obj.mode != 'RGB':
                img_obj = img_obj.convert('RGB')

            img_obj.save(os.path.join(settings.BASE_DIR, 'assets', 'avatar', jpg))
            return f'/assets/avatar/{jpg}'
        else:
            logger.error(f"同步用户头像出现网络异常!`{userid}`{avatar}")
    except (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError):
        logger.error(f"同步用户头像网络异常!`{userid}`{avatar}")
    except:
        logger.error(f"同步用户头像未知异常`{userid}`{avatar}")

    return '/assets/img/logo.svg' 
Example 10
Project: uiautomator2   Author: openatx   File: __init__.py    License: MIT License 5 votes vote down vote up
def request(self, method, url, **kwargs):
        retry = kwargs.pop("retry", True)
        try:
            url = self.__client.path2url(
                url)  # may raise adbutils.AdbError when device offline
            return super().request(method, url, **kwargs)
        except (requests.ConnectionError, requests.ReadTimeout,
                adbutils.AdbError) as e:
            if not retry:
                raise

            # if atx-agent is already running, just raise error
            if isinstance(e, requests.RequestException) and \
                self.__client._is_agent_alive():
                raise

        if not self.__client._serial:
            raise EnvironmentError(
                "http-request to atx-agent error, can only recover from USB")

        logger.warning("atx-agent has something wrong, auto recovering")
        # ReadTimeout: sometime means atx-agent is running but not responsing
        # one reason is futex_wait_queue: https://stackoverflow.com/questions/9801256/app-hangs-on-futex-wait-queue-me-every-a-couple-of-minutes

        # fix atx-agent and request again
        self.__client._prepare_atx_agent()
        url = self.__client.path2url(url)
        return super().request(method, url, **kwargs) 
Example 11
Project: uiautomator2   Author: openatx   File: __init__.py    License: MIT License 5 votes vote down vote up
def _jsonrpc_retry_call(self, *args, **kwargs):
        try:
            return self._jsonrpc_call(*args, **kwargs)
        except (requests.ReadTimeout,
                ServerError,
                UiAutomationNotConnectedError) as e:
            self.reset_uiautomator(str(e)) # uiautomator可能出问题了,强制重启一下
        except (NullObjectExceptionError,
                NullPointerExceptionError,
                StaleObjectExceptionError) as e:
            logger.warning("jsonrpc call got: %s", str(e))
        return self._jsonrpc_call(*args, **kwargs) 
Example 12
Project: uiautomator2   Author: openatx   File: __init__.py    License: MIT License 5 votes vote down vote up
def _is_alive(self):
        try:
            r = self.http.post("/jsonrpc/0", timeout=2, retry=False, data=json.dumps({
                "jsonrpc": "2.0",
                "id": 1,
                "method": "deviceInfo",
            }))
            if r.status_code != 200:
                return False
            if r.json().get("error"):
                return False
            return True
        except (requests.ReadTimeout, EnvironmentError):
            return False 
Example 13
Project: uiautomator2   Author: openatx   File: _selector.py    License: MIT License 5 votes vote down vote up
def wait(self, exists=True, timeout=None):
        """
        Wait until UI Element exists or gone

        Args:
            timeout (float): wait element timeout

        Example:
            d(text="Clock").wait()
            d(text="Settings").wait("gone") # wait until it's gone
        """
        if timeout is None:
            timeout = self.wait_timeout
        http_wait = timeout + 10
        if exists:
            try:
                return self.jsonrpc.waitForExists(self.selector,
                                                  int(timeout * 1000),
                                                  http_timeout=http_wait)
            except requests.ReadTimeout as e:
                warnings.warn("waitForExists readTimeout: %s" % e,
                              RuntimeWarning)
                return self.exists()
        else:
            try:
                return self.jsonrpc.waitUntilGone(self.selector,
                                                  int(timeout * 1000),
                                                  http_timeout=http_wait)
            except requests.ReadTimeout as e:
                warnings.warn("waitForExists readTimeout: %s" % e,
                              RuntimeWarning)
                return not self.exists() 
Example 14
Project: bilibili_member_crawler   Author: cwjokaka   File: worker.py    License: MIT License 5 votes vote down vote up
def _get_member_by_mid(self, mid: int) -> Optional[dict]:
        """
        根据用户id获取其信息
        :param mid: B站用户id
        :return: 用户详情 or None
        """
        get_params = {
            'mid': mid,
            'jsonp': 'jsonp'
        }
        try:
            res_json = requests.get(API_MEMBER_INFO, params=get_params, timeout=WAIT_MAX, proxies=self.cur_proxy,
                                    headers=self.headers).json()
        except ConnectTimeout as e:
            print(f'获取用户id: {mid} 详情失败: 请求接口超时, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ReadTimeout as e:
            print(f'获取用户id: {mid} 详情失败: 接口读取超时, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ValueError as e:
            # 解析json失败基本上就是ip被封了
            print(f'获取用户id: {mid} 详情失败: 解析json出错, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ProxyError as e:
            print(f'获取用户id: {mid} 详情失败: 连接代理失败, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except requests.ConnectionError as e:
            # 可以断定就是代理IP地址无效
            print(f'获取用户id: {mid} 详情失败: 连接错误, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ChunkedEncodingError as e:
            print(f'获取用户id: {mid} 详情失败: 远程主机强迫关闭了一个现有的连接, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        else:
            if res_json['code'] == -404:
                print(f'找不到用户mid:{mid}')
                raise UserNotFoundException(f'找不到用户mid:{mid}')
            if 'data' in res_json:
                return res_json['data']
            print(f'获取用户id: {mid} 详情失败: data字段不存在!')
        return 
Example 15
Project: tiktok-crawler   Author: hackertogether   File: fetch.py    License: MIT License 5 votes vote down vote up
def need_retry(exception):
    """
    need to retry
    :param exception:
    :return:
    """
    result = isinstance(exception, (requests.ConnectionError, requests.ReadTimeout))
    if result:
        print('Exception', type(exception), 'occurred, retrying...')
    return result 
Example 16
Project: tiktok-crawler   Author: hackertogether   File: fetch.py    License: MIT License 5 votes vote down vote up
def fetch(url, **kwargs):
    """
    warp _fetch method
    :param url: fetch url
    :param kwargs: other requests params
    :return: result of _fetch
    """

    @retry(stop_max_attempt_number=retry_max_number, wait_random_min=retry_min_random_wait,
           wait_random_max=retry_max_random_wait, retry_on_exception=need_retry)
    def _fetch(url, **kwargs):
        """
        fetch api response
        :param url: fetch url
        :param kwargs: other requests params
        :return: json of response
        """
        response = requests.get(url, **kwargs)
        if response.status_code != 200:
            raise requests.ConnectionError('Expected status code 200, but got {}'.format(response.status_code))
        return response.json() if response.content else {}

    try:
        result = _fetch(url, **kwargs)
        return result
    # give up retrying
    except (requests.ConnectionError, requests.ReadTimeout):
        return {} 
Example 17
Project: FFXIVBOT   Author: Bluefissure   File: crawl_wb.py    License: GNU General Public License v3.0 5 votes vote down vote up
def crawl():
    wbus = WeiboUser.objects.all()
    for wbu in wbus:
        logging.info("Begin crawling {}".format(wbu.name))
        try:
            crawl_wb(wbu, True)
        except requests.ReadTimeout as e:
            logging.error("crawling {} timeout".format(wbu.name))
        except Exception as e:
            traceback.print_exc()
            logging.error(e)
        time.sleep(1)
        logging.info("Crawl {} finish".format(wbu.name)) 
Example 18
Project: DouYin   Author: Python3WebSpider   File: fetch.py    License: MIT License 5 votes vote down vote up
def need_retry(exception):
    """
    need to retry
    :param exception:
    :return:
    """
    result = isinstance(exception, (requests.ConnectionError, requests.ReadTimeout))
    if result:
        print('Exception', type(exception), 'occurred, retrying...')
    return result 
Example 19
Project: DouYin   Author: Python3WebSpider   File: fetch.py    License: MIT License 5 votes vote down vote up
def fetch(url, **kwargs):
    """
    warp _fetch method
    :param url: fetch url
    :param kwargs: other requests params
    :return: result of _fetch
    """
    
    @retry(stop_max_attempt_number=retry_max_number, wait_random_min=retry_min_random_wait,
           wait_random_max=retry_max_random_wait, retry_on_exception=need_retry)
    def _fetch(url, **kwargs):
        """
        fetch api response
        :param url: fetch url
        :param kwargs: other requests params
        :return: json of response
        """
        kwargs.update({'verify': False})
        kwargs.update({'timeout': fetch_timeout})
        response = requests.get(url, **kwargs)
        if response.status_code != 200:
            raise requests.ConnectionError('Expected status code 200, but got {}'.format(response.status_code))
        return response.json()
    
    try:
        result = _fetch(url, **kwargs)
        return result
    # give up retrying
    except (requests.ConnectionError, requests.ReadTimeout):
        return {} 
Example 20
Project: bazarr   Author: morpheus65535   File: check_update.py    License: GNU General Public License v3.0 5 votes vote down vote up
def updated(restart=True):
    if settings.general.getboolean('update_restart') and restart:
        try:
            requests.get('http://127.0.0.1:' + settings.general.port + settings.general.base_url + 'restart')
        except requests.ConnectionError:
            pass
        except (requests.ConnectTimeout, requests.HTTPError, requests.ReadTimeout, requests.Timeout):
            logging.info('BAZARR Restart failed, please restart Bazarr manually')
            updated(restart=False)
    else:
        database.execute("UPDATE system SET updated='1'") 
Example 21
Project: kuryr-kubernetes   Author: openstack   File: k8s_client.py    License: Apache License 2.0 4 votes vote down vote up
def watch(self, path):
        url = self._base_url + path
        resource_version = None
        header = {}
        timeouts = (CONF.kubernetes.watch_connection_timeout,
                    CONF.kubernetes.watch_read_timeout)
        if self.token:
            header.update({'Authorization': 'Bearer %s' % self.token})

        while True:
            try:
                params = {'watch': 'true'}
                if resource_version:
                    params['resourceVersion'] = resource_version
                with contextlib.closing(
                        self.session.get(
                            url, params=params, stream=True, cert=self.cert,
                            verify=self.verify_server, headers=header,
                            timeout=timeouts)) as response:
                    if not response.ok:
                        raise exc.K8sClientException(response.text)
                    for line in response.iter_lines():
                        line = line.decode('utf-8').strip()
                        if line:
                            line_dict = jsonutils.loads(line)
                            yield line_dict
                            # Saving the resourceVersion in case of a restart.
                            # At this point it's safely passed to handler.
                            m = line_dict.get('object', {}).get('metadata', {})
                            resource_version = m.get('resourceVersion', None)
            except (requests.ReadTimeout, requests.ConnectionError,
                    ssl.SSLError) as e:
                if isinstance(e, ssl.SSLError) and e.args != ('timed out',):
                    raise

                LOG.warning('%ds without data received from watching %s. '
                            'Retrying the connection with resourceVersion=%s.',
                            timeouts[1], path, params.get('resourceVersion'))
            except requests.exceptions.ChunkedEncodingError:
                LOG.warning("Connection to %s closed when watching. This "
                            "mostly happens when Octavia's Amphora closes "
                            "connection due to lack of activity for 50s. "
                            "Since Rocky Octavia this is configurable and "
                            "should be set to at least 20m, so check timeouts "
                            "on Kubernetes API LB listener. Restarting "
                            "connection with resourceVersion=%s.", path,
                            params.get('resourceVersion')) 
Example 22
Project: wttr.in   Author: chubin   File: proxy.py    License: Apache License 2.0 4 votes vote down vote up
def proxy(path):
    """
    Main proxy function. Handles incoming HTTP queries.
    """

    lang = request.args.get('lang', 'en')
    query_string = request.query_string.decode("utf-8")
    query_string = query_string.replace('sr-lat', 'sr')
    query_string = query_string.replace('lang=None', 'lang=en')
    query_string += "&extra=localObsTime"
    query_string += "&includelocation=yes"
    content, headers = _load_content_and_headers(path, query_string)

    if content is None:
        srv = _find_srv_for_query(path, query_string)
        url = '%s/%s?%s' % (srv, path, query_string)

        attempts = 10
        response = None
        while attempts:
            try:
                response = requests.get(url, timeout=2)
            except requests.ReadTimeout:
                attempts -= 1
                continue
            try:
                json.loads(response.content)
                break
            except ValueError:
                attempts -= 1

        _touch_empty_file(path, query_string)
        if response:
            headers = {}
            headers['Content-Type'] = response.headers['content-type']
            _save_content_and_headers(path, query_string, response.content, headers)
            content = response.content
        else:
            content = "{}"
    else:
        print("cache found")

    content = add_translations(content, lang)

    return content, 200, headers 
Example 23
Project: ray   Author: ray-project   File: test_metric.py    License: Apache License 2.0 4 votes vote down vote up
def test_system_metric_endpoints(serve_instance):
    def test_error_counter(flask_request):
        1 / 0

    serve.create_backend("m:v1", test_error_counter)
    serve.create_endpoint("test_metrics", backend="m:v1", route="/measure")
    serve.set_traffic("test_metrics", {"m:v1": 1})

    # Check metrics are exposed under http endpoint
    def test_metric_endpoint():
        requests.get("http://127.0.0.1:8000/measure", timeout=5)
        in_memory_metric = requests.get(
            "http://127.0.0.1:8000/-/metrics", timeout=5).json()

        # We don't want to check the values since this check might be retried.
        in_memory_metric_without_values = []
        for m in in_memory_metric:
            m.pop("value")
            in_memory_metric_without_values.append(m)

        target_metrics = [{
            "info": {
                "name": "num_http_requests",
                "type": "MetricType.COUNTER",
                "route": "/measure"
            },
        }, {
            "info": {
                "name": "num_router_requests",
                "type": "MetricType.COUNTER",
                "endpoint": "test_metrics"
            },
        }, {
            "info": {
                "name": "backend_error_counter",
                "type": "MetricType.COUNTER",
                "backend": "m:v1"
            },
        }]

        for target in target_metrics:
            assert target in in_memory_metric_without_values

    success = False
    for _ in range(3):
        try:
            test_metric_endpoint()
            success = True
            break
        except (AssertionError, requests.ReadTimeout):
            # Metrics may not have been propagated yet
            time.sleep(2)
            print("Metric not correct, retrying...")
    if not success:
        test_metric_endpoint() 
Example 24
Project: PyPlanet   Author: PyPlanet   File: api.py    License: GNU General Public License v3.0 4 votes vote down vote up
def execute(self, method, *args):
		payload = dumps(args, methodname=method, allow_none=True)
		body = gzip.compress(payload.encode('utf8'))
		try:
			res = await self.loop.run_in_executor(None, self.__request, body)
			data, _ = loads(res.text, use_datetime=True)
			if isinstance(data, (tuple, list)) and len(data) > 0 and len(data[0]) > 0:
				if isinstance(data[0][0], dict) and 'faultCode' in data[0][0]:
					raise DedimaniaFault(faultCode=data[0][0]['faultCode'], faultString=data[0][0]['faultString'])
				self.retries = 0
				return data[0]
			raise DedimaniaTransportException('Invalid response from dedimania!')
		except (ConnectionError, ReadTimeout, ConnectionRefusedError, requests.exceptions.ConnectionError) as e:
			raise DedimaniaTransportException(e) from e
		except ConnectTimeout as e:
			raise DedimaniaTransportException(e) from e
		except DedimaniaTransportException:
			# Try to setup new session.
			self.retries += 1
			if self.retries > 5:
				raise DedimaniaTransportException('Dedimania didn\'t gave the right answer after few retries!')
			self.client = requests.session()
			try:
				await self.authenticate()
				return await self.execute(method, *args)
			except Exception as e:
				logger.error('XML-RPC Fault retrieved from Dedimania: {}'.format(str(e)))
				handle_exception(e, __name__, 'execute')
				raise DedimaniaTransportException('Could not retrieve data from dedimania!')
		except DedimaniaFault as e:
			if 'Bad SessionId' in e.faultString or ('SessionId' in e.faultString and 'not found' in e.faultString):
				try:
					self.retries += 1
					if self.retries > 5:
						raise DedimaniaTransportException('Max retries reached for reauthenticating with dedimania!')

					# Save original session ID.
					original_session_id = '{}'.format(self.session_id)

					# Reauthenticate
					await self.authenticate()

					# Replace session_id in args.
					if len(args) > 0 and len(args[0]) > 0 and isinstance(args[0][0], dict) and 'params' in args[0][0]:
						new_params = list(args[0][0]['params'])
						if len(new_params) > 0 and isinstance(new_params[0], str) and new_params[0] == original_session_id:
							new_params[0] = self.session_id
							args[0][0]['params'] = tuple(new_params)

					# Try again.
					return await self.execute(method, *args)
				except:
					return
			logger.error('XML-RPC Fault retrieved from Dedimania: {}'.format(str(e)))
			handle_exception(e, __name__, 'execute', extra_data={
				'dedimania_retries': self.retries,
			})
			raise DedimaniaTransportException('Could not retrieve data from dedimania!') 
Example 25
Project: hoaxy-backend   Author: IUNetSci   File: stream.py    License: GNU General Public License v3.0 4 votes vote down vote up
def stream(self):
        """The main function to handle twitter stream."""
        logger.info("Started streaming.")
        while True:
            try:
                self._authenticate()
                resp = self.client.post(
                    API_URL,
                    data=self.params,
                    stream=True,
                    timeout=self._stall_timeout)
                data_lines = 0  # includes keepalives
                # line is unicode
                for line in resp.iter_lines():
                    self.process_one_line(line)
                    data_lines += 1
                    if data_lines >= 8:
                        # reset backoff status if received at least 8 data
                        # lines (including keep-alive newlines). Stream
                        # seems to send at least 8 keepalives, regardless
                        # of whether authentication was successful or not.
                        logger.debug("Reset backoff")
                        self._reset_backoff()
                        data_lines = 0
                logger.warning("Backing off..")
                self._backoff('tcp')
            except requests.exceptions.ConnectTimeout:
                # wait just a (small) fixed amount of time and try to
                # reconnect.
                msg = "Timeout, retrying in %s.."
                logger.warning(msg, self._conn_timeout_sleep)
                time.sleep(self._conn_timeout_sleep)
            except requests.Timeout:
                # catching requests.Timeout instead of requests.ReadTimeout
                # because we are setting a timeout parameter in the POST
                msg = "Server did not send any data for %ss, backing off"
                logger.warning(msg, self._stall_timeout)
                self._backoff('tcp')
            except requests.ConnectionError:
                logger.warning("Reconnecting to stream endpoint...")
                self._backoff('tcp')
            except requests.HTTPError as err:
                if err.response.status_code == 420:
                    msg = "Got HTTP 420 Error. Backing off.."
                    logger.warning(msg)
                    self._backoff("http_420")
                else:
                    msg = "Got HTTP Error. Backing off.."
                    logger.warning(msg)
                    self._backoff("http")
            except socket.error as err:
                logger.warning('Got socket error: %s, reconnecting!', err)
                self._backoff('tcp')
            finally:
                # close the request connection
                logger.info('Request connection closed!')
                resp.close() 
Example 26
Project: integrations-core   Author: DataDog   File: elastic.py    License: BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def check(self, instance):
        config = from_instance(instance)
        admin_forwarder = config.admin_forwarder

        # Check ES version for this instance and define parameters
        # (URLs and metrics) accordingly
        try:
            version = self._get_es_version(config)
        except AuthenticationError:
            self.log.exception("The ElasticSearch credentials are incorrect")
            raise

        health_url, stats_url, pshard_stats_url, pending_tasks_url = self._get_urls(version, config.cluster_stats)
        stats_metrics = stats_for_version(version)
        pshard_stats_metrics = pshard_stats_for_version(version)

        # Load stats data.
        # This must happen before other URL processing as the cluster name
        # is retrieved here, and added to the tag list.
        stats_url = self._join_url(config.url, stats_url, admin_forwarder)
        stats_data = self._get_data(stats_url, config)
        if stats_data.get('cluster_name'):
            # retrieve the cluster name from the data, and append it to the
            # master tag list.
            cluster_name_tag = "cluster_name:{}".format(stats_data['cluster_name'])
            config.tags.append(cluster_name_tag)
            config.health_tags.append(cluster_name_tag)
        self._process_stats_data(stats_data, stats_metrics, config)

        # Load cluster-wise data
        # Note: this is a cluster-wide query, might TO.
        if config.pshard_stats:
            send_sc = bubble_ex = not config.pshard_graceful_to
            pshard_stats_url = self._join_url(config.url, pshard_stats_url, admin_forwarder)
            try:
                pshard_stats_data = self._get_data(pshard_stats_url, config, send_sc=send_sc)
                self._process_pshard_stats_data(pshard_stats_data, config, pshard_stats_metrics)
            except requests.ReadTimeout as e:
                if bubble_ex:
                    raise
                self.log.warning("Timed out reading pshard-stats from servers (%s) - stats will be missing", e)

        # Load the health data.
        health_url = self._join_url(config.url, health_url, admin_forwarder)
        health_data = self._get_data(health_url, config)
        self._process_health_data(health_data, config, version)

        if config.pending_task_stats:
            # Load the pending_tasks data.
            pending_tasks_url = self._join_url(config.url, pending_tasks_url, admin_forwarder)
            pending_tasks_data = self._get_data(pending_tasks_url, config)
            self._process_pending_tasks_data(pending_tasks_data, config)

        if config.index_stats and version >= [1, 0, 0]:
            try:
                self._get_index_metrics(config, admin_forwarder, version)
            except requests.ReadTimeout as e:
                self.log.warning("Timed out reading index stats from servers (%s) - stats will be missing", e)

        # If we're here we did not have any ES conn issues
        self.service_check(self.SERVICE_CHECK_CONNECT_NAME, AgentCheck.OK, tags=config.service_check_tags)