Python grequests.get() Examples

def _get_async(self, keys, frmt='json', params={}):
        # does not work under pyhon3 so local import
        import grequests
        session = self._get_session()
        try:
            # build the requests
            urls = self._get_all_urls(keys, frmt)
            self.logging.debug("grequests.get processing")
            rs = (grequests.get(url, session=session, params=params)  for key,url in zip(keys, urls))
            # execute them
            self.logging.debug("grequests.map call")
            ret = grequests.map(rs, size=min(self.settings.CONCURRENT, len(keys)))
            self.last_response = ret
            self.logging.debug("grequests.map call done")
            return ret
        except Exception as err:
            self.logging.warning("Error caught in async. " + err.message)
            return [] 

def get_image_urls(girl_urls):

    girl_list = []

    # 建立5个并发连接
    rs = (grequests.get(url) for url in girl_urls)
    responses = grequests.map(rs, size = 5)

    for response in responses:
        parsed_body = html.fromstring(response.text)
        girl_title  = parsed_body.xpath('//title/text()')
        image_urls = parsed_body.xpath('//li[@class="slide "]/img/@src | //li[@class="slide "]/img/@delay')

        # print image_urls
        girl_dict = {girl_title[0] : image_urls}
        girl_list.append(girl_dict)
    
    print "get_girl_urls done!!!"
    return girl_list 

def get_url(url, abort_on_error=False, is_json=True, fetch_timeout=5, auth=None, post_data=None):
    """
    @param post_data: If not None, do a POST request, with the passed data (which should be in the correct string format already)
    """
    headers = {'Connection': 'close', }  # no keepalive
    if auth:
        # auth should be a (username, password) tuple, if specified
        headers['Authorization'] = http_basic_auth_str(auth[0], auth[1])

    try:
        if post_data is not None:
            if is_json:
                headers['content-type'] = 'application/json'
            r = grequests.map((grequests.post(url, data=post_data, timeout=fetch_timeout, headers=headers, verify=False),))[0]
        else:
            r = grequests.map((grequests.get(url, timeout=fetch_timeout, headers=headers, verify=False),))[0]
        if r is None:
            raise Exception("result is None")
    except Exception as e:
        raise Exception("Got get_url request error: %s" % e)
    else:
        if r.status_code != 200 and abort_on_error:
            raise Exception("Bad status code returned: '%s'. result body: '%s'." % (r.status_code, r.text))
    return r.json() if r.text and is_json else r.text 

def parse_and_print_data(data, metric, factors=None, filter=None,
                         template=None, template_params=None):
    if template_params is None:
        template_params = {}

    if not factors:
        factors = {'Tbps': 1000000000000, 'Gbps': 1000000000, 'Mbps': 1000000,
                   'Kbps': 1000, 'Bps': 1}

    factor = parse_factor(
        data.get('meta').get('yValueFormatString'),
        factors
    )
    for traffic in data.get('data'):
        timestamp = int(traffic.get('x') / 1000)

        if not filter(timestamp):
            continue

        value = abs(traffic.get('y')) * factor
        print(template.format(
            metric=metric, value=value, time=timestamp, **template_params)
        ) 

def get_responses(payloads):
        """

        Args:
          payloads(list): list of payloads for GET request

        Returns:
          list: list of dictionaries with data from JSON responses

        """
        batch_url = "{base}stock/market/batch?".format(base=iex_url_base)
        rs = (grequests.get(batch_url, params=payload) for payload in payloads)
        result = grequests.map(rs)
        try:
            outputs = [r.json() for r in result]
        except AttributeError:
            outputs = []
        return outputs 

def filter_projects_by_license(projects, headers, licenses):
    import grequests

    reqs = [grequests.get(constants.REPO_URL.format(full_name=p.full_name), headers=headers)
            for p in projects]
    resps = grequests.map(reqs, exception_handler=request_exception_handler)

    filtered_projects = []
    for i, project in enumerate(projects):
        resp = resps[i]
        if not resp or resp.status_code != 200:
            logging.warning("ignoring %s because no info could be fetched", project.full_name)
            continue

        project_license = resp.json().get("license")
        if not project_license or not project_license.get("spdx_id"):
            continue
        license_id = project_license.get("spdx_id")
        if license_id in licenses:
            project.license = license_id
            filtered_projects.append(project)
    return filtered_projects 

def get_data(self):
        idxs = np.arange(len(self.train_list))
        if self.shuffle:
            self.rng.shuffle(idxs)

        caches = {}
        for i, k in enumerate(idxs):
            path = self.train_list[k]
            label = self.lb_list[k]

            if i % self.preload == 0:
                try:
                    caches = ILSVRCTenth._read_tenth_batch(self.train_list[idxs[i:i+self.preload]])
                except Exception as e:
                    logging.warning('tenth local cache failed, err=%s' % str(e))

            content = caches.get(path, '')
            if not content:
                content = ILSVRCTenth._read_tenth(path)

            img = cv2.imdecode(np.fromstring(content, dtype=np.uint8), cv2.IMREAD_COLOR)
            yield [img, label] 

def scan_process(q_targets, q_results, args, target_process_done):
    reload(socket)
    signal.signal(signal.SIGINT, exit_func)
    s = Scanner(q_results, args.timeout * 60, args=args)
    while True:
        try:
            target = q_targets.get(timeout=0.2)
        except Exception as e:
            if target_process_done.value:
                break
            else:
                continue

        if 'target' in target:
            ret = s.init_from_url(target['target'])
        elif 'file' in target:
            ret = s.init_from_log_file(target['file'])
        else:
            continue

        if ret:
            host, results = s.scan(threads=args.t)
            if results:
                q_results.put((host, results)) 

def Get(*args, **kwargs) -> requests.models.Response:
        """Send a GET request.

        Args:
            *args: arguments for requests.get(); particularly the URL
            **kwargs: optional arguments for requests.get()

        Returns:
            requests.models.Response: response
        """
        return requests.get(
            *args,
            headers=Net.HEADERS,
            timeout=float(castero.config.Config['request_timeout']),
            proxies={
                'http': castero.config.Config['proxy_http'],
                'https': castero.config.Config['proxy_https'],
            },
            **kwargs
        ) 

def test_017_test_scan_all(self):
        """ Test issuing a scan all command in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/scan'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def http_request(self, url, headers=config.default_headers, timeout=20):
        try:
            if not url:
                url = '/'
            if not self.conn_pool:
                return -1, {}, ''
            if self.args.debug:
                self.print_msg('--> %s' % self.base_url + url)
            resp = self.conn_pool.urlopen('GET', self.base_url + url,
                                          headers=headers, assert_same_host=False,
                                          redirect=False, timeout=timeout, retries=0)
            if resp.headers.get('content-type', '').find('text') >= 0 \
                    or resp.headers.get('content-type', '').find('html') >= 0 \
                    or int(resp.headers.get('content-length', '0')) <= 20480:  # 1024 * 20
                html_doc = decode_response_text(resp.data)
            else:
                html_doc = ''

            if resp.status == 502:    # 502出现3次以上，排除该站点
                self.status_502_count += 1
                if self.status_502_count > 3:
                    self.url_queue.queue.clear()
                    try:
                        if self.conn_pool:
                            self.conn_pool.close()
                    except Exception as e:
                        pass
                    self.conn_pool = None
                    # self.print_msg('Website 502: %s' % self.base_url)

            return resp.status, resp.headers, html_doc
        except urllib3.exceptions.MaxRetryError as e:
            return -1, {}, ''
        except TypeError as e:
            return -1, {}, ''
        except Exception as e:
            self.print_msg(str(e))
            return -1, {}, ''

    # check existence of status 404 

def getpost(uid, query_urls):
    import requests
    url = 'http://%s.tumblr.com/api/read?&num=50' % uid
    r = requests.get(url)
    total = re.findall('<posts start="0" total="(.*?)">', r.content)[0]
    total = int(total)
    print uid + ':' + str(total)
    a = [i * 50 for i in range(total / 50 + 1)]
    ul = api_url % uid
    for i in a:
        query_url = ul + str(i)
        query_urls.append(query_url) 

def retrieve_sequentially(ladder_id, force_redownload=False):
    ''' Makes 75 API requests to retrieve all 15,000 players in the given
    ladder. The average latency of a single response from the API is around
    1000ms, so this function takes about 75 seconds to complete.

    A better option is to retrieve the ladder concurrently with retrieve().

    :param ladder_id: The id (name) of the league for the ladder you want
        to retrieve.
    :param ladder_id: Name of the ladder to retrive (league name)
    :return: A list of all player details in the ladder
    '''
    entries = []
    ladder_size = 15000
    offset = 0
    limit = 200

    for _ in xrange(0, ladder_size, limit):
        p = pathofexile.api.get_ladder_segment(
            ladder_id,
            ladder_limit=limit,
            ladder_offset=offset
        )
        entries.extend(p.get('entries'))
        offset += limit

    return entries 

def crawl(self, path, do_not_process_links=False):
        try:
            # increase body size to 200 KB
            headers = dict(config.default_headers, Range='bytes=0-204800')
            status, headers, html_doc = self.http_request(path, headers=headers)
            if path == '/':
                self.index_status, self.index_headers, self.index_html_doc = status, headers, html_doc
            if not self.args.no_crawl and not do_not_process_links and html_doc:
                soup = BeautifulSoup(html_doc, "html.parser")
                for link in soup.find_all('a'):
                    url = link.get('href', '').strip()
                    if url.startswith('..'):
                        continue
                    if not url.startswith('/') and url.find('//') < 0:   # relative path
                        url = path + url
                    url, depth = cal_depth(self, url)
                    # print url, depth
                    if depth <= self.max_depth:
                        self.enqueue(url)
                #
                ret = self.find_text(html_doc)
                if ret:
                    if '/' not in self.results:
                        self.results['/'] = []
                    m = re.search('<title>(.*?)</title>', html_doc)
                    title = m.group(1) if m else ''
                    _ = {'status': status, 'url': '%s%s' % (self.base_url, path), 'title': title, 'vul_type': ret[1]}
                    if _ not in self.results['/']:
                        self.results['/'].append(_)

        except Exception as e:
            self.print_msg('[crawl Exception] %s %s' % (path, str(e)))

    # 

def get(self, url, params={}, headers=None, format_='json'):
        """Calls the get method for a REST endpoint. 

        Args:
            url (str): URL of endpoint
            params (url): params for endpoint
            headers (dict, optional): any additional header attrs. Default to None
            format_ (str, optional): Format requested. Default to json

        Returns:
            dict: Response of request if format is json
            str: Response of request if format is not json
        """
        try:
            greq = grequests.get(url, params=params, headers=headers)
            response = grequests.map([greq])[0]
        except requests.exceptions.ConnectionError:
            print 'ConnectionError'
            return None
        self.info(response.url, _format=False)

        if format_ == 'json':
            try:
                json_data = response.json()
            except ValueError:
                json_data = None
                self.warn('Could not decode json properly')
            return json_data
        else:
            return response.text 

def format_log(self, format_str, *args, **kwargs):
        """Formats a string for logging"""

        if len(args) + len(kwargs) == 0:
            return format_str
        if kwargs.get('_format') is False:
            return format_str
        return format_str.format(*args, **kwargs) 

def test_020_test_scan(self):
        """ Test scanning a board in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/scan/rack_1/40000000'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_019_test_scan(self):
        """ Test scanning a rack in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/scan/rack_1'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_018_test_scan_all(self):
        """ Test issuing a force scan all command in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/scan/force'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def get(self, url):
        print(url)
        return grequests.get(url) 

def test_010_test_boot_target(self):
        """ Test setting boot target info in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/boot_target/rack_1/40000000/0200/pxe'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_015_test_fan(self):
        """ Test getting fan speed in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/fan/rack_1/40000000/0042'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_015_test_led(self):
        """ Test setting led state (off) in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/led/rack_1/40000000/0300/off'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_014_test_led(self):
        """ Test setting led state (on) in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/led/rack_1/40000000/0300/on'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_006_test_power(self):
        """ Test power control (off) in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/power/rack_1/40000000/0100/off'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_007_test_power(self):
        """ Test power control (on) in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/power/rack_1/40000000/0100/on'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_013_test_led(self):
        """ Test getting led state in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/led/rack_1/40000000/0300'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_008_test_asset_info(self):
        """ Test getting asset info in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/asset/rack_1/40000000/0200'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_012_test_location(self):
        """ Test getting location info in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/location/rack_1/40000000'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200) 

def test_009_test_boot_target(self):
        """ Test getting boot target info in IPMI mode.
        """
        responses = []

        def _test_hook(r, **kwargs):
            responses.append(r.status_code)

        url = PREFIX + '/boot_target/rack_1/40000000/0200'
        async.map([async.get(url, hooks={'response': [_test_hook]}, timeout=self.TIMEOUT) for _ in xrange(self.COUNT)], size=self.SIZE)

        self.assertEqual(len(responses), self.COUNT)
        for resp in responses:
            self.assertEqual(resp, 200)