Python google.appengine.api.urlfetch.fetch() Examples

def _new_fixed_fetch(validate_certificate):

    def fixed_fetch(
        url,
        payload=None,
        method="GET",
        headers={},
        allow_truncated=False,
        follow_redirects=True,
        deadline=None,
    ):
        return fetch(
            url,
            payload=payload,
            method=method,
            headers=headers,
            allow_truncated=allow_truncated,
            follow_redirects=follow_redirects,
            deadline=deadline,
            validate_certificate=validate_certificate,
        )

    return fixed_fetch 

def _new_fixed_fetch(validate_certificate):

    def fixed_fetch(
        url,
        payload=None,
        method="GET",
        headers={},
        allow_truncated=False,
        follow_redirects=True,
        deadline=None,
    ):
        return fetch(
            url,
            payload=payload,
            method=method,
            headers=headers,
            allow_truncated=allow_truncated,
            follow_redirects=follow_redirects,
            deadline=deadline,
            validate_certificate=validate_certificate,
        )

    return fixed_fetch 

def _new_fixed_fetch(validate_certificate):

    def fixed_fetch(
        url,
        payload=None,
        method="GET",
        headers={},
        allow_truncated=False,
        follow_redirects=True,
        deadline=None,
    ):
        return fetch(
            url,
            payload=payload,
            method=method,
            headers=headers,
            allow_truncated=allow_truncated,
            follow_redirects=follow_redirects,
            deadline=deadline,
            validate_certificate=validate_certificate,
        )

    return fixed_fetch 

def gh_admin_top():
  stars = util.param('stars', int) or 10000
  page = util.param('page', int) or 1
  per_page = util.param('per_page', int) or 100
  # TODO: fix formatting
  result = urlfetch.fetch('https://api.github.com/search/repositories?q=stars:>=%s&sort=stars&order=asc&page=%d&per_page=%d' % (stars, page, per_page))
  if result.status_code == 200:
    repos = json.loads(result.content)
  else:
    flask.abort(result.status_code)

  for repo in repos['items']:
    account = repo['owner']
    account_db = model.Account.get_or_insert(
      account['login'],
      avatar_url=account['avatar_url'].split('?')[0],
      email=account['email'] if 'email' in account else '',
      name=account['login'],
      followers=account['followers'] if 'followers' in account else 0,
      organization=account['type'] == 'Organization',
      username=account['login'],
    )

  return 'OK %d of %d' % (len(repos['items']), repos['total_count']) 

def call_method(method, data):
  data.update({'access_token': TOKEN})
  data = urllib.urlencode(data)
  try:
    result = urlfetch.fetch(
        BASE_URL.format(method=method, qs=data),
        method=urlfetch.GET,
        deadline=10)
  except DeadlineExceededError as e:
    logging.exception(e)
    return None
  if result.status_code == 200:
    return json.loads(result.content).get('data')
  else:
    logging.error(result.content)
    return None 

def call_method(method, data):
  data.update({'key': TOKEN})
  data = json.dumps(data)
  try:
    result = urlfetch.fetch(
        BASE_URL.format(method=method, api_key=TOKEN),
        payload=data,
        method=urlfetch.POST,
        deadline=10,
        headers={'Content-Type': 'application/json'})
  except DeadlineExceededError as e:
    logging.exception(e)
    return None
  if result.status_code == 200:
    return json.loads(result.content)
  else:
    logging.error(result.content)
    return None 

def ClearDatastore(request):
    """Clears data in the datastore, many at a time (for admins only)."""
    clear = (None, 'None')
    atatime = 10

    msg = ''
    query = db.Query(clear[0])
    rows = query.fetch(atatime)
    length = len(rows)
    if length is 0:
        msg += 'No more rows to delete<br>'
    else:
        msg += 'Deleting %s %s<br>' % (length, clear[1])
        db.delete(rows)
        query = db.Query(clear[0])
        more = query.fetch(1)
        if len(more) is not 0:
            msg += 'Now do it again!'
            msg += '<script>window.location.href="/reflows/clear_datastore";</script>'
    return http.HttpResponse(msg) 

def request(self, method, url, headers, post_data=None):
        try:
            result = urlfetch.fetch(
                url=url,
                method=method,
                headers=headers,
                # Google App Engine doesn't let us specify our own cert bundle.
                # However, that's ok because the CA bundle they use recognizes
                # api.stripe.com.
                validate_certificate=self._verify_ssl_certs,
                # GAE requests time out after 60 seconds, so make sure we leave
                # some time for the application to handle a slow Stripe
                deadline=55,
                payload=post_data
            )
        except urlfetch.Error, e:
            self._handle_request_error(e, url) 

def _handle_request_error(self, e, url):
        if isinstance(e, urlfetch.InvalidURLError):
            msg = ("The Stripe library attempted to fetch an "
                   "invalid URL (%r). This is likely due to a bug "
                   "in the Stripe Python bindings. Please let us know "
                   "at support@stripe.com." % (url,))
        elif isinstance(e, urlfetch.DownloadError):
            msg = "There was a problem retrieving data from Stripe."
        elif isinstance(e, urlfetch.ResponseTooLargeError):
            msg = ("There was a problem receiving all of your data from "
                   "Stripe.  This is likely due to a bug in Stripe. "
                   "Please let us know at support@stripe.com.")
        else:
            msg = ("Unexpected error communicating with Stripe. If this "
                   "problem persists, let us know at support@stripe.com.")

        msg = textwrap.fill(msg) + "\n\n(Network error: " + str(e) + ")"
        raise error.APIConnectionError(msg) 

def send_request(fields):
    config = model.Config.get()

    fields["VERSION"] = "113"
    fields["USER"] =  config.paypal_user
    fields["PWD"] =  config.paypal_password
    fields["SIGNATURE"] = config.paypal_signature

    form_data = urllib.urlencode(fields)

    result = urlfetch.fetch(url=config.paypal_api_url, payload=form_data, method=urlfetch.POST,
                headers={'Content-Type': 'application/x-www-form-urlencoded'})
    result_map = urlparse.parse_qs(result.content)

    if 'ACK' in result_map:
        if result_map['ACK'][0] == "Success":
            return (True, result_map)
   
        logging.warning("Paypal returned an error:")
        logging.warning(pprint.pformat(result_map))
        return (False, result_map)

    logging.warning("Could not contact Paypal:")
    logging.warning(result.content)
    return False, result.content 

def push_important_news():
    key = request.values.get('key')
    news = ndb.Key(urlsafe=key).get()
    form_fields = {
        "token": app.config["PUSHOVER_APP_KEY"],
        "user": app.config["PUSHOVER_USER_KEY"],
        "message": news.summary.encode("utf-8"),
        "url": news.link.encode("utf-8"),
        "url_title": u"点击访问正文".encode("utf-8"),
        "title": news.title.encode("utf-8"),
    }
    form_data = urllib.urlencode(form_fields)
    urlfetch.fetch(url=app.config["PUSH_OVER_URL"],
                   payload=form_data,
                   method=urlfetch.POST,
                   headers={'Content-Type': 'application/x-www-form-urlencoded'},
                   follow_redirects=False,
                   validate_certificate=False)
    return "Done", 200 

def call_method(method, data):
  data = json.dumps(data)
  try:
    result = urlfetch.fetch(
        BASE_URL.format(token=TOKEN, method=method),
        payload=data,
        method=urlfetch.POST,
        deadline=10,
        headers={'Content-Type': 'application/json'})
  except DeadlineExceededError as e:
    logging.exception(e)
    return None
  if result.status_code == 200:
    return json.loads(result.content)
  else:
    logging.error(result.content)
    return None 

def activate(self, from_email, from_name, email, subject, text, html):
        # Some emails contain text/html some contain text/plain they should both be decoded by this point so we merge them to grep through the whole thing
        body = html + text
        activation_url = self.mailgrep(body)
        if activation_url:
            activation_response = urlfetch.fetch(url=activation_url, follow_redirects=True)
            if activation_response.status_code == 200 or activation_response.status_code == 302:
                logging.info("Activation URL successfully requested")
                # In most cases this code is all that is required to successfully activate an account.
                # In some cases registration is more complex than just requesting an activation URL.
                # Sometimes you have to submit your credentials
                # Sometimes there is a CSRF token
                # Sometimes it automatically logs you into the application
                # Or you want to send a notification to yourself on success
                # Here is an example of making additional request based on email sender
                # if from_email == 'thank.you.for.registerting@some.domain.com':
                #     make some other request to finish activation of some.domain.com
                # elif from_email == "thank.you.for.registering.with.us.instead.of.some.domain.com@obviously.the.cooler.domain.com":
                #     make some specific request to activate account on obviously.the.cooler.domain.com
                return True
            logging.error("Activation Failed")
        return False


#Handle emails directly sent to *@YourAppID.appspotmail.com 

def _new_fixed_fetch(validate_certificate):

    def fixed_fetch(
        url,
        payload=None,
        method="GET",
        headers={},
        allow_truncated=False,
        follow_redirects=True,
        deadline=None,
    ):
        return fetch(
            url,
            payload=payload,
            method=method,
            headers=headers,
            allow_truncated=allow_truncated,
            follow_redirects=follow_redirects,
            deadline=deadline,
            validate_certificate=validate_certificate,
        )

    return fixed_fetch 

def get(self):
        auth_token, _ = app_identity.get_access_token(
            'https://www.googleapis.com/auth/cloud-platform')
        logging.info(
            'Using token {} to represent identity {}'.format(
                auth_token, app_identity.get_service_account_name()))

        response = urlfetch.fetch(
            'https://www.googleapis.com/storage/v1/b?project={}'.format(
                app_identity.get_application_id()),
            method=urlfetch.GET,
            headers={
                'Authorization': 'Bearer {}'.format(auth_token)
            }
        )

        if response.status_code != 200:
            raise Exception(
                'Call failed. Status code {}. Body {}'.format(
                    response.status_code, response.content))

        result = json.loads(response.content)
        self.response.headers['Content-Type'] = 'application/json'
        self.response.write(json.dumps(result, indent=2)) 

def get_file_info(self, access_token, name):

		headers = {
			'Content-Type' : 'application/json',
			'Authorization' : 'Bearer ' + access_token
		}

		data = {
    		"path": "/" + name,
		    "include_media_info": False,
		    "include_deleted": False,
		    "include_has_explicit_shared_members": False
		}

		result = urlfetch.fetch(
			payload=json.dumps(data),
			method=urlfetch.POST,
			url='https://api.dropboxapi.com/2/files/get_metadata',
			headers=headers
		)

		if result.status_code != 200:
			raise Exception("Failed to get file metadata from Dropbox. Status: %s, body: %s" % (result.status_code, result.content))
		self.log(result.content)
		return json.loads(result.content) 

def _handle_request_error(self, e, url):
        if isinstance(e, urlfetch.InvalidURLError):
            msg = ("The Shippo library attempted to fetch an "
                   "invalid URL (%r). This is likely due to a bug "
                   "in the Shippo Python bindings. Please let us know "
                   "at support@goshippo.com." % (url,))
        elif isinstance(e, urlfetch.DownloadError):
            msg = "There was a problem retrieving data from Shippo."
        elif isinstance(e, urlfetch.ResponseTooLargeError):
            msg = ("There was a problem receiving all of your data from "
                   "Shippo.  This is likely due to a bug in Shippo. "
                   "Please let us know at support@goshippo.com.")
        else:
            msg = ("Unexpected error communicating with Shippo. If this "
                   "problem persists, let us know at support@goshippo.com.")

        msg = textwrap.fill(msg) + "\n\n(Network error: " + str(e) + ")"
        raise error.APIConnectionError(msg) 

def request(self, method, url, headers, post_data=None):
        try:
            result = urlfetch.fetch(
                url=url,
                method=method,
                headers=headers,
                # Google App Engine doesn't let us specify our own cert bundle.
                # However, that's ok because the CA bundle they use recognizes
                # api.goshippo.com.
                validate_certificate=self._verify_ssl_certs,
                # GAE requests time out after 60 seconds, so make sure we leave
                # some time for the application to handle a slow Shippo
                deadline=55,
                payload=post_data
            )
        except urlfetch.Error as e:
            self._handle_request_error(e, url)

        return result.content, result.status_code 

def _new_fixed_fetch(validate_certificate):

    def fixed_fetch(
        url,
        payload=None,
        method="GET",
        headers={},
        allow_truncated=False,
        follow_redirects=True,
        deadline=None,
    ):
        return fetch(
            url,
            payload=payload,
            method=method,
            headers=headers,
            allow_truncated=allow_truncated,
            follow_redirects=follow_redirects,
            deadline=deadline,
            validate_certificate=validate_certificate,
        )

    return fixed_fetch 

def get(self):
    util.EnableCors(self)

    WP_PLEDGES = 4099
    VERSION_12_AND_UNDER = 59009 

    count = memcache.get('TOTAL-PLEDGES')
    if not count:
      query = model.Pledge.all(keys_only=True).filter('model_version >', 12)
      i = 0
      while True:
          result = query.fetch(1000)
          i = i + len(result)
          if len(result) < 1000:
              break
          cursor = query.cursor()
          query.with_cursor(cursor)
      count = i + WP_PLEDGES + VERSION_12_AND_UNDER
      memcache.set('TOTAL-PLEDGES', count, 120)

    self.response.headers['Content-Type'] = 'application/json'
    json.dump({'count':count}, self.response) 

def set_web_client_id(web_client_id):
  """Changes the configured OAuth2 client ID for the web UI."""
  cfg = AuthWebUIConfig.fetch() or AuthWebUIConfig()
  cfg.modify(
      updated_by=get_current_identity().to_bytes(),
      web_client_id=web_client_id)


################################################################################
## OAuth token check. 

def _initialize_auth_db_cache():
  """Initializes auth runtime and _auth_db in particular.

  Must be called under _auth_db_lock.
  """
  global _auth_db
  global _auth_db_expiration

  assert _auth_db is None
  logging.info('Initial fetch of AuthDB')
  _auth_db = fetch_auth_db()
  _auth_db_expiration = time.time() + _process_cache_expiration_sec
  logging.info('Fetched AuthDB at rev %d', _auth_db.auth_db_rev)

  return _auth_db 

def _initialize_auth_db_cache():
  """Initializes auth runtime and _auth_db in particular.

  Must be called under _auth_db_lock.
  """
  global _auth_db
  global _auth_db_expiration

  assert _auth_db is None
  logging.info('Initial fetch of AuthDB')
  _auth_db = fetch_auth_db()
  _auth_db_expiration = time.time() + _process_cache_expiration_sec
  logging.info('Fetched AuthDB at rev %d', _auth_db.auth_db_rev)

  return _auth_db 

def _fetch_service_certs(service_url):
  protocol = 'https://'
  if utils.is_local_dev_server():
    protocol = ('http://', 'https://')
  assert service_url.startswith(protocol), (service_url, protocol)
  url = '%s/auth/api/v1/server/certificates' % service_url

  # Retry code is adapted from components/net.py. net.py can't be used directly
  # since it depends on components.auth (and dependency cycles between
  # components are bad).
  attempt = 0
  result = None
  while attempt < 4:
    if attempt:
      logging.info('Retrying...')
    attempt += 1
    logging.info('GET %s', url)
    try:
      result = urlfetch.fetch(
          url=url,
          method='GET',
          headers={'X-URLFetch-Service-Id': utils.get_urlfetch_service_id()},
          follow_redirects=False,
          deadline=5,
          validate_certificate=True)
    except (apiproxy_errors.DeadlineExceededError, urlfetch.Error) as e:
      # Transient network error or URL fetch service RPC deadline.
      logging.warning('GET %s failed: %s', url, e)
      continue
    # It MUST return 200 on success, it can't return 403, 404 or >=500.
    if result.status_code != 200:
      logging.warning(
          'GET %s failed, HTTP %d: %r', url, result.status_code, result.content)
      continue
    return json.loads(result.content)

  # All attempts failed, give up.
  msg = 'Failed to grab public certs from %s (HTTP code %s)' % (
      service_url, result.status_code if result else '???')
  raise CertificateError(msg, transient=True) 

def new_auth_db_snapshot():
  """Makes a consistent snapshot of replicated subset of AuthDB entities.

  Returns:
    Tuple (AuthReplicationState, AuthDBSnapshot).
  """
  # Start fetching stuff in parallel.
  state_future = model.replication_state_key().get_async()
  config_future = model.root_key().get_async()
  groups_future = model.AuthGroup.query(ancestor=model.root_key()).fetch_async()
  realms_globals_future = model.realms_globals_key().get_async()
  project_realms_future = model.AuthProjectRealms.query(
      ancestor=model.root_key()).fetch_async()

  # It's fine to block here as long as it's the last fetch.
  ip_whitelist_assignments, ip_whitelists = model.fetch_ip_whitelists()

  snapshot = AuthDBSnapshot(
      config_future.get_result() or model.AuthGlobalConfig(
          key=model.root_key()
      ),
      groups_future.get_result(),
      ip_whitelists,
      ip_whitelist_assignments,
      realms_globals_future.get_result() or model.AuthRealmsGlobals(
          key=model.realms_globals_key()
      ),
      project_realms_future.get_result(),
  )
  return state_future.get_result(), snapshot 

def call_method(method):
  result = urlfetch.fetch(BASE_URL.format(method=method), deadline=10)
  if result.status_code == 200:
    return json.loads(result.content)
  return None


# Use a helper function to define the scope of the callback. 

def _fetch_service_certs(service_url):
  protocol = 'https://'
  if utils.is_local_dev_server():
    protocol = ('http://', 'https://')
  assert service_url.startswith(protocol), (service_url, protocol)
  url = '%s/auth/api/v1/server/certificates' % service_url

  # Retry code is adapted from components/net.py. net.py can't be used directly
  # since it depends on components.auth (and dependency cycles between
  # components are bad).
  attempt = 0
  result = None
  while attempt < 4:
    if attempt:
      logging.info('Retrying...')
    attempt += 1
    logging.info('GET %s', url)
    try:
      result = urlfetch.fetch(
          url=url,
          method='GET',
          headers={'X-URLFetch-Service-Id': utils.get_urlfetch_service_id()},
          follow_redirects=False,
          deadline=5,
          validate_certificate=True)
    except (apiproxy_errors.DeadlineExceededError, urlfetch.Error) as e:
      # Transient network error or URL fetch service RPC deadline.
      logging.warning('GET %s failed: %s', url, e)
      continue
    # It MUST return 200 on success, it can't return 403, 404 or >=500.
    if result.status_code != 200:
      logging.warning(
          'GET %s failed, HTTP %d: %r', url, result.status_code, result.content)
      continue
    return json.loads(result.content)

  # All attempts failed, give up.
  msg = 'Failed to grab public certs from %s (HTTP code %s)' % (
      service_url, result.status_code if result else '???')
  raise CertificateError(msg, transient=True) 

def new_auth_db_snapshot():
  """Makes a consistent snapshot of replicated subset of AuthDB entities.

  Returns:
    Tuple (AuthReplicationState, AuthDBSnapshot).
  """
  # Start fetching stuff in parallel.
  state_future = model.replication_state_key().get_async()
  config_future = model.root_key().get_async()
  groups_future = model.AuthGroup.query(ancestor=model.root_key()).fetch_async()
  realms_globals_future = model.realms_globals_key().get_async()
  project_realms_future = model.AuthProjectRealms.query(
      ancestor=model.root_key()).fetch_async()

  # It's fine to block here as long as it's the last fetch.
  ip_whitelist_assignments, ip_whitelists = model.fetch_ip_whitelists()

  snapshot = AuthDBSnapshot(
      config_future.get_result() or model.AuthGlobalConfig(
          key=model.root_key()
      ),
      groups_future.get_result(),
      ip_whitelists,
      ip_whitelist_assignments,
      realms_globals_future.get_result() or model.AuthRealmsGlobals(
          key=model.realms_globals_key()
      ),
      project_realms_future.get_result(),
  )
  return state_future.get_result(), snapshot 

def get_latest_auth_db():
  """Returns the most recent AuthDB instance, fetching it if necessary.

  Very heavy call. If the absolute consistency is not required, prefer to use
  get_process_auth_db instead. The later is much faster by relying on in-process
  cache (as a downside it may lag behind the most recent state).
  """
  # We just "rush" the update of the internal cache. That way get_latest_auth_db
  # blocks for long only if something in AuthDB has changed, i.e our cached copy
  # becomes stale. By reusing _auth_db (instead of keeping a separate cache or
  # something like that), we keep the memory footprint smaller.
  #
  # Also, to avoid fetching heavy AuthDB objects concurrently (and thus causing
  # OOM), we do the entire transaction under the lock. We can't reuse
  # _auth_db_lock, since it must not be locked for a long time (it would break
  # performance guarantees of 'get_process_auth_db'). We guard everything with
  # _auth_db_fetch_lock (instead of just 'fetch_auth_db') to make sure that once
  # it gets unlocked, waiting threads quickly discover that '_auth_db' is
  # already fresh.
  with _auth_db_fetch_lock:
    # Not using cache at all (usually in tests) => always fetch.
    if not _process_cache_expiration_sec:
      return fetch_auth_db()

    cached = None
    with _auth_db_lock:
      if _auth_db is None:
        return _initialize_auth_db_cache()
      cached = _auth_db

    fetched = fetch_auth_db(known_auth_db=cached)

    with _auth_db_lock:
      return _roll_auth_db_cache(fetched) 

def get_latest_auth_db():
  """Returns the most recent AuthDB instance, fetching it if necessary.

  Very heavy call. If the absolute consistency is not required, prefer to use
  get_process_auth_db instead. The later is much faster by relying on in-process
  cache (as a downside it may lag behind the most recent state).
  """
  # We just "rush" the update of the internal cache. That way get_latest_auth_db
  # blocks for long only if something in AuthDB has changed, i.e our cached copy
  # becomes stale. By reusing _auth_db (instead of keeping a separate cache or
  # something like that), we keep the memory footprint smaller.
  #
  # Also, to avoid fetching heavy AuthDB objects concurrently (and thus causing
  # OOM), we do the entire transaction under the lock. We can't reuse
  # _auth_db_lock, since it must not be locked for a long time (it would break
  # performance guarantees of 'get_process_auth_db'). We guard everything with
  # _auth_db_fetch_lock (instead of just 'fetch_auth_db') to make sure that once
  # it gets unlocked, waiting threads quickly discover that '_auth_db' is
  # already fresh.
  with _auth_db_fetch_lock:
    # Not using cache at all (usually in tests) => always fetch.
    if not _process_cache_expiration_sec:
      return fetch_auth_db()

    cached = None
    with _auth_db_lock:
      if _auth_db is None:
        return _initialize_auth_db_cache()
      cached = _auth_db

    fetched = fetch_auth_db(known_auth_db=cached)

    with _auth_db_lock:
      return _roll_auth_db_cache(fetched)