com.google.appengine.api.urlfetch.FetchOptions Java Examples

public static String getResponseDELETE(String url, Map<String, String> params, Map<String, String> headers) {
	int retry = 0;
	while (retry < 3) {
		long start = System.currentTimeMillis();
		try {
			URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
			String urlStr = ToolString.toUrl(url.trim(), params);
			logger.debug("DELETE : " + urlStr);
			HTTPRequest httpRequest = new HTTPRequest(new URL(urlStr), HTTPMethod.DELETE, FetchOptions.Builder.withDeadline(deadline));
			HTTPResponse response = fetcher.fetch(httpRequest);
			return processResponse(response);
		} catch (Throwable e) {
			retry++;
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else if (retry < 3) {
				logger.warn("retrying after " + (System.currentTimeMillis() - start) + " and " + retry + " retries\n" + e.getClass() + e.getMessage());
			} else {
				logger.error(e.getClass() + "\n" + ToolString.stack2string(e));
			}
		}
	}
	return null;
}
 

public static String getResponseProxyPOST(URL url) throws MalformedURLException, UnsupportedEncodingException, IOException {
	URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
	String base64payload = "base64url=" + Base64UrlSafe.encodeServer(url.toString());
	String urlStr = url.toString();
	if (urlStr.contains("?")) {
		base64payload = "base64url=" + Base64UrlSafe.encodeServer(urlStr.substring(0, urlStr.indexOf("?")));
		base64payload += "&base64content=" + Base64UrlSafe.encodeServer(urlStr.substring(urlStr.indexOf("?") + 1));
	} else {
		base64payload = "base64url=" + Base64UrlSafe.encodeServer(urlStr);
	}
	HTTPRequest httpRequest = new HTTPRequest(new URL(HttpClientGAE.POSTPROXY_PHP), HTTPMethod.POST, FetchOptions.Builder.withDeadline(30d).doNotValidateCertificate());
	httpRequest.setPayload(base64payload.getBytes(UTF8));
	HTTPResponse response = fetcher.fetch(httpRequest);
	String processResponse = HttpClientGAE.processResponse(response);
	logger.info("proxying " + url + "\nprocessResponse:" + processResponse);
	return processResponse;
}
 

private HttpResponse fetch(String url, HttpRequestOptions requestOptions,
    HTTPMethod method, String content) throws IOException {

  final FetchOptions options = getFetchOptions(requestOptions);

  String currentUrl = url;

  for (int i = 0; i <= requestOptions.getMaxRedirects(); i++) {

    HTTPRequest httpRequest = new HTTPRequest(new URL(currentUrl),
        method, options);

    addHeaders(httpRequest, requestOptions);

    if (method == HTTPMethod.POST && content != null) {
      httpRequest.setPayload(content.getBytes());
    }

    HTTPResponse httpResponse;
    try {
      httpResponse = fetchService.fetch(httpRequest);
    } catch (ResponseTooLargeException e) {
      return new TooLargeResponse(currentUrl);
    }

    if (!isRedirect(httpResponse.getResponseCode())) {
      boolean isResponseTooLarge =
        (getContentLength(httpResponse) > requestOptions.getMaxBodySize());
      return new AppEngineFetchResponse(httpResponse,
          isResponseTooLarge, currentUrl);
    } else {
      currentUrl = getResponseHeader(httpResponse, "Location").getValue();
    }
  }
  throw new IOException("exceeded maximum number of redirects");
}
 

@Override
protected UrlFetchRequest buildRequest(String method, String url) throws IOException {
  Preconditions.checkArgument(supportsMethod(method), "HTTP method %s not supported", method);
  HTTPMethod httpMethod;
  if (method.equals(HttpMethods.DELETE)) {
    httpMethod = HTTPMethod.DELETE;
  } else if (method.equals(HttpMethods.GET)) {
    httpMethod = HTTPMethod.GET;
  } else if (method.equals(HttpMethods.HEAD)) {
    httpMethod = HTTPMethod.HEAD;
  } else if (method.equals(HttpMethods.POST)) {
    httpMethod = HTTPMethod.POST;
  } else if (method.equals(HttpMethods.PATCH)) {
    httpMethod = HTTPMethod.PATCH;
  } else {
    httpMethod = HTTPMethod.PUT;
  }
  // fetch options
  FetchOptions fetchOptions =
      FetchOptions.Builder.doNotFollowRedirects().disallowTruncate().validateCertificate();
  switch (certificateValidationBehavior) {
    case VALIDATE:
      fetchOptions.validateCertificate();
      break;
    case DO_NOT_VALIDATE:
      fetchOptions.doNotValidateCertificate();
      break;
    default:
      break;
  }
  return new UrlFetchRequest(fetchOptions, httpMethod, url);
}
 

@VisibleForTesting
HTTPRequest makeRequest(GcsFilename filename, @Nullable Map<String, String> queryStrings,
    HTTPMethod method, long timeoutMillis, byte[] payload) {
  HTTPRequest request = new HTTPRequest(makeUrl(filename, queryStrings), method,
      FetchOptions.Builder.disallowTruncate()
          .doNotFollowRedirects()
          .validateCertificate()
          .setDeadline(timeoutMillis / 1000.0));
  for (HTTPHeader header : headers) {
    request.addHeader(header);
  }
  request.addHeader(USER_AGENT);
  if (payload != null && payload.length > 0) {
    request.setHeader(new HTTPHeader(CONTENT_LENGTH, String.valueOf(payload.length)));
    try {
      request.setHeader(new HTTPHeader(CONTENT_MD5,
          BaseEncoding.base64().encode(MessageDigest.getInstance("MD5").digest(payload))));
    } catch (NoSuchAlgorithmException e) {
      log.severe(
          "Unable to get a MessageDigest instance, no Content-MD5 header sent.\n" + e.toString());
    }
    request.setPayload(payload);
  } else {
    request.setHeader(ZERO_CONTENT_LENGTH);
  }
  return request;
}
 

@Test
public void testFollowRedirects() throws Exception {
    final URL redirect = getUrl("redirect");
    FetchOptions options = buildFetchOptions();
    options.followRedirects();
    testOptions(redirect, options, new ResponseHandler() {
        public void handle(HTTPResponse response) throws Exception {
            URL finalURL = response.getFinalUrl();
            Assert.assertEquals(getUrl(""), finalURL);
        }
    });
}
 

@Test
public void testDoNotFollowRedirects() throws Exception {
    final URL redirect = getUrl("redirect");
    FetchOptions options = buildFetchOptions();
    options.doNotFollowRedirects();
    testOptions(redirect, options, new ResponseHandler() {
        public void handle(HTTPResponse response) throws Exception {
            Assert.assertEquals(302, response.getResponseCode());
        }
    });
}
 

@Test(expected = IOException.class)
public void fetchNonExistentLocalAppThrowsException() throws Exception {
    URL selfURL = new URL("BOGUS-" + appUrlBase + "/");
    FetchOptions fetchOptions = FetchOptions.Builder.withDefaults()
        .doNotFollowRedirects()
        .setDeadline(10.0);
    HTTPRequest httpRequest = new HTTPRequest(selfURL, HTTPMethod.GET, fetchOptions);
    URLFetchService urlFetchService = URLFetchServiceFactory.getURLFetchService();
    HTTPResponse httpResponse = urlFetchService.fetch(httpRequest);
    fail("expected exception, got " + httpResponse.getResponseCode());
}
 

@Test
public void testGetters() throws Exception {
    HTTPRequest request = new HTTPRequest(getFetchUrl(), HTTPMethod.PATCH, FetchOptions.Builder.withDefaults());
    request.addHeader(new HTTPHeader("foo", "bar"));
    request.setPayload("qwerty".getBytes());

    Assert.assertEquals(getFetchUrl(), request.getURL());
    Assert.assertEquals(HTTPMethod.PATCH, request.getMethod());
    Assert.assertNotNull(request.getFetchOptions());
    Assert.assertNotNull(request.getHeaders());
    Assert.assertEquals(1, request.getHeaders().size());
    assertEquals(new HTTPHeader("foo", "bar"), request.getHeaders().get(0));
    Assert.assertArrayEquals("qwerty".getBytes(), request.getPayload());
}
 

@Test
public void testFollowRedirects() throws Exception {
    final URL redirect = getUrl("redirect");
    FetchOptions options = FetchOptions.Builder.followRedirects();
    testOptions(redirect, options, new ResponseHandler() {
        public void handle(HTTPResponse response) throws Exception {
            URL finalURL = response.getFinalUrl();
            Assert.assertEquals(getUrl(""), finalURL);
        }
    });
}
 

@Test
public void testFollowRedirectsExternal() throws Exception {
    final URL redirectUrl = new URL("http://google.com/");
    final String expectedDestinationURLPrefix = "http://www.google.";

    FetchOptions options = FetchOptions.Builder.followRedirects();

    HTTPRequest request = new HTTPRequest(redirectUrl, HTTPMethod.GET, options);
    URLFetchService service = URLFetchServiceFactory.getURLFetchService();
    HTTPResponse response = service.fetch(request);
    String destinationUrl = response.getFinalUrl().toString();
    assertTrue("Did not get redirected.", destinationUrl.startsWith(expectedDestinationURLPrefix));
}
 

@Test
public void testDoNotFollowRedirects() throws Exception {
    final URL redirect = getUrl("redirect");
    FetchOptions options = FetchOptions.Builder.doNotFollowRedirects();
    testOptions(redirect, options, new ResponseHandler() {
        public void handle(HTTPResponse response) throws Exception {
            Assert.assertEquals(302, response.getResponseCode());
        }
    });
}
 

/**
 * Returns a new instance of the default {@link FetchOptions} used by this requestor. This
 * method exists primarily since {@link FetchOptions} provides no copy methods.
 *
 * @return new instance of default fetch options used by this requestor.
 */
public static FetchOptions newDefaultOptions() {
    return FetchOptions.Builder.withDefaults()
        .validateCertificate()
        .doNotFollowRedirects()
        .disallowTruncate()
        // seconds
        .setDeadline((DEFAULT_CONNECT_TIMEOUT_MILLIS + DEFAULT_READ_TIMEOUT_MILLIS)/1000.0);
}
 

public static String getResponsePUT(String url, Map<String, String> params, String json, Map<String, String> headers) {
	int retry = 0;
	while (retry < 3) {
		long start = System.currentTimeMillis();
		try {
			URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
			String urlStr = ToolString.toUrl(url.trim(), params);
			logger.debug("PUT : " + urlStr);
			HTTPRequest httpRequest = new HTTPRequest(new URL(urlStr), HTTPMethod.PUT, FetchOptions.Builder.withDeadline(deadline));
			if (headers != null) {
				for (String header : headers.keySet()) {
					httpRequest.addHeader(new HTTPHeader(header, headers.get(header)));
				}
			}
			httpRequest.setPayload(json.getBytes());
			HTTPResponse response = fetcher.fetch(httpRequest);
			return processResponse(response);
		} catch (Throwable e) {
			retry++;
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else if (retry < 3) {
				logger.warn("retrying after " + (System.currentTimeMillis() - start) + " and " + retry + " retries\n" + e.getClass() + e.getMessage());
			} else {
				logger.error(e.getClass() + "\n" + ToolString.stack2string(e));
			}
		}
	}
	return null;
}
 

public static String getResponseGET(String url, Map<String, String> params, Map<String, String> headers) {
	int retry = 0;
	while (retry < 3) {
		long start = System.currentTimeMillis();
		try {
			URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
			String urlStr = ToolString.toUrl(url.trim(), params);
			HTTPRequest httpRequest = new HTTPRequest(new URL(urlStr), HTTPMethod.GET, FetchOptions.Builder.withDeadline(deadline));
			if (headers != null) {
				for (String name : headers.keySet()) {
					httpRequest.addHeader(new HTTPHeader(name, headers.get(name)));
				}
			}
			return processResponse(fetcher.fetch(httpRequest));
		} catch (Throwable e) {
			retry++;
			if (e instanceof RuntimeException) {
				throw (RuntimeException) e;
			} else if (retry < 3) {
				logger.warn("retrying after " + (System.currentTimeMillis() - start) + " and " + retry + " retries\n" + e.getClass() + e.getMessage());
			} else {
				logger.error(e.getClass() + "\n" + ToolString.stack2string(e));
			}
		}
	}
	return null;
}
 

private FetchOptions getFetchOptions(HttpRequestOptions requestOptions) {
  return FetchOptions.Builder.disallowTruncate()
      .doNotFollowRedirects()
      .setDeadline(requestOptions.getConnTimeout() / 1000.0);
}
 

UrlFetchRequest(FetchOptions fetchOptions, HTTPMethod method, String url) throws IOException {
  request = new HTTPRequest(new URL(url), method, fetchOptions);
}
 

protected FetchOptions buildFetchOptions() {
    return FetchOptions.Builder.withDefaults();
}
 

@Test
public void testAllowTruncate() throws Exception {
    FetchOptions options = buildFetchOptions();
    options.allowTruncate();
    testOptions(options);
}
 

@Test
public void testDisallowTruncate() throws Exception {
    FetchOptions options = buildFetchOptions();
    options.disallowTruncate();
    testOptions(options);
}
 

@Test
public void testValidateCertificate() throws Exception {
    FetchOptions options = buildFetchOptions();
    options.validateCertificate();
    testOptions(options);
}
 

@Test
public void testDoNotValidateCertificate() throws Exception {
    FetchOptions options = buildFetchOptions();
    options.doNotValidateCertificate();
    testOptions(options);
}
 

@Test
public void testWithDeadline() throws Exception {
    FetchOptions options = buildFetchOptions();
    options.setDeadline(10 * 1000.0);
    testOptions(options);
}
 

protected void testOptions(FetchOptions options) throws Exception {
    testOptions(options, NOOP);
}
 

protected void testOptions(FetchOptions options, ResponseHandler handler) throws Exception {
    URL url = getFetchUrl();
    testOptions(url, options, handler);
}
 

protected void testOptions(URL url, FetchOptions options, ResponseHandler handler) throws Exception {
    testOptions(url, HTTPMethod.GET, options, handler);
}
 

protected void testOptions(URL url, HTTPMethod method, FetchOptions options, ResponseHandler handler) throws Exception {
    HTTPRequest request = new HTTPRequest(url, method, options);
    URLFetchService service = URLFetchServiceFactory.getURLFetchService();
    HTTPResponse response = service.fetch(request);
    handler.handle(response);
}
 

@Test
public void testWithDefaults() throws Exception {
    FetchOptions options = FetchOptions.Builder.withDefaults();
    testOptions(options);
}
 

@Test
public void testAllowTruncate() throws Exception {
    FetchOptions options = FetchOptions.Builder.allowTruncate();
    testOptions(options);
}
 

@Test
public void testDisallowTruncate() throws Exception {
    FetchOptions options = FetchOptions.Builder.disallowTruncate();
    testOptions(options);
}