Java Code Examples for org.apache.http.client.methods.HttpGet#setHeader()

public static String getUserInfo(ServerConfiguration serverConfiguration,
                                 AuthenticationToken token) throws IOException {

    HttpClient httpClient = new DefaultHttpClient();
    HttpGet get = new HttpGet(serverConfiguration.getUserInfoUri());

    get.setHeader("Authorization", String.format("Bearer %s", token.getAccessTokenValue()));
    HttpResponse response = httpClient.execute(get);

    BufferedReader bufferedReader = new BufferedReader(
            new InputStreamReader(response.getEntity().getContent()));

    String jsonString = "";
    String line;
    while ((line = bufferedReader.readLine()) != null) {
        jsonString = jsonString + line;
    }
    return jsonString;
}
 

@ParameterizedTest
@ArgumentsSource(BaseUriProvider.class)
void testGetWithoutPreCompression(String baseUri) throws Exception {
    try (CloseableHttpClient hc = HttpClients.createMinimal()) {
        final HttpGet request = new HttpGet(baseUri + "/compressed/foo_alone.txt");
        request.setHeader("Accept-Encoding", "gzip");
        try (CloseableHttpResponse res = hc.execute(request)) {
            assertThat(res.getFirstHeader("Content-Encoding")).isNull();
            assertThat(headerOrNull(res, "Content-Type")).isEqualTo(
                    "text/plain; charset=utf-8");
            final byte[] content = content(res);
            assertThat(new String(content, StandardCharsets.UTF_8)).isEqualTo("foo_alone");

            // Confirm path not cached when cache disabled.
            assertThat(PathAndQuery.cachedPaths())
                    .doesNotContain("/compressed/foo_alone.txt");
        }
    }
}
 

private HttpGet setupRequest() {
	HttpGet httpget = new HttpGet(this.endpointUrl);
	
	if (this.config != null) {
		httpget.setConfig(this.config);
	}

	// see also https://docs.cloudfoundry.org/concepts/http-routing.html
	httpget.setHeader(HTTP_HEADER_CF_APP_INSTANCE, this.instanceId);
	
	// provided for recursive scraping / loopback detection
	httpget.setHeader(EndpointConstants.HTTP_HEADER_PROMREGATOR_INSTANCE_IDENTIFIER, this.promregatorUUID.toString());
	
	if (this.ae != null) {
		this.ae.enrichWithAuthentication(httpget);
	}
	return httpget;
}
 

/**
 * Get 压缩发送请求，解压缩接收返回数据. If both gzip and deflate compression will be
 * accepted in the HTTP response. please choose the method
 * 
 * @param String address 请求地址
 * @param Map<String, Object> params 请求参数
 * @return String 响应内容
 * @throws ClientProtocolException
 * @throws IOException
 */
public String getWithCompression(String address, Map<String, Object> params) throws ClientProtocolException,
        IOException {

    String paramsStr = buildGetData(params);
    HttpGet httpGet = new HttpGet(address + paramsStr);
    HttpResponse httpResponse = null;
    try {
        httpGet.setHeader("User-Agent", agent);
        httpGet.setHeader("Accept-Encoding", "gzip,deflate");

        httpResponse = httpClient.execute(httpGet);
        return uncompress(httpResponse);
    }finally {
        if (httpResponse != null) {
            try {
                EntityUtils.consume(httpResponse.getEntity()); //会自动释放连接
            } catch (Exception e) {
            }
        }
    }
}
 

@Test
public void givenClientWithCustomUserAgentHeader_whenExecuteRequest_shouldReturn200() throws IOException {
    //given
    String userAgent = "BaeldungAgent/1.0";
    serviceMock.stubFor(get(urlEqualTo("/detail"))
      .withHeader("User-Agent", equalTo(userAgent))
      .willReturn(aResponse()
        .withStatus(200)));

    HttpClient httpClient = HttpClients.createDefault();
    HttpGet httpGet = new HttpGet("http://localhost:8089/detail");
    httpGet.setHeader(HttpHeaders.USER_AGENT, userAgent);

    //when
    HttpResponse response = httpClient.execute(httpGet);

    //then
    assertEquals(response.getStatusLine().getStatusCode(), 200);
}
 

public HttpServiceResponse executeGet(HttpClient httpClient, String requestUri, Map<String, String> headers) {
	HttpGet httpGet = new HttpGet(requestUri);
       
       if (headers != null) {
       	for (Entry<String, String> headerEntry : headers.entrySet()) {
       		httpGet.setHeader(headerEntry.getKey(), headerEntry.getValue());
       	}
       }

	try {
		HttpResponse httpResponse = httpClient.execute(httpGet);

		return new HttpServiceResponse(httpGet, httpResponse);
	} catch (IOException ex) {
    	log.error("Failed to execute get request", ex);
	}

	return null;
}
 

@Test
public void testPathBasedStaticInclude() throws IOException {
    TestHttpClient client = new TestHttpClient();
    try {
        HttpGet get = new HttpGet(DefaultServer.getDefaultServerURL() + "/servletContext/dispatch");
        get.setHeader("forward", "/snippet.html");
        HttpResponse result = client.execute(get);
        Assert.assertEquals(StatusCodes.OK, result.getStatusLine().getStatusCode());
        final String response = HttpClientUtils.readResponse(result);
        Assert.assertEquals("SnippetText", response);
    } finally {
        client.getConnectionManager().shutdown();
    }
}
 

@Test
public void testGetWithSpecifiedHttpHeader() throws Exception {
    URL url = buildURL(true, false, null);
    HttpGet httpget = new HttpGet(url.toURI());
    httpget.setHeader("org.wildfly.useStreamAsResponse", "0");
    readHttpResponse(getHttpClient(url).execute(httpget), 200);
}
 

private HttpGet createGetRequest(String accessToken) {
	HttpGet httpGet = new HttpGet(rule.getApplicationServerUri() + HelloJavaServlet.ENDPOINT);
	if(accessToken != null) {
		httpGet.setHeader(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken);
	}
	return httpGet;
}
 

@Test
public void testNameBasedInclude() throws IOException {
    TestHttpClient client = new TestHttpClient();
    try {
        HttpGet get = new HttpGet(DefaultServer.getDefaultServerURL() + "/servletContext/dispatch");
        get.setHeader("include", "include");
        get.setHeader("name", "true");
        HttpResponse result = client.execute(get);
        Assert.assertEquals(StatusCodes.OK, result.getStatusLine().getStatusCode());
        final String response = HttpClientUtils.readResponse(result);
        Assert.assertEquals(IncludeServlet.MESSAGE + "Name!included", response);
    } finally {
        client.getConnectionManager().shutdown();
    }
}
 

@Test
public final void givenConfigOnRequest_whenRequestHasCustomUserAgent_thenCorrect() throws ClientProtocolException, IOException {
    client = HttpClients.custom().build();
    final HttpGet request = new HttpGet(SAMPLE_URL);
    request.setHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 Firefox/26.0");
    response = client.execute(request);
}
 

@Test
public final void whenSettingCookiesOnARequest_thenCorrect() throws IOException {
    instance = HttpClientBuilder.create().build();
    final HttpGet request = new HttpGet("http://www.github.com");
    request.setHeader("Cookie", "JSESSIONID=1234");

    response = instance.execute(request);

    assertThat(response.getStatusLine().getStatusCode(), equalTo(200));
}
 

@Override
public void executeQuery(String query, String queryID) {
	Instant start = Instant.now();

	try {
		String qEncoded = URLEncoder.encode(query);
		String addChar = "?";
		if (service.contains("?")) {
			addChar = "&";
		}
		String url = service + addChar + "query=" + qEncoded;
		HttpGet request = new HttpGet(url);
		RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(timeOut.intValue())
				.setConnectTimeout(timeOut.intValue()).build();

		if(this.responseType != null)
			request.setHeader(HttpHeaders.ACCEPT, this.responseType);

		request.setConfig(requestConfig);
		CloseableHttpClient client = HttpClients.createDefault();
		CloseableHttpResponse response = client.execute(request);

		// method to process the result in background
		super.processHttpResponse(queryID, start, client, response);

	} catch (Exception e) {
		LOGGER.warn("Worker[{{}} : {{}}]: Could not execute the following query\n{{}}\n due to", this.workerType,
				this.workerID, query, e);
		super.addResults(new QueryExecutionStats(queryID, COMMON.QUERY_UNKNOWN_EXCEPTION, durationInMilliseconds(start, Instant.now())));
	}

}
 

@Test
public void testGetWithUnmatchedContentType() throws Exception {
    URL url = buildURL(true, true, null);
    HttpGet httpget = new HttpGet(url.toURI());
    httpget.setHeader("Accept", "text/html");
    HttpResponse response = getHttpClient(url).execute(httpget);
    readHttpResponse(response, 200);

    String contentType = response.getEntity().getContentType().getValue();
    Assert.assertTrue(contentType, contentType.contains("application/octet-stream"));
}
 

@Test
public final void givenUsingNewApi_whenRequestHasCustomContentType_thenCorrect() throws ClientProtocolException, IOException {
    client = HttpClients.custom().build();
    final HttpGet request = new HttpGet(SAMPLE_URL);
    request.setHeader(HttpHeaders.CONTENT_TYPE, "application/json");
    response = client.execute(request);
}
 

public static void profile() throws Exception {
    String accountUrl = keycloak.getDeployment().getAccountUrl();
    HttpGet get = new HttpGet(accountUrl);
    get.setHeader("Accept", "application/json");
    get.setHeader("Authorization", "Bearer " + keycloak.getTokenString(10, TimeUnit.SECONDS));

    HttpResponse response = keycloak.getDeployment().getClient().execute(get);
    if (response.getStatusLine().getStatusCode() == 200) {
        print(response.getEntity().getContent());
    } else {
        System.out.println(response.getStatusLine().toString());
    }
}
 

@GetMapping(path = "/apidocs")
public String greeting() {
    HttpClient client = getHttpClient();

    String apidocs = "http://localhost:"+serverPort+serverPath+"/STU3/metadata";

    HttpGet request = new HttpGet(apidocs);
    request.setHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE);
    request.setHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE);

    try {

        HttpResponse response = client.execute(request);
        log.trace("Response {}", response.getStatusLine());
        if (response.getStatusLine().toString().contains("200")) {

            String encoding = "UTF-8";
            String body = IOUtils.toString(response.getEntity().getContent(), encoding);
            log.trace("{}",body);

            CapabilityStatement capabilityStatement = (CapabilityStatement) ctx.newJsonParser().parseResource(body);

            return parseConformanceStatement(capabilityStatement);

        }

    } catch (UnknownHostException e) {
        log.error("Host not known");
    } catch (IOException ex) {
        log.error("IO Exception {}", ex.getMessage());
    }

    return "Unable to resolve swagger/openapi documentation from " + apidocs;
}
 

@Test
public void testGetWithMatchedContentType() throws Exception {
    URL url = buildURL(true, true, null);
    HttpGet httpget = new HttpGet(url.toURI());
    httpget.setHeader("Accept", "text/plain");
    HttpResponse response = getHttpClient(url).execute(httpget);
    readHttpResponse(response, 200);

    String contentType = response.getEntity().getContentType().getValue();
    Assert.assertTrue(contentType, contentType.contains("text/plain"));
}
 

/**
 * 根据url下载网页内容
 *
 * @param url
 * @return
 */
public static String getHttpContent(String url) {

    CloseableHttpClient httpClient = null;
    HttpHost proxy = null;
    if (IPProxyRepository.size() > 0) {  // 如果ip代理地址库不为空，则设置代理
        proxy = getRandomProxy();
        httpClient = HttpClients.custom().setProxy(proxy).build();  // 创建httpclient对象
    } else {
        httpClient = HttpClients.custom().build();  // 创建httpclient对象
    }
    HttpGet request = new HttpGet(url); // 构建htttp get请求
    request.setHeader("user-agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0");
    /*
    HttpHost proxy = null;
    CloseableHttpClient httpClient = HttpClients.custom().build();
    HttpGet request = new HttpGet(url); // 构建htttp get请求
    */
    /**
     * 设置超时时间
     * setConnectTimeout：设置连接超时时间，单位毫秒。
     * setConnectionRequestTimeout：设置从connect Manager获取Connection 超时时间，单位毫秒。这个属性是新加的属性，因为目前版本是可以共享连接池的。
     * setSocketTimeout：请求获取数据的超时时间，单位毫秒。 如果访问一个接口，多少时间内无法返回数据，就直接放弃此次调用。
     */
    RequestConfig requestConfig = RequestConfig.custom()
            .setConnectTimeout(5000).setConnectionRequestTimeout(1000)
            .setSocketTimeout(5000).build();
    request.setConfig(requestConfig);
    String host = null;
    Integer port = null;
    if(proxy != null) {
        host = proxy.getHostName();
        port = proxy.getPort();
    }
    try {
        long start = System.currentTimeMillis();    // 开始时间
        CloseableHttpResponse response = httpClient.execute(request);
        long end = System.currentTimeMillis();      // 结束时间
        logger.info("下载网页：{}，消耗时长：{} ms，代理信息：{}", url, end - start, host + ":" + port);
        return EntityUtils.toString(response.getEntity());
    } catch (Exception e) {
        logger.error("下载网页：{}出错，代理信息：{}，", url, host + ":" + port);
        // 如果该url为列表url，则将其添加到高优先级队列中
        if (url.contains("list.jd.com") || url.contains("list.suning.com")) {   // 这里为硬编码
            String domain = SpiderUtil.getTopDomain(url);   // jd.com
            retryUrl(url, domain + SpiderConstants.SPIDER_DOMAIN_HIGHER_SUFFIX);    // 添加url到jd.com.higher中
        }
        /**
         * 为什么要加入到高优先级队列中？
         * 如果该url为第一个种子url，但是解析却失败了，那么url仓库中会一直没有url，虽然爬虫程序还在执行，
         * 但是会一直提示没有url，这时就没有意义了，还需要尝试的另外一个原因是，下载网页失败很大可能是
         * 因为：
         *      1.此刻网络突然阻塞
         *      2.代理地址被限制，也就是被封了
         * 所以将其重新添加到高优先级队列中，再进行解析目前来说是比较不错的解决方案
         */
        e.printStackTrace();
    }

    return null;
}
 

public HttpResponse conditionalGet(final URI uri, final String modified) throws IOException {
  HttpGet get = new HttpGet(uri);
  get.setHeader(IF_MODIFIED_SINCE, modified);
  return consume(execute(get));
}