org.apache.http.conn.routing.HttpRoutePlanner Java Examples

private void processOnRetry(RetryEvent event, Duration waitInterval,
                            HttpRoutePlanner planer, OnRetryConsumer onRetryConsumer) {

    InetAddress targetAddress = null;
    if (planer instanceof RoundRobinRouterPlaner) {
        // inform planer about error on last roue
        HttpRoute httpRoute = ((RoundRobinRouterPlaner)planer).lastRouteCauseError();
        targetAddress = Try.of(() -> httpRoute.getTargetHost().getAddress()).getOrElse((InetAddress) null);
    } else if (proxy != null) {
        targetAddress = Try.of(() -> InetAddress.getByName(proxy.getHost())).getOrElse((InetAddress) null);
    }

    // inform caller about retry
    if (onRetryConsumer != null) {
        onRetryConsumer.onRetry(targetAddress, event.getNumberOfRetryAttempts(),
                waitInterval, event.getLastThrowable());
    }
}
 

@Override
protected RequestDirector createClientRequestDirector(
    HttpRequestExecutor requestExec,
    ClientConnectionManager conman,
    ConnectionReuseStrategy reustrat,
    ConnectionKeepAliveStrategy kastrat,
    HttpRoutePlanner rouplan,
    HttpProcessor httpProcessor,
    HttpRequestRetryHandler retryHandler,
    RedirectHandler redirectHandler,
    AuthenticationHandler targetAuthHandler,
    AuthenticationHandler proxyAuthHandler,
    UserTokenHandler stateHandler,
    HttpParams params) {
  return new RequestDirector() {
    @Beta
    public HttpResponse execute(HttpHost target, HttpRequest request, HttpContext context)
        throws HttpException, IOException {
      return new BasicHttpResponse(HttpVersion.HTTP_1_1, responseCode, null);
    }
  };
}
 

private HttpRoutePlanner createRoutePlanner() {
    // If http.nonProxyHosts is not set, then "localhost" is used as a default value, which prevents setting a proxy on localhost.
    if (System.getProperty("http.nonProxyHosts") == null) {
        System.setProperty("http.nonProxyHosts", "");
    }
    return new SystemDefaultRoutePlanner(ProxySelector.getDefault());
}
 

private HttpRoutePlanner createHttpRoutePlanner(final Http4FileSystemConfigBuilder builder,
        final FileSystemOptions fileSystemOptions) {
    final HttpHost proxyHost = getProxyHttpHost(builder, fileSystemOptions);

    if (proxyHost != null) {
        return new DefaultProxyRoutePlanner(proxyHost);
    }

    return new SystemDefaultRoutePlanner(ProxySelector.getDefault());
}
 

@Override
protected RequestDirector createClientRequestDirector(
        final HttpRequestExecutor requestExec,
        final ClientConnectionManager conman,
        final ConnectionReuseStrategy reustrat,
        final ConnectionKeepAliveStrategy kastrat,
        final HttpRoutePlanner rouplan,
        final HttpProcessor httpProcessor,
        final HttpRequestRetryHandler retryHandler,
        final RedirectHandler redirectHandler,
        final AuthenticationHandler targetAuthHandler,
        final AuthenticationHandler proxyAuthHandler,
        final UserTokenHandler stateHandler,
        final HttpParams params) {
    return new LibRequestDirector(
            requestExec,
            conman,
            reustrat,
            kastrat,
            rouplan,
            httpProcessor,
            retryHandler,
            redirectHandler,
            targetAuthHandler,
            proxyAuthHandler,
            stateHandler,
            params);
}
 

private HttpRoutePlanner getRoutePlanner() {
    if (isUsingProxy()) {
        return new DefaultProxyRoutePlanner(new HttpHost(getProxyHost(), getProxyPort()));
    } else {
        return new DefaultRoutePlanner(null);
    }
}
 

private void decorateRoutePlanner(CrawlerHttpClient crawlerHttpClient) {
    HttpRoutePlanner routePlanner = crawlerHttpClient.getRoutePlanner();
    if (!(routePlanner instanceof ProxyBindRoutPlanner)) {
        log.warn("自定义了代理发生器,vscrawler的代理功能将不会生效");
        return;
    }

    VSCrawlerRoutePlanner vsCrawlerRoutePlanner = new VSCrawlerRoutePlanner((ProxyBindRoutPlanner) routePlanner,
            ipPool, proxyPlanner, this);
    crawlerHttpClient.setRoutePlanner(vsCrawlerRoutePlanner);

}
 

/**
 * Ensure proxy settings are NOT applied to http client
 * @throws SecurityException 
 * @throws NoSuchFieldException 
 * @throws IllegalAccessException 
 * @throws IllegalArgumentException 
 */
@Test
public void shouldNotAddProxyToClient() throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException{
	
	HttpClientBuilder builder = HttpClientBuilder.create();
	
	ClientConfiguration config = new ClientConfiguration();

	HttpClientSettings settings = HttpClientSettings.adapt(config);
	
	DefaultTokenManager.addProxyConfig(builder, settings);
	builder.build();
	Field field = builder.getClass().getDeclaredField("routePlanner");    
	field.setAccessible(true);
	HttpRoutePlanner httpRoutePlanner = (HttpRoutePlanner) field.get(builder);
	
	assertNull(httpRoutePlanner);
}
 

/**
 * Ensure proxy settings are applied to http client
 * @throws SecurityException 
 * @throws NoSuchFieldException 
 * @throws IllegalAccessException 
 * @throws IllegalArgumentException 
 */
@Test
public void shouldAddProxyToClient() throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException{
	
	HttpClientBuilder builder = HttpClientBuilder.create();
	
	ClientConfiguration config = new ClientConfiguration().withProxyHost("127.0.0.1").withProxyPort(8080);

	HttpClientSettings settings = HttpClientSettings.adapt(config);
	
	DefaultTokenManager.addProxyConfig(builder, settings);
	builder.build();
	Field field = builder.getClass().getDeclaredField("routePlanner");    
	field.setAccessible(true);
	HttpRoutePlanner httpRoutePlanner = (HttpRoutePlanner) field.get(builder);
	
	assertNotNull(httpRoutePlanner);
}
 

@Test
public void httpRoutePlannerCantBeUsedWithProxy_SystemPropertiesDisabled() {
    System.setProperty("http.proxyHost", "localhost");
    System.setProperty("http.proxyPort", "1234");

    ProxyConfiguration proxyConfig = ProxyConfiguration.builder()
                                                       .useSystemPropertyValues(Boolean.FALSE)
                                                       .build();

    ApacheHttpClient.builder()
                    .proxyConfiguration(proxyConfig)
                    .httpRoutePlanner(Mockito.mock(HttpRoutePlanner.class))
                    .build();
}
 

@Test
public void httpRoutePlannerCantBeUsedWithProxy_SystemPropertiesEnabled() {
    System.setProperty("http.proxyHost", "localhost");
    System.setProperty("http.proxyPort", "1234");

    assertThatThrownBy(() -> {
        ApacheHttpClient.builder()
                        .httpRoutePlanner(Mockito.mock(HttpRoutePlanner.class))
                        .build();
    }).isInstanceOf(IllegalArgumentException.class);
}
 

@Test
public void httpRoutePlannerCantBeUsedWithProxy() {
    ProxyConfiguration proxyConfig = ProxyConfiguration.builder()
                                                  .endpoint(URI.create("http://localhost:1234"))
                                                  .useSystemPropertyValues(Boolean.FALSE)
                                                  .build();
    assertThatThrownBy(() -> {
        ApacheHttpClient.builder()
                        .proxyConfiguration(proxyConfig)
                        .httpRoutePlanner(Mockito.mock(HttpRoutePlanner.class))
                        .build();
    }).isInstanceOf(IllegalArgumentException.class);
}
 

public ClientHttpRequestFactory createRequestFactory(HttpProxyConfiguration httpProxyConfiguration, boolean trustSelfSignedCerts) {
    HttpClientBuilder httpClientBuilder = HttpClients.custom()
                                                     .useSystemProperties();

    if (trustSelfSignedCerts) {
        httpClientBuilder.setSslcontext(buildSslContext());
        httpClientBuilder.setHostnameVerifier(BROWSER_COMPATIBLE_HOSTNAME_VERIFIER);
    }

    if (httpProxyConfiguration != null) {
        HttpHost proxy = new HttpHost(httpProxyConfiguration.getProxyHost(), httpProxyConfiguration.getProxyPort());
        httpClientBuilder.setProxy(proxy);

        if (httpProxyConfiguration.isAuthRequired()) {
            BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(new AuthScope(httpProxyConfiguration.getProxyHost(),
                                                             httpProxyConfiguration.getProxyPort()),
                                               new UsernamePasswordCredentials(httpProxyConfiguration.getUsername(),
                                                                               httpProxyConfiguration.getPassword()));
            httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }

        HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy);
        httpClientBuilder.setRoutePlanner(routePlanner);
    }

    HttpClient httpClient = httpClientBuilder.build();

    return new HttpComponentsClientHttpRequestFactory(httpClient);
}
 

private static void verifyProcessedUriMatchesExpectedOutput(String inputHostString, String expectedHostString) throws HttpException {
    HttpRoutePlanner routePlanner = new VespaHttpClientBuilder.HttpToHttpsRoutePlanner();
    HttpRoute newRoute = routePlanner.determineRoute(HttpHost.create(inputHostString), mock(HttpRequest.class), new HttpClientContext());
    HttpHost target = newRoute.getTargetHost();
    assertEquals(expectedHostString, target.toURI());
}
 

/**
 * Configure the proxy with the required credential if needed
 *
 * @param httpClientBuilder
 * @param credentialsProvider
 * @param url
 * @return {@link HttpClientBuilder}
 */
private HttpClientBuilder configureProxy(HttpClientBuilder httpClientBuilder, CredentialsProvider credentialsProvider, String url) {
	if (proxyConfig == null) {
		return httpClientBuilder;
	}

	final String protocol = getURL(url).getProtocol();
	final boolean proxyHTTPS = Protocol.isHttps(protocol) && (proxyConfig.getHttpsProperties() != null);
	final boolean proxyHTTP = Protocol.isHttp(protocol) && (proxyConfig.getHttpProperties() != null);

	ProxyProperties proxyProps = null;
	if (proxyHTTPS) {
		LOG.debug("Use proxy https parameters");
		proxyProps = proxyConfig.getHttpsProperties();
	} else if (proxyHTTP) {
		LOG.debug("Use proxy http parameters");
		proxyProps = proxyConfig.getHttpProperties();
	} else {
		return httpClientBuilder;
	}

	String proxyHost = proxyProps.getHost();
	int proxyPort = proxyProps.getPort();
	String proxyUser = proxyProps.getUser();
	String proxyPassword = proxyProps.getPassword();
	String proxyExcludedHosts = proxyProps.getExcludedHosts();

	if (Utils.isStringNotEmpty(proxyUser) && Utils.isStringNotEmpty(proxyPassword)) {
		AuthScope proxyAuth = new AuthScope(proxyHost, proxyPort);
		UsernamePasswordCredentials proxyCredentials = new UsernamePasswordCredentials(proxyUser, proxyPassword);
		credentialsProvider.setCredentials(proxyAuth, proxyCredentials);
	}

	LOG.debug("proxy host/port: {}:{}", proxyHost, proxyPort);
	// TODO SSL peer shut down incorrectly when protocol is https
	final HttpHost proxy = new HttpHost(proxyHost, proxyPort, Protocol.HTTP.getName());

	if (Utils.isStringNotEmpty(proxyExcludedHosts)) {
		final String[] hosts = proxyExcludedHosts.split("[,; ]");

		HttpRoutePlanner routePlanner = new DefaultProxyRoutePlanner(proxy) {
			@Override
			public HttpRoute determineRoute(final HttpHost host, final HttpRequest request, final HttpContext context) throws HttpException {

				String hostname = (host != null ? host.getHostName() : null);

				if ((hosts != null) && (hostname != null)) {
					for (String h : hosts) {
						if (hostname.equalsIgnoreCase(h)) {
							// bypass proxy for that hostname
							return new HttpRoute(host);
						}
					}
				}
				return super.determineRoute(host, request, context);
			}
		};

		httpClientBuilder.setRoutePlanner(routePlanner);
	}

	return httpClientBuilder.setProxy(proxy);
}
 

private static HttpRoutePlanner createProxyRoutePlanner() {
    // use the standard JRE ProxySelector to get proxy information
    Message.verbose("Using JRE standard ProxySelector for configuring HTTP proxy");
    return new SystemDefaultRoutePlanner(ProxySelector.getDefault());
}
 

private HttpRoutePlanner getNewProxyRoutePlanner() {
    HttpHost httpHost = new HttpHost(proxy.getHost(), proxy.getPort());
    return new DefaultProxyRoutePlanner(httpHost);
}
 

public void setHttpRoutePlanner(HttpRoutePlanner httpRoutePlanner) {
    httpRoutePlanner(httpRoutePlanner);
}
 

public LibRequestDirector(
        final HttpRequestExecutor requestExec,
        final ClientConnectionManager conman,
        final ConnectionReuseStrategy reustrat,
        final ConnectionKeepAliveStrategy kastrat,
        final HttpRoutePlanner rouplan,
        final HttpProcessor httpProcessor,
        final HttpRequestRetryHandler retryHandler,
        final RedirectHandler redirectHandler,
        final AuthenticationHandler targetAuthHandler,
        final AuthenticationHandler proxyAuthHandler,
        final UserTokenHandler userTokenHandler,
        final HttpParams params) {

    if (requestExec == null) {
        throw new IllegalArgumentException("Request executor may not be null.");
    }
    if (conman == null) {
        throw new IllegalArgumentException("Client connection manager may not be null.");
    }
    if (reustrat == null) {
        throw new IllegalArgumentException("Connection reuse strategy may not be null.");
    }
    if (kastrat == null) {
        throw new IllegalArgumentException("Connection keep alive strategy may not be null.");
    }
    if (rouplan == null) {
        throw new IllegalArgumentException("Route planner may not be null.");
    }
    if (httpProcessor == null) {
        throw new IllegalArgumentException("HTTP protocol processor may not be null.");
    }
    if (retryHandler == null) {
        throw new IllegalArgumentException("HTTP request retry handler may not be null.");
    }
    if (redirectHandler == null) {
        throw new IllegalArgumentException("Redirect handler may not be null.");
    }
    if (targetAuthHandler == null) {
        throw new IllegalArgumentException("Target authentication handler may not be null.");
    }
    if (proxyAuthHandler == null) {
        throw new IllegalArgumentException("Proxy authentication handler may not be null.");
    }
    if (userTokenHandler == null) {
        throw new IllegalArgumentException("User token handler may not be null.");
    }
    if (params == null) {
        throw new IllegalArgumentException("HTTP parameters may not be null");
    }
    this.requestExec       = requestExec;
    this.connManager       = conman;
    this.reuseStrategy     = reustrat;
    this.keepAliveStrategy = kastrat;
    this.routePlanner      = rouplan;
    this.httpProcessor     = httpProcessor;
    this.retryHandler      = retryHandler;
    this.redirectHandler   = redirectHandler;
    this.targetAuthHandler = targetAuthHandler;
    this.proxyAuthHandler  = proxyAuthHandler;
    this.userTokenHandler  = userTokenHandler;
    this.params            = params;

    this.managedConn       = null;

    this.execCount = 0;
    this.redirectCount = 0;
    this.maxRedirects = this.params.getIntParameter(ClientPNames.MAX_REDIRECTS, 100);
    this.targetAuthState = new AuthState();
    this.proxyAuthState = new AuthState();
}
 

/**
 * Build an HTTP client with the given router planer.
 *
 * @param planer
 *         The router planer for http client, used for load balancing
 *
 * @return The new HTTP client instance.
 */
private CloseableHttpClient buildClient(HttpRoutePlanner planer) {


    final HttpClientBuilder clientBuilder = this.createClientBuilder();

    this.applyTimeouts(clientBuilder);
    clientBuilder.setRoutePlanner(planer);

    return clientBuilder.build();
}
 

/**
 * Configuration that defines an HTTP route planner that computes the route an HTTP request should take.
 * May not be used in conjunction with {@link #proxyConfiguration(ProxyConfiguration)}.
 */
Builder httpRoutePlanner(HttpRoutePlanner proxyConfiguration);