Java Code Examples for org.apache.http.impl.client.HttpClientBuilder#setMaxConnPerRoute()

public HttpHelper(){
    HttpClientBuilder builder = HttpClientBuilder.create();
    builder.setMaxConnPerRoute(50);
    builder.setMaxConnTotal(100);

    // 创建支持忽略证书的https
    try {
        SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {

            @Override
            public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
                return true;
            }
        }).build();

        httpclient = HttpClientBuilder.create()
            .setSSLContext(sslContext)
            .setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create()
                .register("http", PlainConnectionSocketFactory.INSTANCE)
                .register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
                .build()))
            .build();
    } catch (Throwable e) {
        // ignore
    }
}
 

/**
 * Configures the http client.
 *
 * @param builder client builder to configure
 */
protected void configureClient(HttpClientBuilder builder)
{
    if ( getRemoteHttpCacheAttributes().getMaxConnectionsPerHost() > 0 )
    {
        builder.setMaxConnTotal(getRemoteHttpCacheAttributes().getMaxConnectionsPerHost());
        builder.setMaxConnPerRoute(getRemoteHttpCacheAttributes().getMaxConnectionsPerHost());
    }

    builder.setDefaultRequestConfig(RequestConfig.custom()
            .setConnectTimeout(getRemoteHttpCacheAttributes().getConnectionTimeoutMillis())
            .setSocketTimeout(getRemoteHttpCacheAttributes().getSocketTimeoutMillis())
            // By default we instruct HttpClient to ignore cookies.
            .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
            .build());
}
 

public HttpClientBuilder clientBuilder() {
    HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
    RequestConfig.Builder builder = RequestConfig.custom();
    builder.setConnectionRequestTimeout(requestConnTimeout);
    builder.setSocketTimeout(socketTimeout);
    builder.setConnectTimeout(connTimeout);
    httpClientBuilder.setDefaultRequestConfig(builder.build());
    httpClientBuilder.setMaxConnPerRoute(maxConnPerRoute);
    httpClientBuilder.setMaxConnTotal(maxConnTotal);
    return httpClientBuilder;
}
 

public static byte[] getBytes(String url, int timeout) throws Exception {
    long start = System.currentTimeMillis();
    HttpClientBuilder builder = HttpClientBuilder.create();
    builder.setMaxConnPerRoute(50);
    builder.setMaxConnTotal(100);
    CloseableHttpClient httpclient = builder.build();
    URI uri = new URIBuilder(url).build();
    RequestConfig config = custom().setConnectTimeout(timeout)
        .setConnectionRequestTimeout(timeout)
        .setSocketTimeout(timeout)
        .build();
    HttpGet httpGet = new HttpGet(uri);
    HttpClientContext context = HttpClientContext.create();
    context.setRequestConfig(config);
    CloseableHttpResponse response = httpclient.execute(httpGet, context);
    try {
        int statusCode = response.getStatusLine().getStatusCode();
        long end = System.currentTimeMillis();
        long cost = end - start;
        if (logger.isWarnEnabled()) {
            logger.warn("post " + url + ", cost : " + cost);
        }
        if (statusCode == HttpStatus.SC_OK) {
            return EntityUtils.toByteArray(response.getEntity());
        } else {
            String errorMsg = EntityUtils.toString(response.getEntity());
            throw new RuntimeException("requestGet remote error, url=" + uri.toString() + ", code=" + statusCode
                                       + ", error msg=" + errorMsg);
        }
    } finally {
        response.close();
        httpGet.releaseConnection();
    }
}
 

/**
 * Creates the <tt>HttpClientBuilder</tt> that will be used by this WebClient.
 * Extensions may override this method in order to create a customized
 * <tt>HttpClientBuilder</tt> instance (e.g. with a custom
 * {@link org.apache.http.conn.ClientConnectionManager} to perform
 * some tracking; see feature request 1438216).
 * @return the <tt>HttpClientBuilder</tt> that will be used by this WebConnection
 */
protected HttpClientBuilder createHttpClientBuilder() {
    final HttpClientBuilder builder = HttpClientBuilder.create();
    builder.setRedirectStrategy(new HtmlUnitRedirectStrategie());
    configureTimeout(builder, getTimeout());
    configureHttpsScheme(builder);
    builder.setMaxConnPerRoute(6);

    builder.setConnectionManagerShared(true);
    return builder;
}
 

public static byte[] getBytes(String url, int timeout) throws Exception {
    long start = System.currentTimeMillis();
    HttpClientBuilder builder = HttpClientBuilder.create();
    builder.setMaxConnPerRoute(50);
    builder.setMaxConnTotal(100);
    CloseableHttpClient httpclient = builder.build();
    URI uri = new URIBuilder(url).build();
    RequestConfig config = custom().setConnectTimeout(timeout)
        .setConnectionRequestTimeout(timeout)
        .setSocketTimeout(timeout)
        .build();
    HttpGet httpGet = new HttpGet(uri);
    HttpClientContext context = HttpClientContext.create();
    context.setRequestConfig(config);
    CloseableHttpResponse response = httpclient.execute(httpGet, context);
    try {
        int statusCode = response.getStatusLine().getStatusCode();
        long end = System.currentTimeMillis();
        long cost = end - start;
        if (logger.isWarnEnabled()) {
            logger.warn("post " + url + ", cost : " + cost);
        }
        if (statusCode == HttpStatus.SC_OK) {
            return EntityUtils.toByteArray(response.getEntity());
        } else {
            String errorMsg = EntityUtils.toString(response.getEntity());
            throw new RuntimeException("requestGet remote error, url=" + uri.toString() + ", code=" + statusCode
                                       + ", error msg=" + errorMsg);
        }
    } finally {
        response.close();
        httpGet.releaseConnection();
    }
}
 

@Override
public IHttpClient buildHttpClient(HttpClientConfig config)
{
    HttpClientBuilder builder = HttpClientBuilder.create();
    builder.setDefaultHeaders(config.createHeaders());
    if (config.hasCookieStore())
    {
        builder.setDefaultCookieStore(config.getCookieStore());
    }
    if (config.hasCredentials())
    {
        AuthScope authScope = config.hasAuthScope() ? config.getAuthScope()
                : ClientBuilderUtils.DEFAULT_AUTH_SCOPE;
        CredentialsProvider credProvider = ClientBuilderUtils.createCredentialsProvider(authScope,
                config.getCredentials());
        builder.setDefaultCredentialsProvider(credProvider);
    }

    sslContextFactory.getSslContext(SSLConnectionSocketFactory.SSL, !config.isSslCertificateCheckEnabled())
            .ifPresent(builder::setSSLContext);

    if (!config.isSslHostnameVerificationEnabled())
    {
        builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
    }
    builder.setConnectionManager(config.getConnectionManager());
    builder.setMaxConnTotal(config.getMaxTotalConnections());
    builder.setMaxConnPerRoute(config.getMaxConnectionsPerRoute());
    builder.addInterceptorLast(config.getLastRequestInterceptor());
    builder.addInterceptorLast(config.getLastResponseInterceptor());
    builder.setRedirectStrategy(config.getRedirectStrategy());
    builder.setRetryHandler(config.getHttpRequestRetryHandler());
    Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setConnectionRequestTimeout(config.getConnectionRequestTimeout());
    requestConfigBuilder.setConnectTimeout(config.getConnectTimeout());
    requestConfigBuilder.setCircularRedirectsAllowed(config.isCircularRedirectsAllowed());
    requestConfigBuilder.setSocketTimeout(config.getSocketTimeout());
    Optional.ofNullable(config.getCookieSpec()).ifPresent(requestConfigBuilder::setCookieSpec);
    builder.setDefaultRequestConfig(requestConfigBuilder.build());
    builder.setDefaultSocketConfig(SocketConfig.copy(SocketConfig.DEFAULT)
            .setSoTimeout(config.getSocketTimeout())
            .build());
    builder.setDnsResolver(config.getDnsResolver());

    HttpClient httpClient = new HttpClient();
    httpClient.setCloseableHttpClient(builder.build());
    if (config.hasBaseUrl())
    {
        httpClient.setHttpHost(HttpHost.create(config.getBaseUrl()));
    }
    httpClient.setSkipResponseEntity(config.isSkipResponseEntity());
    return httpClient;
}
 

private void download(BinlogFile binlogFile) throws Throwable {
    String downloadLink = binlogFile.getDownloadLink();
    String fileName = binlogFile.getFileName();

    downloadLink = downloadLink.trim();
    CloseableHttpClient httpClient = null;
    if (downloadLink.startsWith("https")) {
        HttpClientBuilder builder = HttpClientBuilder.create();
        builder.setMaxConnPerRoute(50);
        builder.setMaxConnTotal(100);
        // 创建支持忽略证书的https
        final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {

            @Override
            public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
                return true;
            }
        }).build();

        httpClient = HttpClientBuilder.create()
            .setSSLContext(sslContext)
            .setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create()
                .register("http", PlainConnectionSocketFactory.INSTANCE)
                .register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
                .build()))
            .build();
    } else {
        httpClient = HttpClientBuilder.create().setMaxConnPerRoute(50).setMaxConnTotal(100).build();
    }

    HttpGet httpGet = new HttpGet(downloadLink);
    RequestConfig requestConfig = RequestConfig.custom()
        .setConnectTimeout(TIMEOUT)
        .setConnectionRequestTimeout(TIMEOUT)
        .setSocketTimeout(TIMEOUT)
        .build();
    httpGet.setConfig(requestConfig);
    HttpResponse response = httpClient.execute(httpGet);
    int statusCode = response.getStatusLine().getStatusCode();
    if (statusCode != HttpResponseStatus.OK.code()) {
        throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:" + statusCode);
    }
    saveFile(new File(destDir), "mysql-bin." + fileName, response);
}
 

/**
 * Initialize an {@link HttpClient} for performing requests. Proxy settings will
 * be read from the configuration and applied transparently.
 */
public HttpClientAdapter() {
    Optional<SSLConnectionSocketFactory> sslSocketFactory = Optional.empty();
    try {
        SSLContext sslContext = SSLContext.getDefault();
        sslSocketFactory = Optional.of(new SSLConnectionSocketFactory(
                sslContext,
                new String[]{"TLSv1", "TLSv1.1", "TLSv1.2"},
                null,
                SSLConnectionSocketFactory.getDefaultHostnameVerifier()));
    }
    catch (NoSuchAlgorithmException e) {
        log.warn("No such algorithm. Using default context", e);
    }

    HttpClientBuilder clientBuilder = HttpClientBuilder.create();
    sslSocketFactory.ifPresent(sf -> clientBuilder.setSSLSocketFactory(sf));

    clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
    Builder requestConfigBuilder = RequestConfig.custom()
            .setConnectTimeout(Config.get().getInt(HTTP_CONNECTION_TIMEOUT, 5) * TO_MILLISECONDS)
            .setSocketTimeout(Config.get().getInt(HTTP_SOCKET_TIMEOUT, 5 * 60) * TO_MILLISECONDS)
            .setCookieSpec(CookieSpecs.IGNORE_COOKIES);

    // Store the proxy settings
    String proxyHostname = ConfigDefaults.get().getProxyHost();
    if (!StringUtils.isBlank(proxyHostname)) {
        int proxyPort = ConfigDefaults.get().getProxyPort();

        proxyHost = new HttpHost(proxyHostname, proxyPort);
        clientBuilder.setProxy(proxyHost);

        String proxyUsername = ConfigDefaults.get().getProxyUsername();
        String proxyPassword = ConfigDefaults.get().getProxyPassword();
        if (!StringUtils.isBlank(proxyUsername) &&
                !StringUtils.isBlank(proxyPassword)) {
            Credentials proxyCredentials = new UsernamePasswordCredentials(
                    proxyUsername, proxyPassword);

            credentialsProvider.setCredentials(new AuthScope(proxyHostname, proxyPort),
                    proxyCredentials);
        }

        // Explicitly exclude the NTLM authentication scheme
        requestConfigBuilder =  requestConfigBuilder.setProxyPreferredAuthSchemes(
                                Arrays.asList(AuthSchemes.DIGEST, AuthSchemes.BASIC));

        clientBuilder.setProxyAuthenticationStrategy(new ProxyAuthenticationStrategy());

        clientBuilder.setRoutePlanner(new CustomProxyRoutePlanner(proxyHost));
    }

    // Read proxy exceptions from the "no_proxy" config option
    String noProxy = Config.get().getString(NO_PROXY);
    if (!StringUtils.isBlank(noProxy)) {
        for (String domain : Arrays.asList(noProxy.split(","))) {
            noProxyDomains.add(domain.toLowerCase().trim());
        }
    }

    requestConfig = requestConfigBuilder.build();
    clientBuilder.setMaxConnPerRoute(Config.get().getInt(MAX_CONNCECTIONS, 1));
    clientBuilder.setMaxConnTotal(Config.get().getInt(MAX_CONNCECTIONS, 1));
    httpClient = clientBuilder.build();
}
 

protected FedX(Config config, Cache cache, Statistics statistics, EndpointListProvider endpointListProvider, SummaryProvider summaryProvider) {
	    this.config = config;
	    this.cache = cache;
	    this.statistics = statistics;
	    this.endpointListProvider = endpointListProvider;
	    this.summaryProvider = summaryProvider;
	    
	       // initialize httpclient parameters
        HttpClientBuilder httpClientBuilder = HttpClientBuilders.getSSLTrustAllHttpClientBuilder();
        httpClientBuilder.setMaxConnTotal(config.getMaxHttpConnectionCount());
        httpClientBuilder.setMaxConnPerRoute(config.getMaxHttpConnectionCountPerRoute());

        //httpClientBuilder.evictExpiredConnections();
        //httpClientBuilder.setConnectionReuseStrategy(new NoConnectionReuseStrategy());
        //httpClientBuilder.setConnectionTimeToLive(1000, TimeUnit.MILLISECONDS);
        //httpClientBuilder.disableAutomaticRetries();

//      httpClientBuilder.setKeepAliveStrategy(new ConnectionKeepAliveStrategy(){
//
//          @Override
//          public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
//              return 0;
//          }});
        
        httpClient = httpClientBuilder.build();
        
	    synchronized (log) {
    	    if (monitoring == null) {
    	        monitoring = MonitoringFactory.createMonitoring(config);
    	    }
	    }
        
        executor = Executors.newCachedThreadPool();
        
        scheduler = new ControlledWorkerScheduler(config.getWorkerThreads(), "Evaluation Scheduler");
        if (log.isDebugEnabled()) {
            log.debug("Scheduler for async operations initialized with " + config.getWorkerThreads() + " worker threads.");
        }
        
        // initialize prefix declarations, if any
        String prefixFile = config.getPrefixDeclarations();
        if (prefixFile != null) {
            prefixDeclarations = new Properties();
            try {
                prefixDeclarations.load(new FileInputStream(new File(prefixFile)));
            } catch (IOException e) {
                throw new FedXRuntimeException("Error loading prefix properties: " + e.getMessage());
            }
        }
		open = true;
	}
 

protected HttpClientBuilder getClientBuilder(boolean useSSL, String url,
    Map<String, String> stringHeaders, int socketTimeout) {

  HttpClientBuilder cb = HttpClientBuilder.create();

  /*
   * Setup SSL
   */
  if (useSSL) {
    /*
     * All trusting SSL context
     */
    try {
      cb.setSSLContext(getSSLContext());
    } catch (Exception e) {
      throw new RuntimeException(e);
    }

    /*
     * All trusting hostname verifier
     */
    cb.setSSLHostnameVerifier(new HostnameVerifier() {
      public boolean verify(String s, SSLSession sslSession) {
        return true;
      }
    });
  }

  /*
   * Add default headers
   */
  ArrayList<BasicHeader> headers = new ArrayList<BasicHeader>(stringHeaders.size());
  stringHeaders.forEach((k, v) -> headers.add(new BasicHeader(k, v)));
  cb.setDefaultHeaders(headers);

  /*
   * Set socket timeout and transport threads
   */
  SocketConfig sc = SocketConfig.custom().setSoTimeout(socketTimeout).build();
  cb.setDefaultSocketConfig(sc);
  cb.setMaxConnPerRoute(this.config.getThreads());
  cb.setMaxConnTotal(this.config.getThreads());

  return cb;
}
 

public HttpClient createClient() {
    HttpClientBuilder clientBuilder;
    if (connectionParams.useTlsConfigFromEnvironment()) {
        clientBuilder = VespaHttpClientBuilder.create();
    } else {
        clientBuilder = HttpClientBuilder.create();
        if (connectionParams.getSslContext() != null) {
            setSslContext(clientBuilder, connectionParams.getSslContext());
        } else {
            SslContextBuilder builder = new SslContextBuilder();
            if (connectionParams.getPrivateKey() != null && connectionParams.getCertificate() != null) {
                builder.withKeyStore(connectionParams.getPrivateKey(), connectionParams.getCertificate());
            }
            if (connectionParams.getCaCertificates() != null) {
                builder.withTrustStore(connectionParams.getCaCertificates());
            }
            setSslContext(clientBuilder, builder.build());
        }
        if (connectionParams.getHostnameVerifier() != null) {
            clientBuilder.setSSLHostnameVerifier(connectionParams.getHostnameVerifier());
        }
        clientBuilder.setUserTokenHandler(context -> null); // https://stackoverflow.com/a/42112034/1615280
    }
    clientBuilder.setMaxConnPerRoute(1);
    clientBuilder.setMaxConnTotal(1);
    clientBuilder.setConnectionTimeToLive(connectionParams.getConnectionTimeToLive().getSeconds(), TimeUnit.SECONDS);
    clientBuilder.setUserAgent(String.format("vespa-http-client (%s)", Vtag.currentVersion));
    clientBuilder.setDefaultHeaders(Collections.singletonList(new BasicHeader(Headers.CLIENT_VERSION, Vtag.currentVersion)));
    clientBuilder.disableContentCompression();
    // Try to disable the disabling to see if system tests become stable again.
    // clientBuilder.disableAutomaticRetries();
    RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setSocketTimeout(0);
    if (connectionParams.getProxyHost() != null) {
        requestConfigBuilder.setProxy(new HttpHost(connectionParams.getProxyHost(), connectionParams.getProxyPort()));
    }
    clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build());

    log.fine("Creating HttpClient: " + " ConnectionTimeout "
                    + " SocketTimeout 0 secs "
                    + " proxyhost (can be null) " + connectionParams.getProxyHost()
                    + ":" + connectionParams.getProxyPort()
                    + (useSsl ? " using ssl " : " not using ssl")
    );
    return clientBuilder.build();
}
 

@Override
public HttpClient createHttpClient(FilterConfig filterConfig) {
  final String serviceRole = filterConfig.getInitParameter(PARAMETER_SERVICE_ROLE);
  HttpClientBuilder builder;
  GatewayConfig gatewayConfig = (GatewayConfig) filterConfig.getServletContext().getAttribute(GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE);
  GatewayServices services = (GatewayServices) filterConfig.getServletContext()
      .getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE);
  if (gatewayConfig != null && gatewayConfig.isMetricsEnabled()) {
    MetricsService metricsService = services.getService(ServiceType.METRICS_SERVICE);
    builder = metricsService.getInstrumented(HttpClientBuilder.class);
  } else {
    builder = HttpClients.custom();
  }

  // Conditionally set a custom SSLContext
  SSLContext sslContext = createSSLContext(services, filterConfig, serviceRole);
  if(sslContext != null) {
    builder.setSSLSocketFactory(new SSLConnectionSocketFactory(sslContext));
  }

  if (Boolean.parseBoolean(System.getProperty(GatewayConfig.HADOOP_KERBEROS_SECURED))) {
    CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(AuthScope.ANY, new UseJaasCredentials());

    Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
        .register(AuthSchemes.SPNEGO, new KnoxSpnegoAuthSchemeFactory(true))
        .build();

    builder.setDefaultAuthSchemeRegistry(authSchemeRegistry)
        .setDefaultCookieStore(new HadoopAuthCookieStore(gatewayConfig))
        .setDefaultCredentialsProvider(credentialsProvider);
  } else {
    builder.setDefaultCookieStore(new NoCookieStore());
  }

  builder.setKeepAliveStrategy( DefaultConnectionKeepAliveStrategy.INSTANCE );
  builder.setConnectionReuseStrategy( DefaultConnectionReuseStrategy.INSTANCE );
  builder.setRedirectStrategy( new NeverRedirectStrategy() );
  builder.setRetryHandler( new NeverRetryHandler() );

  int maxConnections = getMaxConnections( filterConfig );
  builder.setMaxConnTotal( maxConnections );
  builder.setMaxConnPerRoute( maxConnections );

  builder.setDefaultRequestConfig(getRequestConfig(filterConfig, serviceRole));

  // See KNOX-1530 for details
  builder.disableContentCompression();

  return builder.build();
}
 

private void download(BinlogFile binlogFile) throws Throwable {
    String downloadLink = binlogFile.getDownloadLink();
    String fileName = binlogFile.getFileName();

    downloadLink = downloadLink.trim();
    CloseableHttpClient httpClient = null;
    if (downloadLink.startsWith("https")) {
        HttpClientBuilder builder = HttpClientBuilder.create();
        builder.setMaxConnPerRoute(50);
        builder.setMaxConnTotal(100);
        // 创建支持忽略证书的https
        final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {

            @Override
            public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
                return true;
            }
        }).build();

        httpClient = HttpClientBuilder.create()
            .setSSLContext(sslContext)
            .setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create()
                .register("http", PlainConnectionSocketFactory.INSTANCE)
                .register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
                .build()))
            .build();
    } else {
        httpClient = HttpClientBuilder.create().setMaxConnPerRoute(50).setMaxConnTotal(100).build();
    }

    HttpGet httpGet = new HttpGet(downloadLink);
    RequestConfig requestConfig = RequestConfig.custom()
        .setConnectTimeout(TIMEOUT)
        .setConnectionRequestTimeout(TIMEOUT)
        .setSocketTimeout(TIMEOUT)
        .build();
    httpGet.setConfig(requestConfig);
    HttpResponse response = httpClient.execute(httpGet);
    int statusCode = response.getStatusLine().getStatusCode();
    if (statusCode != HttpResponseStatus.OK.code()) {
        throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:" + statusCode);
    }
    saveFile(new File(destDir), "mysql-bin." + fileName, response);
}