Java Code Examples for org.apache.http.impl.client.HttpClientBuilder#setMaxConnTotal()

The following examples show how to use org.apache.http.impl.client.HttpClientBuilder#setMaxConnTotal() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: canal   File: HttpHelper.java    License: Apache License 2.0 6 votes vote down vote up
public HttpHelper(){
    HttpClientBuilder builder = HttpClientBuilder.create();
    builder.setMaxConnPerRoute(50);
    builder.setMaxConnTotal(100);

    // 创建支持忽略证书的https
    try {
        SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {

            @Override
            public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
                return true;
            }
        }).build();

        httpclient = HttpClientBuilder.create()
            .setSSLContext(sslContext)
            .setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create()
                .register("http", PlainConnectionSocketFactory.INSTANCE)
                .register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
                .build()))
            .build();
    } catch (Throwable e) {
        // ignore
    }
}
 
Example 2
Source Project: commons-jcs   File: AbstractHttpClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Configures the http client.
 *
 * @param builder client builder to configure
 */
protected void configureClient(HttpClientBuilder builder)
{
    if ( getRemoteHttpCacheAttributes().getMaxConnectionsPerHost() > 0 )
    {
        builder.setMaxConnTotal(getRemoteHttpCacheAttributes().getMaxConnectionsPerHost());
        builder.setMaxConnPerRoute(getRemoteHttpCacheAttributes().getMaxConnectionsPerHost());
    }

    builder.setDefaultRequestConfig(RequestConfig.custom()
            .setConnectTimeout(getRemoteHttpCacheAttributes().getConnectionTimeoutMillis())
            .setSocketTimeout(getRemoteHttpCacheAttributes().getSocketTimeoutMillis())
            // By default we instruct HttpClient to ignore cookies.
            .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
            .build());
}
 
Example 3
Source Project: seppb   File: JenkinsFactory.java    License: MIT License 5 votes vote down vote up
public HttpClientBuilder clientBuilder() {
    HttpClientBuilder httpClientBuilder = HttpClientBuilder.create();
    RequestConfig.Builder builder = RequestConfig.custom();
    builder.setConnectionRequestTimeout(requestConnTimeout);
    builder.setSocketTimeout(socketTimeout);
    builder.setConnectTimeout(connTimeout);
    httpClientBuilder.setDefaultRequestConfig(builder.build());
    httpClientBuilder.setMaxConnPerRoute(maxConnPerRoute);
    httpClientBuilder.setMaxConnTotal(maxConnTotal);
    return httpClientBuilder;
}
 
Example 4
Source Project: canal-1.1.3   File: HttpHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static byte[] getBytes(String url, int timeout) throws Exception {
    long start = System.currentTimeMillis();
    HttpClientBuilder builder = HttpClientBuilder.create();
    builder.setMaxConnPerRoute(50);
    builder.setMaxConnTotal(100);
    CloseableHttpClient httpclient = builder.build();
    URI uri = new URIBuilder(url).build();
    RequestConfig config = custom().setConnectTimeout(timeout)
        .setConnectionRequestTimeout(timeout)
        .setSocketTimeout(timeout)
        .build();
    HttpGet httpGet = new HttpGet(uri);
    HttpClientContext context = HttpClientContext.create();
    context.setRequestConfig(config);
    CloseableHttpResponse response = httpclient.execute(httpGet, context);
    try {
        int statusCode = response.getStatusLine().getStatusCode();
        long end = System.currentTimeMillis();
        long cost = end - start;
        if (logger.isWarnEnabled()) {
            logger.warn("post " + url + ", cost : " + cost);
        }
        if (statusCode == HttpStatus.SC_OK) {
            return EntityUtils.toByteArray(response.getEntity());
        } else {
            String errorMsg = EntityUtils.toString(response.getEntity());
            throw new RuntimeException("requestGet remote error, url=" + uri.toString() + ", code=" + statusCode
                                       + ", error msg=" + errorMsg);
        }
    } finally {
        response.close();
        httpGet.releaseConnection();
    }
}
 
Example 5
/**
 * Utility method to create a {@link CloseableHttpClient} to make http POSTs against Splunk's
 * HEC.
 *
 * @param maxConnections max number of parallel connections.
 * @param disableCertificateValidation should disable certificate validation.
 */
private CloseableHttpClient getHttpClient(
    int maxConnections, boolean disableCertificateValidation)
    throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {

  HttpClientBuilder builder = ApacheHttpTransport.newDefaultHttpClientBuilder();

  if (genericUrl().getScheme().equalsIgnoreCase(HTTPS_PROTOCOL_PREFIX)) {
    LOG.info("SSL connection requested");

    HostnameVerifier hostnameVerifier =
        disableCertificateValidation
            ? NoopHostnameVerifier.INSTANCE
            : new DefaultHostnameVerifier();

    SSLContextBuilder sslContextBuilder = SSLContextBuilder.create();
    if (disableCertificateValidation) {
      LOG.info("Certificate validation is disabled");
      sslContextBuilder.loadTrustMaterial((TrustStrategy) (chain, authType) -> true);
    }

    SSLConnectionSocketFactory connectionSocketFactory =
        new SSLConnectionSocketFactory(sslContextBuilder.build(), hostnameVerifier);
    builder.setSSLSocketFactory(connectionSocketFactory);
  }

  builder.setMaxConnTotal(maxConnections);
  builder.setDefaultRequestConfig(
      RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build());

  return builder.build();
}
 
Example 6
Source Project: beam   File: HttpEventPublisher.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link CloseableHttpClient} to make HTTP POSTs against Splunk's HEC.
 *
 * @param maxConnections max number of parallel connections
 * @param disableCertificateValidation should disable certificate validation
 */
private CloseableHttpClient getHttpClient(
    int maxConnections, boolean disableCertificateValidation)
    throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException {

  HttpClientBuilder builder = ApacheHttpTransport.newDefaultHttpClientBuilder();

  if (genericUrl().getScheme().equalsIgnoreCase(HTTPS_PROTOCOL_PREFIX)) {
    LOG.info("SSL connection requested");

    HostnameVerifier hostnameVerifier =
        disableCertificateValidation
            ? NoopHostnameVerifier.INSTANCE
            : new DefaultHostnameVerifier();

    SSLContextBuilder sslContextBuilder = SSLContextBuilder.create();
    if (disableCertificateValidation) {
      LOG.info("Certificate validation is disabled");
      sslContextBuilder.loadTrustMaterial((TrustStrategy) (chain, authType) -> true);
    }

    SSLConnectionSocketFactory connectionSocketFactory =
        new SSLConnectionSocketFactory(sslContextBuilder.build(), hostnameVerifier);
    builder.setSSLSocketFactory(connectionSocketFactory);
  }

  builder.setMaxConnTotal(maxConnections);
  builder.setDefaultRequestConfig(
      RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build());

  return builder.build();
}
 
Example 7
Source Project: canal   File: HttpHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static byte[] getBytes(String url, int timeout) throws Exception {
    long start = System.currentTimeMillis();
    HttpClientBuilder builder = HttpClientBuilder.create();
    builder.setMaxConnPerRoute(50);
    builder.setMaxConnTotal(100);
    CloseableHttpClient httpclient = builder.build();
    URI uri = new URIBuilder(url).build();
    RequestConfig config = custom().setConnectTimeout(timeout)
        .setConnectionRequestTimeout(timeout)
        .setSocketTimeout(timeout)
        .build();
    HttpGet httpGet = new HttpGet(uri);
    HttpClientContext context = HttpClientContext.create();
    context.setRequestConfig(config);
    CloseableHttpResponse response = httpclient.execute(httpGet, context);
    try {
        int statusCode = response.getStatusLine().getStatusCode();
        long end = System.currentTimeMillis();
        long cost = end - start;
        if (logger.isWarnEnabled()) {
            logger.warn("post " + url + ", cost : " + cost);
        }
        if (statusCode == HttpStatus.SC_OK) {
            return EntityUtils.toByteArray(response.getEntity());
        } else {
            String errorMsg = EntityUtils.toString(response.getEntity());
            throw new RuntimeException("requestGet remote error, url=" + uri.toString() + ", code=" + statusCode
                                       + ", error msg=" + errorMsg);
        }
    } finally {
        response.close();
        httpGet.releaseConnection();
    }
}
 
Example 8
Source Project: vividus   File: HttpClientFactory.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public IHttpClient buildHttpClient(HttpClientConfig config)
{
    HttpClientBuilder builder = HttpClientBuilder.create();
    builder.setDefaultHeaders(config.createHeaders());
    if (config.hasCookieStore())
    {
        builder.setDefaultCookieStore(config.getCookieStore());
    }
    if (config.hasCredentials())
    {
        AuthScope authScope = config.hasAuthScope() ? config.getAuthScope()
                : ClientBuilderUtils.DEFAULT_AUTH_SCOPE;
        CredentialsProvider credProvider = ClientBuilderUtils.createCredentialsProvider(authScope,
                config.getCredentials());
        builder.setDefaultCredentialsProvider(credProvider);
    }

    sslContextFactory.getSslContext(SSLConnectionSocketFactory.SSL, !config.isSslCertificateCheckEnabled())
            .ifPresent(builder::setSSLContext);

    if (!config.isSslHostnameVerificationEnabled())
    {
        builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
    }
    builder.setConnectionManager(config.getConnectionManager());
    builder.setMaxConnTotal(config.getMaxTotalConnections());
    builder.setMaxConnPerRoute(config.getMaxConnectionsPerRoute());
    builder.addInterceptorLast(config.getLastRequestInterceptor());
    builder.addInterceptorLast(config.getLastResponseInterceptor());
    builder.setRedirectStrategy(config.getRedirectStrategy());
    builder.setRetryHandler(config.getHttpRequestRetryHandler());
    Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setConnectionRequestTimeout(config.getConnectionRequestTimeout());
    requestConfigBuilder.setConnectTimeout(config.getConnectTimeout());
    requestConfigBuilder.setCircularRedirectsAllowed(config.isCircularRedirectsAllowed());
    requestConfigBuilder.setSocketTimeout(config.getSocketTimeout());
    Optional.ofNullable(config.getCookieSpec()).ifPresent(requestConfigBuilder::setCookieSpec);
    builder.setDefaultRequestConfig(requestConfigBuilder.build());
    builder.setDefaultSocketConfig(SocketConfig.copy(SocketConfig.DEFAULT)
            .setSoTimeout(config.getSocketTimeout())
            .build());
    builder.setDnsResolver(config.getDnsResolver());

    HttpClient httpClient = new HttpClient();
    httpClient.setCloseableHttpClient(builder.build());
    if (config.hasBaseUrl())
    {
        httpClient.setHttpHost(HttpHost.create(config.getBaseUrl()));
    }
    httpClient.setSkipResponseEntity(config.isSkipResponseEntity());
    return httpClient;
}
 
Example 9
Source Project: canal-1.1.3   File: BinlogDownloadQueue.java    License: Apache License 2.0 4 votes vote down vote up
private void download(BinlogFile binlogFile) throws Throwable {
    String downloadLink = binlogFile.getDownloadLink();
    String fileName = binlogFile.getFileName();

    downloadLink = downloadLink.trim();
    CloseableHttpClient httpClient = null;
    if (downloadLink.startsWith("https")) {
        HttpClientBuilder builder = HttpClientBuilder.create();
        builder.setMaxConnPerRoute(50);
        builder.setMaxConnTotal(100);
        // 创建支持忽略证书的https
        final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {

            @Override
            public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
                return true;
            }
        }).build();

        httpClient = HttpClientBuilder.create()
            .setSSLContext(sslContext)
            .setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create()
                .register("http", PlainConnectionSocketFactory.INSTANCE)
                .register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
                .build()))
            .build();
    } else {
        httpClient = HttpClientBuilder.create().setMaxConnPerRoute(50).setMaxConnTotal(100).build();
    }

    HttpGet httpGet = new HttpGet(downloadLink);
    RequestConfig requestConfig = RequestConfig.custom()
        .setConnectTimeout(TIMEOUT)
        .setConnectionRequestTimeout(TIMEOUT)
        .setSocketTimeout(TIMEOUT)
        .build();
    httpGet.setConfig(requestConfig);
    HttpResponse response = httpClient.execute(httpGet);
    int statusCode = response.getStatusLine().getStatusCode();
    if (statusCode != HttpResponseStatus.OK.code()) {
        throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:" + statusCode);
    }
    saveFile(new File(destDir), "mysql-bin." + fileName, response);
}
 
Example 10
/**
 * Initialize an {@link HttpClient} for performing requests. Proxy settings will
 * be read from the configuration and applied transparently.
 */
public HttpClientAdapter() {
    Optional<SSLConnectionSocketFactory> sslSocketFactory = Optional.empty();
    try {
        SSLContext sslContext = SSLContext.getDefault();
        sslSocketFactory = Optional.of(new SSLConnectionSocketFactory(
                sslContext,
                new String[]{"TLSv1", "TLSv1.1", "TLSv1.2"},
                null,
                SSLConnectionSocketFactory.getDefaultHostnameVerifier()));
    }
    catch (NoSuchAlgorithmException e) {
        log.warn("No such algorithm. Using default context", e);
    }

    HttpClientBuilder clientBuilder = HttpClientBuilder.create();
    sslSocketFactory.ifPresent(sf -> clientBuilder.setSSLSocketFactory(sf));

    clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
    Builder requestConfigBuilder = RequestConfig.custom()
            .setConnectTimeout(Config.get().getInt(HTTP_CONNECTION_TIMEOUT, 5) * TO_MILLISECONDS)
            .setSocketTimeout(Config.get().getInt(HTTP_SOCKET_TIMEOUT, 5 * 60) * TO_MILLISECONDS)
            .setCookieSpec(CookieSpecs.IGNORE_COOKIES);

    // Store the proxy settings
    String proxyHostname = ConfigDefaults.get().getProxyHost();
    if (!StringUtils.isBlank(proxyHostname)) {
        int proxyPort = ConfigDefaults.get().getProxyPort();

        proxyHost = new HttpHost(proxyHostname, proxyPort);
        clientBuilder.setProxy(proxyHost);

        String proxyUsername = ConfigDefaults.get().getProxyUsername();
        String proxyPassword = ConfigDefaults.get().getProxyPassword();
        if (!StringUtils.isBlank(proxyUsername) &&
                !StringUtils.isBlank(proxyPassword)) {
            Credentials proxyCredentials = new UsernamePasswordCredentials(
                    proxyUsername, proxyPassword);

            credentialsProvider.setCredentials(new AuthScope(proxyHostname, proxyPort),
                    proxyCredentials);
        }

        // Explicitly exclude the NTLM authentication scheme
        requestConfigBuilder =  requestConfigBuilder.setProxyPreferredAuthSchemes(
                                Arrays.asList(AuthSchemes.DIGEST, AuthSchemes.BASIC));

        clientBuilder.setProxyAuthenticationStrategy(new ProxyAuthenticationStrategy());

        clientBuilder.setRoutePlanner(new CustomProxyRoutePlanner(proxyHost));
    }

    // Read proxy exceptions from the "no_proxy" config option
    String noProxy = Config.get().getString(NO_PROXY);
    if (!StringUtils.isBlank(noProxy)) {
        for (String domain : Arrays.asList(noProxy.split(","))) {
            noProxyDomains.add(domain.toLowerCase().trim());
        }
    }

    requestConfig = requestConfigBuilder.build();
    clientBuilder.setMaxConnPerRoute(Config.get().getInt(MAX_CONNCECTIONS, 1));
    clientBuilder.setMaxConnTotal(Config.get().getInt(MAX_CONNCECTIONS, 1));
    httpClient = clientBuilder.build();
}
 
Example 11
protected FedX(Config config, Cache cache, Statistics statistics, EndpointListProvider endpointListProvider, SummaryProvider summaryProvider) {
	    this.config = config;
	    this.cache = cache;
	    this.statistics = statistics;
	    this.endpointListProvider = endpointListProvider;
	    this.summaryProvider = summaryProvider;
	    
	       // initialize httpclient parameters
        HttpClientBuilder httpClientBuilder = HttpClientBuilders.getSSLTrustAllHttpClientBuilder();
        httpClientBuilder.setMaxConnTotal(config.getMaxHttpConnectionCount());
        httpClientBuilder.setMaxConnPerRoute(config.getMaxHttpConnectionCountPerRoute());

        //httpClientBuilder.evictExpiredConnections();
        //httpClientBuilder.setConnectionReuseStrategy(new NoConnectionReuseStrategy());
        //httpClientBuilder.setConnectionTimeToLive(1000, TimeUnit.MILLISECONDS);
        //httpClientBuilder.disableAutomaticRetries();

//      httpClientBuilder.setKeepAliveStrategy(new ConnectionKeepAliveStrategy(){
//
//          @Override
//          public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
//              return 0;
//          }});
        
        httpClient = httpClientBuilder.build();
        
	    synchronized (log) {
    	    if (monitoring == null) {
    	        monitoring = MonitoringFactory.createMonitoring(config);
    	    }
	    }
        
        executor = Executors.newCachedThreadPool();
        
        scheduler = new ControlledWorkerScheduler(config.getWorkerThreads(), "Evaluation Scheduler");
        if (log.isDebugEnabled()) {
            log.debug("Scheduler for async operations initialized with " + config.getWorkerThreads() + " worker threads.");
        }
        
        // initialize prefix declarations, if any
        String prefixFile = config.getPrefixDeclarations();
        if (prefixFile != null) {
            prefixDeclarations = new Properties();
            try {
                prefixDeclarations.load(new FileInputStream(new File(prefixFile)));
            } catch (IOException e) {
                throw new FedXRuntimeException("Error loading prefix properties: " + e.getMessage());
            }
        }
		open = true;
	}
 
Example 12
protected HttpClientBuilder getClientBuilder(boolean useSSL, String url,
    Map<String, String> stringHeaders, int socketTimeout) {

  HttpClientBuilder cb = HttpClientBuilder.create();

  /*
   * Setup SSL
   */
  if (useSSL) {
    /*
     * All trusting SSL context
     */
    try {
      cb.setSSLContext(getSSLContext());
    } catch (Exception e) {
      throw new RuntimeException(e);
    }

    /*
     * All trusting hostname verifier
     */
    cb.setSSLHostnameVerifier(new HostnameVerifier() {
      public boolean verify(String s, SSLSession sslSession) {
        return true;
      }
    });
  }

  /*
   * Add default headers
   */
  ArrayList<BasicHeader> headers = new ArrayList<BasicHeader>(stringHeaders.size());
  stringHeaders.forEach((k, v) -> headers.add(new BasicHeader(k, v)));
  cb.setDefaultHeaders(headers);

  /*
   * Set socket timeout and transport threads
   */
  SocketConfig sc = SocketConfig.custom().setSoTimeout(socketTimeout).build();
  cb.setDefaultSocketConfig(sc);
  cb.setMaxConnPerRoute(this.config.getThreads());
  cb.setMaxConnTotal(this.config.getThreads());

  return cb;
}
 
Example 13
Source Project: vespa   File: ApacheGatewayConnection.java    License: Apache License 2.0 4 votes vote down vote up
public HttpClient createClient() {
    HttpClientBuilder clientBuilder;
    if (connectionParams.useTlsConfigFromEnvironment()) {
        clientBuilder = VespaHttpClientBuilder.create();
    } else {
        clientBuilder = HttpClientBuilder.create();
        if (connectionParams.getSslContext() != null) {
            setSslContext(clientBuilder, connectionParams.getSslContext());
        } else {
            SslContextBuilder builder = new SslContextBuilder();
            if (connectionParams.getPrivateKey() != null && connectionParams.getCertificate() != null) {
                builder.withKeyStore(connectionParams.getPrivateKey(), connectionParams.getCertificate());
            }
            if (connectionParams.getCaCertificates() != null) {
                builder.withTrustStore(connectionParams.getCaCertificates());
            }
            setSslContext(clientBuilder, builder.build());
        }
        if (connectionParams.getHostnameVerifier() != null) {
            clientBuilder.setSSLHostnameVerifier(connectionParams.getHostnameVerifier());
        }
        clientBuilder.setUserTokenHandler(context -> null); // https://stackoverflow.com/a/42112034/1615280
    }
    clientBuilder.setMaxConnPerRoute(1);
    clientBuilder.setMaxConnTotal(1);
    clientBuilder.setConnectionTimeToLive(connectionParams.getConnectionTimeToLive().getSeconds(), TimeUnit.SECONDS);
    clientBuilder.setUserAgent(String.format("vespa-http-client (%s)", Vtag.currentVersion));
    clientBuilder.setDefaultHeaders(Collections.singletonList(new BasicHeader(Headers.CLIENT_VERSION, Vtag.currentVersion)));
    clientBuilder.disableContentCompression();
    // Try to disable the disabling to see if system tests become stable again.
    // clientBuilder.disableAutomaticRetries();
    RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setSocketTimeout(0);
    if (connectionParams.getProxyHost() != null) {
        requestConfigBuilder.setProxy(new HttpHost(connectionParams.getProxyHost(), connectionParams.getProxyPort()));
    }
    clientBuilder.setDefaultRequestConfig(requestConfigBuilder.build());

    log.fine("Creating HttpClient: " + " ConnectionTimeout "
                    + " SocketTimeout 0 secs "
                    + " proxyhost (can be null) " + connectionParams.getProxyHost()
                    + ":" + connectionParams.getProxyPort()
                    + (useSsl ? " using ssl " : " not using ssl")
    );
    return clientBuilder.build();
}
 
Example 14
Source Project: knox   File: DefaultHttpClientFactory.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public HttpClient createHttpClient(FilterConfig filterConfig) {
  final String serviceRole = filterConfig.getInitParameter(PARAMETER_SERVICE_ROLE);
  HttpClientBuilder builder;
  GatewayConfig gatewayConfig = (GatewayConfig) filterConfig.getServletContext().getAttribute(GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE);
  GatewayServices services = (GatewayServices) filterConfig.getServletContext()
      .getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE);
  if (gatewayConfig != null && gatewayConfig.isMetricsEnabled()) {
    MetricsService metricsService = services.getService(ServiceType.METRICS_SERVICE);
    builder = metricsService.getInstrumented(HttpClientBuilder.class);
  } else {
    builder = HttpClients.custom();
  }

  // Conditionally set a custom SSLContext
  SSLContext sslContext = createSSLContext(services, filterConfig, serviceRole);
  if(sslContext != null) {
    builder.setSSLSocketFactory(new SSLConnectionSocketFactory(sslContext));
  }

  if (Boolean.parseBoolean(System.getProperty(GatewayConfig.HADOOP_KERBEROS_SECURED))) {
    CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(AuthScope.ANY, new UseJaasCredentials());

    Registry<AuthSchemeProvider> authSchemeRegistry = RegistryBuilder.<AuthSchemeProvider>create()
        .register(AuthSchemes.SPNEGO, new KnoxSpnegoAuthSchemeFactory(true))
        .build();

    builder.setDefaultAuthSchemeRegistry(authSchemeRegistry)
        .setDefaultCookieStore(new HadoopAuthCookieStore(gatewayConfig))
        .setDefaultCredentialsProvider(credentialsProvider);
  } else {
    builder.setDefaultCookieStore(new NoCookieStore());
  }

  builder.setKeepAliveStrategy( DefaultConnectionKeepAliveStrategy.INSTANCE );
  builder.setConnectionReuseStrategy( DefaultConnectionReuseStrategy.INSTANCE );
  builder.setRedirectStrategy( new NeverRedirectStrategy() );
  builder.setRetryHandler( new NeverRetryHandler() );

  int maxConnections = getMaxConnections( filterConfig );
  builder.setMaxConnTotal( maxConnections );
  builder.setMaxConnPerRoute( maxConnections );

  builder.setDefaultRequestConfig(getRequestConfig(filterConfig, serviceRole));

  // See KNOX-1530 for details
  builder.disableContentCompression();

  return builder.build();
}
 
Example 15
Source Project: canal   File: BinlogDownloadQueue.java    License: Apache License 2.0 4 votes vote down vote up
private void download(BinlogFile binlogFile) throws Throwable {
    String downloadLink = binlogFile.getDownloadLink();
    String fileName = binlogFile.getFileName();

    downloadLink = downloadLink.trim();
    CloseableHttpClient httpClient = null;
    if (downloadLink.startsWith("https")) {
        HttpClientBuilder builder = HttpClientBuilder.create();
        builder.setMaxConnPerRoute(50);
        builder.setMaxConnTotal(100);
        // 创建支持忽略证书的https
        final SSLContext sslContext = new SSLContextBuilder().loadTrustMaterial(null, new TrustStrategy() {

            @Override
            public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
                return true;
            }
        }).build();

        httpClient = HttpClientBuilder.create()
            .setSSLContext(sslContext)
            .setConnectionManager(new PoolingHttpClientConnectionManager(RegistryBuilder.<ConnectionSocketFactory> create()
                .register("http", PlainConnectionSocketFactory.INSTANCE)
                .register("https", new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE))
                .build()))
            .build();
    } else {
        httpClient = HttpClientBuilder.create().setMaxConnPerRoute(50).setMaxConnTotal(100).build();
    }

    HttpGet httpGet = new HttpGet(downloadLink);
    RequestConfig requestConfig = RequestConfig.custom()
        .setConnectTimeout(TIMEOUT)
        .setConnectionRequestTimeout(TIMEOUT)
        .setSocketTimeout(TIMEOUT)
        .build();
    httpGet.setConfig(requestConfig);
    HttpResponse response = httpClient.execute(httpGet);
    int statusCode = response.getStatusLine().getStatusCode();
    if (statusCode != HttpResponseStatus.OK.code()) {
        throw new RuntimeException("download failed , url:" + downloadLink + " , statusCode:" + statusCode);
    }
    saveFile(new File(destDir), "mysql-bin." + fileName, response);
}