Java Code Examples for org.apache.http.impl.conn.PoolingClientConnectionManager#setMaxTotal()

The following examples show how to use org.apache.http.impl.conn.PoolingClientConnectionManager#setMaxTotal() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: RestClient.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private void init(String host, int port, String userName, String password) {
    this.host = host;
    this.port = port;
    this.userName = userName;
    this.password = password;
    this.baseUrl = SCHEME_HTTP + host + ":" + port + KYLIN_API_PATH;

    final HttpParams httpParams = new BasicHttpParams();
    HttpConnectionParams.setSoTimeout(httpParams, httpSocketTimeoutMs);
    HttpConnectionParams.setConnectionTimeout(httpParams, httpConnectionTimeoutMs);

    final PoolingClientConnectionManager cm = new PoolingClientConnectionManager();
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    cm.setDefaultMaxPerRoute(config.getRestClientDefaultMaxPerRoute());
    cm.setMaxTotal(config.getRestClientMaxTotal());

    client = new DefaultHttpClient(cm, httpParams);

    if (userName != null && password != null) {
        CredentialsProvider provider = new BasicCredentialsProvider();
        UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(userName, password);
        provider.setCredentials(AuthScope.ANY, credentials);
        client.setCredentialsProvider(provider);
    }
}

Example 2

Source File: DockerClient.java From docker-java with Apache License 2.0

6 votes

private DockerClient(Config config) {
	restEndpointUrl = config.url + "/v" + config.version;
	ClientConfig clientConfig = new DefaultClientConfig();
	//clientConfig.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, Boolean.TRUE);

	SchemeRegistry schemeRegistry = new SchemeRegistry();
	schemeRegistry.register(new Scheme("http", config.url.getPort(), PlainSocketFactory.getSocketFactory()));
	schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));

	PoolingClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
	// Increase max total connection
	cm.setMaxTotal(1000);
	// Increase default max connection per route
	cm.setDefaultMaxPerRoute(1000);

	HttpClient httpClient = new DefaultHttpClient(cm);
	client = new ApacheHttpClient4(new ApacheHttpClient4Handler(httpClient, null, false), clientConfig);

	client.setReadTimeout(10000);
	//Experimental support for unix sockets:
	//client = new UnixSocketClient(clientConfig);

	client.addFilter(new JsonClientFilter());
	client.addFilter(new LoggingFilter());
}

Example 3

Source File: RestClient.java From kylin with Apache License 2.0

6 votes

private void init(String host, int port, String userName, String password) {
    this.host = host;
    this.port = port;
    this.userName = userName;
    this.password = password;
    this.baseUrl = SCHEME_HTTP + host + ":" + port + KYLIN_API_PATH;

    final HttpParams httpParams = new BasicHttpParams();
    HttpConnectionParams.setSoTimeout(httpParams, httpSocketTimeoutMs);
    HttpConnectionParams.setConnectionTimeout(httpParams, httpConnectionTimeoutMs);

    final PoolingClientConnectionManager cm = new PoolingClientConnectionManager();
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    cm.setDefaultMaxPerRoute(config.getRestClientDefaultMaxPerRoute());
    cm.setMaxTotal(config.getRestClientMaxTotal());

    client = new DefaultHttpClient(cm, httpParams);

    if (userName != null && password != null) {
        CredentialsProvider provider = new BasicCredentialsProvider();
        UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(userName, password);
        provider.setCredentials(AuthScope.ANY, credentials);
        client.setCredentialsProvider(provider);
    }
}

Example 4

Source File: PlayManager.java From raccoon4 with Apache License 2.0

6 votes

/**
 * create a proxy client
 * 
 * @return either a client or null if none is configured
 * @throws KeyManagementException
 * @throws NumberFormatException
 *           if that port could not be parsed.
 * @throws NoSuchAlgorithmException
 */
private static HttpClient createProxyClient(PlayProfile profile)
		throws KeyManagementException, NoSuchAlgorithmException {
	if (profile.getProxyAddress() == null) {
		return null;
	}

	PoolingClientConnectionManager connManager = new PoolingClientConnectionManager(
			SchemeRegistryFactory.createDefault());
	connManager.setMaxTotal(100);
	connManager.setDefaultMaxPerRoute(30);

	DefaultHttpClient client = new DefaultHttpClient(connManager);
	client.getConnectionManager().getSchemeRegistry()
			.register(Utils.getMockedScheme());
	HttpHost proxy = new HttpHost(profile.getProxyAddress(),
			profile.getProxyPort());
	client.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
	if (profile.getProxyUser() != null && profile.getProxyPassword() != null) {
		client.getCredentialsProvider().setCredentials(
				new AuthScope(proxy),
				new UsernamePasswordCredentials(profile.getProxyUser(), profile
						.getProxyPassword()));
	}
	return client;
}

Example 5

Source File: HttpUtil.java From Leo with Apache License 2.0

5 votes

/**
 * 默认构造函数
 */
public HttpUtil() {
	this.charset = "UTF-8"; 
	PoolingClientConnectionManager pccm = new PoolingClientConnectionManager();
       pccm.setMaxTotal(100); //设置整个连接池最大链接数
	httpClient = new DefaultHttpClient(pccm);
	httpClient.getParams().setParameter(ClientPNames.COOKIE_POLICY,CookiePolicy.BROWSER_COMPATIBILITY);
	httpClient.getConnectionManager().closeIdleConnections(30,TimeUnit.SECONDS);
}

Example 6

Source File: RestClient.java From attic-stratos with Apache License 2.0

5 votes

public RestClient() {
    PoolingClientConnectionManager cm = new PoolingClientConnectionManager();
    // Increase max total connection to 200
    cm.setMaxTotal(200);
    // Increase default max connection per route to 50
    cm.setDefaultMaxPerRoute(50);

    httpClient = new DefaultHttpClient(cm);
}

Example 7

Source File: TalosClientFactory.java From galaxy-sdk-java with Apache License 2.0

5 votes

public static HttpClient generateHttpClient(final int maxTotalConnections,
    final int maxTotalConnectionsPerRoute, int connTimeout) {
  SchemeRegistry schemeRegistry = new SchemeRegistry();
  schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
  schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));

  PoolingClientConnectionManager conMgr = new PoolingClientConnectionManager(schemeRegistry);
  conMgr.setMaxTotal(maxTotalConnections);
  conMgr.setDefaultMaxPerRoute(maxTotalConnectionsPerRoute);

  HttpParams httpParams = new BasicHttpParams();
  HttpConnectionParams.setConnectionTimeout(httpParams, connTimeout);
  return new DefaultHttpClient(conMgr, httpParams);
}

Example 8

Source File: HBaseIndexerMapper.java From hbase-indexer with Apache License 2.0

5 votes

private DirectSolrClassicInputDocumentWriter createClassicSolrWriter(Context context,
                                                                     Map<String, String> indexConnectionParams)
        throws IOException {
    PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
    connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams));
    connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams));

    HttpClient httpClient = new DefaultHttpClient(connectionManager);
    List<SolrClient> solrServers = createHttpSolrClients(indexConnectionParams, httpClient);

    return new DirectSolrClassicInputDocumentWriter(
            context.getConfiguration().get(INDEX_NAME_CONF_KEY), solrServers);
}

Example 9

Source File: GooglePlayAPI.java From dummydroid with Apache License 2.0

5 votes

/**
 * Connection manager to allow concurrent connections.
 *
 * @return {@link ClientConnectionManager} instance
 */
public static ClientConnectionManager getConnectionManager() {
	PoolingClientConnectionManager connManager = new PoolingClientConnectionManager(
			SchemeRegistryFactory.createDefault());
	connManager.setMaxTotal(100);
	connManager.setDefaultMaxPerRoute(30);
	return connManager;
}

Example 10

Source File: HttpClientPoolFactory.java From monasca-common with Apache License 2.0

5 votes

HttpClientPoolFactory(String host, int port, boolean useHttps, int timeout,
  boolean clientAuth, String keyStore, String keyPass,
  String trustStore, String trustPass, String adminToken,
  int maxActive, long timeBetweenEvictionRunsMillis,
  long minEvictableIdleTimeMillis) {
  // Setup auth URL
  String protocol = useHttps ? "https://" : "http://";
  String urlStr = protocol + host + ":" + port;
  uri = URI.create(urlStr);

  // Setup connection pool
  SchemeRegistry schemeRegistry = new SchemeRegistry();
  if (protocol.startsWith("https")) {
    SSLSocketFactory sslf = sslFactory(keyStore, keyPass, trustStore,
      trustPass, clientAuth);
    schemeRegistry.register(new Scheme("https", port, sslf));
  } else {
    schemeRegistry.register(new Scheme("http", port, PlainSocketFactory
      .getSocketFactory()));
  }
  connMgr = new PoolingClientConnectionManager(schemeRegistry,
    minEvictableIdleTimeMillis, TimeUnit.MILLISECONDS);

  connMgr.setMaxTotal(maxActive);
  connMgr.setDefaultMaxPerRoute(maxActive);

  // Http connection timeout
  HttpParams params = new BasicHttpParams();
  params.setParameter(CoreConnectionPNames.SO_TIMEOUT, timeout);
  params.setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, timeout);

  // Create a single client
  client = new DefaultHttpClient(connMgr, params);

  // Create and start the connection pool cleaner
  cleaner = new HttpPoolCleaner(connMgr, timeBetweenEvictionRunsMillis,
    minEvictableIdleTimeMillis);
  new Thread(cleaner).start();

}

Example 11

Source File: LivyRestClient.java From kylin with Apache License 2.0

5 votes

private void init() {
    final HttpParams httpParams = new BasicHttpParams();
    HttpConnectionParams.setSoTimeout(httpParams, httpSocketTimeoutMs);
    HttpConnectionParams.setConnectionTimeout(httpParams, httpConnectionTimeoutMs);

    PoolingClientConnectionManager cm = new PoolingClientConnectionManager();
    cm.setDefaultMaxPerRoute(config.getRestClientDefaultMaxPerRoute());
    cm.setMaxTotal(config.getRestClientMaxTotal());

    baseUrl = config.getLivyUrl();
    client = new DefaultHttpClient(cm, httpParams);
}

Example 12

Source File: GooglePlayAPI.java From raccoon4 with Apache License 2.0

5 votes

/**
 * Connection manager to allow concurrent connections.
 * 
 * @return {@link ClientConnectionManager} instance
 */
public static ClientConnectionManager getConnectionManager() {
	PoolingClientConnectionManager connManager = new PoolingClientConnectionManager(
			SchemeRegistryFactory.createDefault());
	connManager.setMaxTotal(100);
	connManager.setDefaultMaxPerRoute(30);
	return connManager;
}

Example 13

Source File: VmApiProxyDelegate.java From appengine-java-vm-runtime with Apache License 2.0

4 votes

private static ClientConnectionManager createConnectionManager() {
  PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
  connectionManager.setMaxTotal(VmApiProxyEnvironment.MAX_CONCURRENT_API_CALLS);
  connectionManager.setDefaultMaxPerRoute(VmApiProxyEnvironment.MAX_CONCURRENT_API_CALLS);
  return connectionManager;
}

Example 14

Source File: UsergridExternalProvider.java From usergrid with Apache License 2.0

4 votes

private Client getJerseyClient() {

        if (jerseyClient == null) {

            synchronized (this) {

                // create HTTPClient and with configured connection pool

                int poolSize = 100; // connections
                final String poolSizeStr = properties.getProperty(CENTRAL_CONNECTION_POOL_SIZE);
                if (poolSizeStr != null) {
                    poolSize = Integer.parseInt(poolSizeStr);
                }

                PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
                connectionManager.setMaxTotal(poolSize);

                int timeout = 20000; // ms
                final String timeoutStr = properties.getProperty(CENTRAL_CONNECTION_TIMEOUT);
                if (timeoutStr != null) {
                    timeout = Integer.parseInt(timeoutStr);
                }

                int readTimeout = 20000; // ms
                final String readTimeoutStr = properties.getProperty(CENTRAL_READ_TIMEOUT);
                if (readTimeoutStr != null) {
                    readTimeout = Integer.parseInt(readTimeoutStr);
                }

                ClientConfig clientConfig = new ClientConfig();
                clientConfig.register(new JacksonFeature());
                clientConfig.property(ApacheClientProperties.CONNECTION_MANAGER, connectionManager);
                clientConfig.connectorProvider(new ApacheConnectorProvider());

                jerseyClient = ClientBuilder.newClient(clientConfig);
                jerseyClient.property(ClientProperties.CONNECT_TIMEOUT, timeout);
                jerseyClient.property(ClientProperties.READ_TIMEOUT, readTimeout);
            }
        }

        return jerseyClient;

    }

Example 15

Source File: HttpClientFactoryImpl.java From swift-explorer with Apache License 2.0

4 votes

protected PoolingClientConnectionManager initConnectionManager() {
    PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
    connectionManager.setMaxTotal(50);
    connectionManager.setDefaultMaxPerRoute(25);
    return connectionManager;
}

Example 16

Source File: RestProtocol.java From dubbox with Apache License 2.0

4 votes

protected <T> T doRefer(Class<T> serviceType, URL url) throws RpcException {
        if (connectionMonitor == null) {
            connectionMonitor = new ConnectionMonitor();
        }

        // TODO more configs to add

        PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
        // 20 is the default maxTotal of current PoolingClientConnectionManager
        connectionManager.setMaxTotal(url.getParameter(Constants.CONNECTIONS_KEY, 20));
        connectionManager.setDefaultMaxPerRoute(url.getParameter(Constants.CONNECTIONS_KEY, 20));

        connectionMonitor.addConnectionManager(connectionManager);

//        BasicHttpContext localContext = new BasicHttpContext();

        DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager);

        httpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() {
            public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
                HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
                while (it.hasNext()) {
                    HeaderElement he = it.nextElement();
                    String param = he.getName();
                    String value = he.getValue();
                    if (value != null && param.equalsIgnoreCase("timeout")) {
                        return Long.parseLong(value) * 1000;
                    }
                }
                // TODO constant
                return 30 * 1000;
            }
        });

        HttpParams params = httpClient.getParams();
        // TODO currently no xml config for Constants.CONNECT_TIMEOUT_KEY so we directly reuse Constants.TIMEOUT_KEY for now
        HttpConnectionParams.setConnectionTimeout(params, url.getParameter(Constants.TIMEOUT_KEY, Constants.DEFAULT_TIMEOUT));
        HttpConnectionParams.setSoTimeout(params, url.getParameter(Constants.TIMEOUT_KEY, Constants.DEFAULT_TIMEOUT));
        HttpConnectionParams.setTcpNoDelay(params, true);
        HttpConnectionParams.setSoKeepalive(params, true);

        ApacheHttpClient4Engine engine = new ApacheHttpClient4Engine(httpClient/*, localContext*/);

        ResteasyClient client = new ResteasyClientBuilder().httpEngine(engine).build();
        clients.add(client);

        client.register(RpcContextFilter.class);
        for (String clazz : Constants.COMMA_SPLIT_PATTERN.split(url.getParameter(Constants.EXTENSION_KEY, ""))) {
            if (!StringUtils.isEmpty(clazz)) {
                try {
                    client.register(Thread.currentThread().getContextClassLoader().loadClass(clazz.trim()));
                } catch (ClassNotFoundException e) {
                    throw new RpcException("Error loading JAX-RS extension class: " + clazz.trim(), e);
                }
            }
        }

        // TODO protocol
        ResteasyWebTarget target = client.target("http://" + url.getHost() + ":" + url.getPort() + "/" + getContextPath(url));
        return target.proxy(serviceType);
    }

Example 17

Source File: RestProtocol.java From dubbox with Apache License 2.0

4 votes

protected <T> T doRefer(Class<T> serviceType, URL url) throws RpcException {
        if (connectionMonitor == null) {
            connectionMonitor = new ConnectionMonitor();
        }

        // TODO more configs to add

        PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
        // 20 is the default maxTotal of current PoolingClientConnectionManager
        connectionManager.setMaxTotal(url.getParameter(Constants.CONNECTIONS_KEY, 20));
        connectionManager.setDefaultMaxPerRoute(url.getParameter(Constants.CONNECTIONS_KEY, 20));

        connectionMonitor.addConnectionManager(connectionManager);

//        BasicHttpContext localContext = new BasicHttpContext();

        DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager);

        httpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() {
            public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
                HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
                while (it.hasNext()) {
                    HeaderElement he = it.nextElement();
                    String param = he.getName();
                    String value = he.getValue();
                    if (value != null && param.equalsIgnoreCase("timeout")) {
                        return Long.parseLong(value) * 1000;
                    }
                }
                // TODO constant
                return 30 * 1000;
            }
        });

        HttpParams params = httpClient.getParams();
        // TODO currently no xml config for Constants.CONNECT_TIMEOUT_KEY so we directly reuse Constants.TIMEOUT_KEY for now
        HttpConnectionParams.setConnectionTimeout(params, url.getParameter(Constants.TIMEOUT_KEY, Constants.DEFAULT_TIMEOUT));
        HttpConnectionParams.setSoTimeout(params, url.getParameter(Constants.TIMEOUT_KEY, Constants.DEFAULT_TIMEOUT));
        HttpConnectionParams.setTcpNoDelay(params, true);
        HttpConnectionParams.setSoKeepalive(params, true);

        ApacheHttpClient4Engine engine = new ApacheHttpClient4Engine(httpClient/*, localContext*/);

        ResteasyClient client = new ResteasyClientBuilder().httpEngine(engine).build();
        clients.add(client);

        client.register(RpcContextFilter.class);
        for (String clazz : Constants.COMMA_SPLIT_PATTERN.split(url.getParameter(Constants.EXTENSION_KEY, ""))) {
            if (!StringUtils.isEmpty(clazz)) {
                try {
                    client.register(Thread.currentThread().getContextClassLoader().loadClass(clazz.trim()));
                } catch (ClassNotFoundException e) {
                    throw new RpcException("Error loading JAX-RS extension class: " + clazz.trim(), e);
                }
            }
        }

        // TODO protocol
        ResteasyWebTarget target = client.target("http://" + url.getHost() + ":" + url.getPort() + "/" + getContextPath(url));
        return target.proxy(serviceType);
    }

Example 18

Source File: RestProtocol.java From dubbox-hystrix with Apache License 2.0

4 votes

protected <T> T doRefer(Class<T> serviceType, URL url) throws RpcException {
        if (connectionMonitor == null) {
            connectionMonitor = new ConnectionMonitor();
        }

        // TODO more configs to add

        PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager();
        // 20 is the default maxTotal of current PoolingClientConnectionManager
        connectionManager.setMaxTotal(url.getParameter(Constants.CONNECTIONS_KEY, 20));
        connectionManager.setDefaultMaxPerRoute(url.getParameter(Constants.CONNECTIONS_KEY, 20));

        connectionMonitor.addConnectionManager(connectionManager);

//        BasicHttpContext localContext = new BasicHttpContext();

        DefaultHttpClient httpClient = new DefaultHttpClient(connectionManager);

        httpClient.setKeepAliveStrategy(new ConnectionKeepAliveStrategy() {
            public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
                HeaderElementIterator it = new BasicHeaderElementIterator(response.headerIterator(HTTP.CONN_KEEP_ALIVE));
                while (it.hasNext()) {
                    HeaderElement he = it.nextElement();
                    String param = he.getName();
                    String value = he.getValue();
                    if (value != null && param.equalsIgnoreCase("timeout")) {
                        return Long.parseLong(value) * 1000;
                    }
                }
                // TODO constant
                return 30 * 1000;
            }
        });

        HttpParams params = httpClient.getParams();
        // TODO currently no xml config for Constants.CONNECT_TIMEOUT_KEY so we directly reuse Constants.TIMEOUT_KEY for now
        HttpConnectionParams.setConnectionTimeout(params, url.getParameter(Constants.TIMEOUT_KEY, Constants.DEFAULT_TIMEOUT));
        HttpConnectionParams.setSoTimeout(params, url.getParameter(Constants.TIMEOUT_KEY, Constants.DEFAULT_TIMEOUT));
        HttpConnectionParams.setTcpNoDelay(params, true);
        HttpConnectionParams.setSoKeepalive(params, true);

        ApacheHttpClient4Engine engine = new ApacheHttpClient4Engine(httpClient/*, localContext*/);

        ResteasyClient client = new ResteasyClientBuilder().httpEngine(engine).build();
        clients.add(client);

        client.register(RpcContextFilter.class);
        for (String clazz : Constants.COMMA_SPLIT_PATTERN.split(url.getParameter(Constants.EXTENSION_KEY, ""))) {
            if (!StringUtils.isEmpty(clazz)) {
                try {
                    client.register(Thread.currentThread().getContextClassLoader().loadClass(clazz.trim()));
                } catch (ClassNotFoundException e) {
                    throw new RpcException("Error loading JAX-RS extension class: " + clazz.trim(), e);
                }
            }
        }

        // TODO protocol
        ResteasyWebTarget target = client.target("http://" + url.getHost() + ":" + url.getPort() + "/" + getContextPath(url));
        return target.proxy(serviceType);
    }

Example 19

Source File: HttpFactory.java From rainbow with Apache License 2.0

4 votes

private HttpFactory()
{
    try
    {
        int pooledHttpConn = 10;
        if (pooledHttpConn < 24)
        {
            pooledHttpConn = 24;
        }
        SchemeRegistry schemeRegistry = new SchemeRegistry();
        schemeRegistry.register(new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
        schemeRegistry.register(new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));

        cm = new PoolingClientConnectionManager(schemeRegistry);
        // Increase max total connection to 200
        cm.setMaxTotal(pooledHttpConn);
        // Increase default max connection per route to 20
        cm.setDefaultMaxPerRoute(2);
        // Increase max connections for localhost:80 to 50
        //HttpHost googleResearch = new HttpHost("research.google.com", 80);
        //HttpHost wikipediaEn = new HttpHost("en.wikipedia.org", 80);
        //cm.setMaxPerRoute(new HttpRoute(googleResearch), pooledHttpConn / 6);
        //cm.setMaxPerRoute(new HttpRoute(wikipediaEn), pooledHttpConn / 2);
        userAgents = new ArrayList<String>();
        rand = new Random(System.currentTimeMillis());
        userAgents
                .add("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.160 Safari/537.22");// linux
        // chrome25
        userAgents
                .add("Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.160 Safari/537.22");// linux
        // chrome
        userAgents
                .add("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.22 (KHTML, like Gecko) Chrome/25.0.1364.172 Safari/537.22");// win7
        // chrome
        userAgents.add("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:8.0) Gecko/20100101 Firefox/8.0");// win7
        // firefox
        userAgents.add("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)");// win7
        // ie9
        userAgents.add("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)");// win7
        // ie10
        userAgents.add("Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0)");// win7
        // ie8
        userAgents.add("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 5.1; Trident/5.0)");// xp
        // ie9
        userAgents.add("Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)");// xp
        // ie8
        userAgents
                .add("Mozilla/5.0 (X11; U; Linux i686; en-US) AppleWebKit/534.15 (KHTML, like Gecko) Ubuntu/10.10 Chromium/10.0.613.0 Chrome/10.0.613.0 Safari/534.15");// ubuntu11
        // chrome10
        userAgents
                .add("Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.15 (KHTML, like Gecko) Chrome/10.0.613.0 Safari/534.15");
        userAgents.add("Opera/9.80 (X11; Linux x86_64; U; en) Presto/2.7.62 Version/11.00");
        userAgents.add("Opera/9.80 (X11; Linux i686; U; en) Presto/2.7.62 Version/11.00");
        userAgents.add("Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00");
        userAgents.add("Opera/9.80 (Windows NT 5.1; U; en) Presto/2.7.39 Version/11.00");
        userAgents.add("Mozilla/4.0 (compatible; MSIE 8.0; X11; Linux x86_64; en) Opera 11.00");
        userAgents.add("Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; en) Opera 11.00");
        userAgents.add("Opera/9.80 (X11; Linux i686; U; en) Presto/2.5.27 Version/10.60");
        userAgents.add("Mozilla/5.0 (Windows NT 5.1; rv:2.0b9pre) Gecko/20110105 Firefox/4.0b9pre");
        userAgents
                .add("Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20091218 Firefox 3.6b5");
        userAgents.add("Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.9.2.9) Gecko/20100913 Firefox/3.6.9");
    } catch (Exception e)
    {
        e.printStackTrace();
    }
}

Example 20

Source File: ThreadPoolHttpClient.java From albert with MIT License

4 votes

public void test() throws Exception {
    exe = Executors.newFixedThreadPool(POOL_SIZE);
    HttpParams params =new BasicHttpParams();
    /* 从连接池中取连接的超时时间 */ 
    ConnManagerParams.setTimeout(params, 1000);
    /* 连接超时 */ 
    HttpConnectionParams.setConnectionTimeout(params, 2000); 
    /* 请求超时 */
    HttpConnectionParams.setSoTimeout(params, 4000);
    SchemeRegistry schemeRegistry = new SchemeRegistry();
    schemeRegistry.register(
            new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));

    //ClientConnectionManager cm = new PoolingClientConnectionManager(schemeRegistry);
    PoolingClientConnectionManager cm=new PoolingClientConnectionManager(schemeRegistry);
    cm.setMaxTotal(10);
    final HttpClient httpClient = new DefaultHttpClient(cm,params);

    // URIs to perform GETs on
    final String[] urisToGet = urls;
    /* 有多少url创建多少线程，url多时机子撑不住
    // create a thread for each URI
    GetThread[] threads = new GetThread[urisToGet.length];
    for (int i = 0; i < threads.length; i++) {
        HttpGet httpget = new HttpGet(urisToGet[i]);
        threads[i] = new GetThread(httpClient, httpget);            
    }
    // start the threads
    for (int j = 0; j < threads.length; j++) {
        threads[j].start();
    }

    // join the threads，等待所有请求完成
    for (int j = 0; j < threads.length; j++) {
        threads[j].join();
    }
    使用线程池*/
    for (int i = 0; i < urisToGet.length; i++) {
        final int j=i;
        System.out.println(j);
        HttpGet httpget = new HttpGet(urisToGet[i]);
        exe.execute( new GetThread(httpClient, httpget));
    }
    
    
    //创建线程池，每次调用POOL_SIZE
    /*
    for (int i = 0; i < urisToGet.length; i++) {
        final int j=i;
        System.out.println(j);
        exe.execute(new Thread() {
            @Override
            public void run() {
                this.setName("threadsPoolClient"+j);
                
                    try {
                        this.sleep(100);
                        System.out.println(j);
                    } catch (InterruptedException e) {
                        // TODO Auto-generated catch block
                        e.printStackTrace();
                    }
                    
                    HttpGet httpget = new HttpGet(urisToGet[j]);
                    new GetThread(httpClient, httpget).get();
                }
                
                
            
        });
    }
    
    */
    //exe.shutdown();
    System.out.println("Done");
}