Java Code Examples for org.elasticsearch.client.RestClientBuilder#build()

The following examples show how to use org.elasticsearch.client.RestClientBuilder#build() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractBeyonderTest.java    From elasticsearch-beyonder with Apache License 2.0 6 votes vote down vote up
private static void startRestClient() throws IOException {
    if (client == null) {
        RestClientBuilder builder = RestClient.builder(HttpHost.create(testCluster));
        if (testClusterUser != null) {
            final CredentialsProvider credentialsProvider =
                    new BasicCredentialsProvider();
            credentialsProvider.setCredentials(AuthScope.ANY,
                    new UsernamePasswordCredentials(testClusterUser, testClusterPass));
            builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder
                    .setDefaultCredentialsProvider(credentialsProvider));
        }

        client = builder.build();
        testClusterRunning();
    }
}
 
Example 2
Source File: ElasticsearchRestClientInstrumentationIT.java    From apm-agent-java with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void startElasticsearchContainerAndClient() throws IOException {
    // Start the container
    container = new ElasticsearchContainer(ELASTICSEARCH_CONTAINER_VERSION);
    container.start();

    final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(USER_NAME, PASSWORD));

    RestClientBuilder builder =  RestClient.builder(HttpHost.create(container.getHttpHostAddress()))
        .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
    lowLevelClient = builder.build();
    client = new RestHighLevelClient(lowLevelClient);

    lowLevelClient.performRequest("PUT", "/" + INDEX);
    reporter.reset();
}
 
Example 3
Source File: ElasticSearchFilter.java    From timbuctoo with GNU General Public License v3.0 6 votes vote down vote up
@JsonCreator
public ElasticSearchFilter(@JsonProperty("hostname") String hostname, @JsonProperty("port") int port,
                           @JsonProperty("username") Optional<String> username,
                           @JsonProperty("password") Optional<String> password) {
  Header[] headers = {
    new BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"),
    new BasicHeader("Role", "Read")};
  final RestClientBuilder restClientBuilder = RestClient.builder(new HttpHost(hostname, port))
    .setDefaultHeaders(headers);
  if (username.isPresent() && !username.get().isEmpty() && password.isPresent() && !password.get().isEmpty()) {
    final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();

    credentialsProvider.setCredentials(
      AuthScope.ANY,
      new UsernamePasswordCredentials(username.get(), password.get())
    );

    restClientBuilder.setHttpClientConfigCallback(b -> b.setDefaultCredentialsProvider(credentialsProvider));
  }
  restClient = restClientBuilder.build();
  mapper = new ObjectMapper();
}
 
Example 4
Source File: ElasticsearchConnection.java    From components with Apache License 2.0 6 votes vote down vote up
public static RestClient createClient(ElasticsearchDatastoreProperties datastore) throws MalformedURLException {
    String urlStr = datastore.nodes.getValue();
    String[] urls = urlStr.split(",");
    HttpHost[] hosts = new HttpHost[urls.length];
    int i = 0;
    for (String address : urls) {
        URL url = new URL("http://" + address);
        hosts[i] = new HttpHost(url.getHost(), url.getPort(), url.getProtocol());
        i++;
    }
    RestClientBuilder restClientBuilder = RestClient.builder(hosts);
    if (datastore.auth.useAuth.getValue()) {
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(datastore.auth.userId.getValue(), datastore.auth.password.getValue()));
        restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {

            public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
                return httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
            }
        });
    }
    return restClientBuilder.build();
}
 
Example 5
Source File: ElasticsearchAuditLogSink.java    From Groza with Apache License 2.0 6 votes vote down vote up
@PostConstruct
public void init() {
    try {
        log.trace("Adding elastic rest endpoint... host [{}], port [{}], scheme name [{}]",
                host, port, schemeName);
        RestClientBuilder builder = RestClient.builder(
                new HttpHost(host, port, schemeName));

        if (StringUtils.isNotEmpty(userName) &&
                StringUtils.isNotEmpty(password)) {
            log.trace("...using username [{}] and password ***", userName);
            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(AuthScope.ANY,
                    new UsernamePasswordCredentials(userName, password));
            builder.setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
        }

        this.restClient = builder.build();
    } catch (Exception e) {
        log.error("Sink init failed!", e);
        throw new RuntimeException(e.getMessage(), e);
    }
}
 
Example 6
Source File: RestClientConfig.java    From ProjectStudy with MIT License 5 votes vote down vote up
/**
 * LowLevelRestConfig
 *
 * @param
 * @return org.elasticsearch.client.RestClient
 * @author wliduo[[email protected]]
 * @date 2019/8/12 18:56
 */
@Bean
public RestClient restClient() {
    // 如果有多个从节点可以持续在内部new多个HttpHost,参数1是IP,参数2是端口,参数3是通信协议
    RestClientBuilder clientBuilder = RestClient.builder(new HttpHost(hostname, port, "http"));
    // 设置Header编码
    Header[] defaultHeaders = {new BasicHeader("content-type", "application/json")};
    clientBuilder.setDefaultHeaders(defaultHeaders);
    // 添加其他配置,这些配置都是可选的,详情配置可看https://blog.csdn.net/jacksonary/article/details/82729556
    return clientBuilder.build();
}
 
Example 7
Source File: TestElasticSearchRestDAOV6.java    From conductor with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void startServer() throws Exception {
    System.setProperty(ElasticSearchConfiguration.EMBEDDED_PORT_PROPERTY_NAME, "9204");
    System.setProperty(ElasticSearchConfiguration.ELASTIC_SEARCH_URL_PROPERTY_NAME, "http://localhost:9204");
    System.setProperty(ElasticSearchConfiguration.ELASTIC_SEARCH_INDEX_BATCH_SIZE_PROPERTY_NAME, "1");

    configuration = new SystemPropertiesElasticSearchConfiguration();

    String host = configuration.getEmbeddedHost();
    int port = configuration.getEmbeddedPort();
    String clusterName = configuration.getEmbeddedClusterName();

    embeddedElasticSearch = new EmbeddedElasticSearchV6(clusterName, host, port);
    embeddedElasticSearch.start();

    ElasticSearchRestClientBuilderProvider restClientProvider =
            new ElasticSearchRestClientBuilderProvider(configuration);

    RestClientBuilder restClientBuilder = restClientProvider.get();
    restClient = restClientBuilder.build();

    Map<String, String> params = new HashMap<>();
    params.put("wait_for_status", "yellow");
    params.put("timeout", "30s");

    restClient.performRequest("GET", "/_cluster/health", params);

    objectMapper = new JsonMapperProvider().get();
    indexDAO = new ElasticSearchRestDAOV6(restClientBuilder, configuration, objectMapper);
}
 
Example 8
Source File: MetricStoreImpl.java    From griffin with Apache License 2.0 5 votes vote down vote up
public MetricStoreImpl(@Value("${elasticsearch.host}") String host,
                       @Value("${elasticsearch.port}") int port,
                       @Value("${elasticsearch.scheme:http}") String scheme,
                       @Value("${elasticsearch.user:}") String user,
                       @Value("${elasticsearch.password:}") String password) {
    HttpHost httpHost = new HttpHost(host, port, scheme);
    RestClientBuilder builder = RestClient.builder(httpHost);
    if (!user.isEmpty() && !password.isEmpty()) {
        String encodedAuth = buildBasicAuthString(user, password);
        Header[] requestHeaders = new Header[]{
            new BasicHeader(org.apache.http.HttpHeaders.AUTHORIZATION,
                encodedAuth)};
        builder.setDefaultHeaders(requestHeaders);
    }
    this.client = builder.build();
    HttpHeaders responseHeaders = new HttpHeaders();
    responseHeaders.setContentType(MediaType.APPLICATION_JSON);
    this.responseHeaders = responseHeaders;
    String urlBase = String.format("/%s/%s", INDEX, TYPE);
    this.urlGet = urlBase.concat("/_search?filter_path=hits.hits._source");
    this.urlPost = urlBase.concat("/_bulk");
    this.urlDelete = urlBase.concat("/_delete_by_query");
    this.indexMetaData = String.format(
        "{ \"index\" : { \"_index\" : " +
            "\"%s\",\"_type\" : \"%s\" } }%n",
        INDEX,
        TYPE);
    this.mapper = new ObjectMapper();
}
 
Example 9
Source File: VulnerabilityTrendGenerator.java    From pacbot with Apache License 2.0 5 votes vote down vote up
private RestClient getRestClient() {
        RestClientBuilder builder = RestClient.builder(new HttpHost(esHost, esPort));
        builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
            @Override
            public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
                return requestConfigBuilder.setConnectionRequestTimeout(0).setSocketTimeout(60000);
            }
        }).setMaxRetryTimeoutMillis(60000);
        return builder.build();
}
 
Example 10
Source File: AssetGroupExceptionServiceImpl.java    From pacbot with Apache License 2.0 5 votes vote down vote up
private RestClient getRestClient() {
    if (restClient == null) {
    	String esHost = config.getElasticSearch().getDevIngestHost();
		int esPort = config.getElasticSearch().getDevIngestPort();
        RestClientBuilder builder = RestClient.builder(new HttpHost(esHost, esPort));
        builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
            @Override
            public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
                return requestConfigBuilder.setConnectionRequestTimeout(0);
            }
        });
        restClient = builder.build();
    }
    return restClient;
}
 
Example 11
Source File: TargetTypesServiceImpl.java    From pacbot with Apache License 2.0 5 votes vote down vote up
private RestClient getRestClient() {
    if (restClient == null) {
    	String esHost = config.getElasticSearch().getDevIngestHost();
		int esPort = config.getElasticSearch().getDevIngestPort();
        RestClientBuilder builder = RestClient.builder(new HttpHost(esHost, esPort));
        builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
            @Override
            public RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
                return requestConfigBuilder.setConnectionRequestTimeout(0);
            }
        });
        restClient = builder.build();
    }
    return restClient;
}
 
Example 12
Source File: RestClientConfig.java    From ProjectStudy with MIT License 5 votes vote down vote up
/**
 * LowLevelRestConfig
 *
 * @param
 * @return org.elasticsearch.client.RestClient
 * @author wliduo[[email protected]]
 * @date 2019/8/12 18:56
 */
@Bean
public RestClient restClient() {
    // 如果有多个从节点可以持续在内部new多个HttpHost,参数1是IP,参数2是端口,参数3是通信协议
    RestClientBuilder clientBuilder = RestClient.builder(new HttpHost(hostname, port, "http"));
    // 设置Header编码
    Header[] defaultHeaders = {new BasicHeader("content-type", "application/json")};
    clientBuilder.setDefaultHeaders(defaultHeaders);
    // 添加其他配置,这些配置都是可选的,详情配置可看https://blog.csdn.net/jacksonary/article/details/82729556
    return clientBuilder.build();
}
 
Example 13
Source File: ElasticSearchClientServiceImpl.java    From nifi with Apache License 2.0 4 votes vote down vote up
private void setupClient(ConfigurationContext context) throws MalformedURLException, InitializationException {
    final String hosts = context.getProperty(HTTP_HOSTS).evaluateAttributeExpressions().getValue();
    String[] hostsSplit = hosts.split(",[\\s]*");
    this.url = hostsSplit[0];
    final SSLContextService sslService =
            context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextService.class);
    final String username = context.getProperty(USERNAME).evaluateAttributeExpressions().getValue();
    final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions().getValue();

    final Integer connectTimeout = context.getProperty(CONNECT_TIMEOUT).asInteger();
    final Integer readTimeout    = context.getProperty(SOCKET_TIMEOUT).asInteger();
    final Integer retryTimeout   = context.getProperty(RETRY_TIMEOUT).asInteger();

    HttpHost[] hh = new HttpHost[hostsSplit.length];
    for (int x = 0; x < hh.length; x++) {
        URL u = new URL(hostsSplit[x]);
        hh[x] = new HttpHost(u.getHost(), u.getPort(), u.getProtocol());
    }

    final SSLContext sslContext;
    try {
        sslContext = (sslService != null && (sslService.isKeyStoreConfigured() || sslService.isTrustStoreConfigured()))
            ? sslService.createSSLContext(SslContextFactory.ClientAuth.NONE) : null;
    } catch (Exception e) {
        getLogger().error("Error building up SSL Context from the supplied configuration.", e);
        throw new InitializationException(e);
    }

    RestClientBuilder builder = RestClient.builder(hh)
        .setHttpClientConfigCallback(httpClientBuilder -> {
            if (sslContext != null) {
                httpClientBuilder = httpClientBuilder.setSSLContext(sslContext);
            }

            if (username != null && password != null) {
                final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
                credentialsProvider.setCredentials(AuthScope.ANY,
                        new UsernamePasswordCredentials(username, password));
                httpClientBuilder = httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
            }

            return httpClientBuilder;
        })
        .setRequestConfigCallback(requestConfigBuilder -> {
            requestConfigBuilder.setConnectTimeout(connectTimeout);
            requestConfigBuilder.setSocketTimeout(readTimeout);
            return requestConfigBuilder;
        })
        .setMaxRetryTimeoutMillis(retryTimeout);

    this.client = builder.build();
}
 
Example 14
Source File: ElasticDataStoreFactory.java    From elasticgeo with GNU General Public License v3.0 4 votes vote down vote up
private RestClient createRestClient(Map<String, Serializable> params, String user, String password) throws IOException {
    final String hostName = getValue(HOSTNAME, params);
    final String[] hosts = hostName.split(",");
    final Integer defaultPort = getValue(HOSTPORT, params);
    final Boolean sslRejectUnauthorized = getValue(SSL_REJECT_UNAUTHORIZED, params);
    final String adminUser = getValue(USER, params);
    final String type = user == null || adminUser == null || user.equals(adminUser) ? "ADMIN" : "PROXY_USER";

    final Pattern pattern = Pattern.compile("(?<scheme>https?)?(://)?(?<host>[^:]+):?(?<port>\\d+)?");
    final HttpHost[] httpHosts = new HttpHost[hosts.length];
    final AuthScope[] auths = new AuthScope[hosts.length];
    for (int index=0; index < hosts.length; index++) {
        final Matcher matcher = pattern.matcher(hosts[index].trim());
        if (matcher.find()) {
            final String scheme = matcher.group("scheme") != null ? matcher.group("scheme") : "http";
            final String host = matcher.group("host");
            final Integer port = matcher.group("port") != null ? Integer.valueOf(matcher.group("port")) : defaultPort;
            httpHosts[index] = new HttpHost(host, port, scheme);
            auths[index] = new AuthScope(host, port);
        } else {
            throw new IOException("Unable to parse host");
        }
    }

    final RestClientBuilder builder = createClientBuilder(httpHosts);

    if (user != null) {
        builder.setRequestConfigCallback((b) -> {
            LOGGER.finest(String.format("Calling %s setRequestConfigCallback", type));
            return b.setAuthenticationEnabled(true);
        });
    }

    builder.setHttpClientConfigCallback((httpClientBuilder) -> {
        LOGGER.finest(String.format("Calling %s customizeHttpClient", type));

        httpClientBuilder.setThreadFactory((run) -> {
            final Thread thread = new Thread(run);
            thread.setDaemon(true);
            thread.setName(String.format("esrest-asynchttp-%s-%d", type, httpThreads.getAndIncrement()));
            return thread;
        });

        httpClientBuilder.useSystemProperties();

        if (!sslRejectUnauthorized) {
            httpClientBuilder.setSSLHostnameVerifier((host,session) -> true);
            try {
                httpClientBuilder.setSSLContext(SSLContextBuilder.create().loadTrustMaterial((chain,authType) ->true).build());
            } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) {
                throw new UncheckedIOException(new IOException("Unable to create SSLContext", e));
            }
        }

        if (user != null) {
            final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            final Credentials credentials = new org.apache.http.auth.UsernamePasswordCredentials(user, password);
            for (AuthScope scope : auths) {
                credentialsProvider.setCredentials(scope, credentials);
            }

            httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
        }

        return httpClientBuilder;
    });

    LOGGER.fine(String.format("Building a %s RestClient for %s @ %s:%d", type, user, hostName, defaultPort));
    return builder.build();
}
 
Example 15
Source File: ESMetricExporter.java    From sofa-lookout with Apache License 2.0 4 votes vote down vote up
public ESMetricExporter(String name, Registry registry, ESProperties esProperties) {
    super(name, registry, 100, DataType.METRIC);

    int timeout = esProperties.getTimeout();

    HttpHost httpHost = new HttpHost(esProperties.getHost(), esProperties.getPort(), "http");
    RestClientBuilder restClientBuilder = RestClient.builder(httpHost)
            .setMaxRetryTimeoutMillis(timeout)
            .setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder.setSocketTimeout(timeout));
    String username = esProperties.getUsername();
    String password = esProperties.getPassword();

    // basic auth
    if (StringUtils.isNotEmpty(username)) {
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY,
                new UsernamePasswordCredentials(username, password));
        restClientBuilder.setHttpClientConfigCallback(
                httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
    }

    client = restClientBuilder.build();

    String index = esProperties.getIndex();
    if (esProperties.getOperation().isAuto()) {
        logger.info("ElasticSearch operator is active");
        ESOperatorBuilder esOperatorBuilder = new ESOperatorBuilder(ESDataType.METRIC)
                .httpHost(httpHost.toURI())
                .index(esProperties.getIndex())
                .mapping(esProperties.getType());
        esOperatorBuilder.build().initializeDatabase();
        //replace index with alias
        index = esOperatorBuilder.getAlias();
    }


    String type = esProperties.getType();
    if ("metrics".equalsIgnoreCase(index)) {
        actionMetaData = String.format("{ \"index\" : {  \"_type\" : \"%s\" } }%n", type);
    } else {
        actionMetaData = String.format("{ \"index\" : { \"_index\" : \"%s\", \"_type\" : \"%s\" } }%n", index, type);
    }
    endpoint = String.format("/%s/%s/_bulk", index, type);


}
 
Example 16
Source File: ElasticsearchRestWriter.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
private static RestClient buildRestClient(List<InetSocketTransportAddress> hosts, int threadCount, boolean sslEnabled,
    String keyStoreType, String keyStoreFilePassword, String identityFilepath, String trustStoreType,
    String trustStoreFilePassword, String cacertsFilepath) throws Exception {


  HttpHost[] httpHosts = new HttpHost[hosts.size()];
  String scheme = sslEnabled?"https":"http";
  for (int h = 0; h < httpHosts.length; h++) {
    InetSocketTransportAddress host = hosts.get(h);
    httpHosts[h] = new HttpHost(host.getAddress(), host.getPort(), scheme);
  }

  RestClientBuilder builder = RestClient.builder(httpHosts);

  if (sslEnabled) {
    log.info("ssl configuration: trustStoreType = {}, cacertsFilePath = {}", trustStoreType, cacertsFilepath);
    KeyStore truststore = KeyStore.getInstance(trustStoreType);
    FileInputStream trustInputStream = new FileInputStream(cacertsFilepath);
    try {
      truststore.load(trustInputStream, trustStoreFilePassword.toCharArray());
    }
    finally {
      trustInputStream.close();
    }
    SSLContextBuilder sslBuilder = SSLContexts.custom().loadTrustMaterial(truststore, null);

    log.info("ssl key configuration: keyStoreType = {}, keyFilePath = {}", keyStoreType, identityFilepath);

    KeyStore keystore = KeyStore.getInstance(keyStoreType);
    FileInputStream keyInputStream = new FileInputStream(identityFilepath);
    try {
      keystore.load(keyInputStream, keyStoreFilePassword.toCharArray());
    }
    finally {
      keyInputStream.close();
    }
    sslBuilder.loadKeyMaterial(keystore, keyStoreFilePassword.toCharArray());

    final SSLContext sslContext = sslBuilder.build();
    builder = builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder
        // Set ssl context
        .setSSLContext(sslContext).setSSLHostnameVerifier(new NoopHostnameVerifier())
        // Configure number of threads for clients
        .setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(threadCount).build()));
  } else {
    builder = builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder
        // Configure number of threads for clients
        .setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(threadCount).build()));
  }

  // Configure timeouts
  builder.setRequestConfigCallback(requestConfigBuilder -> requestConfigBuilder
      .setConnectionRequestTimeout(0)); // Important, otherwise the client has spurious timeouts

  return builder.build();
}
 
Example 17
Source File: IndexerSingleton.java    From scava with Eclipse Public License 2.0 4 votes vote down vote up
private boolean createHighLevelClient(RestClientBuilder restClientBuilder)
{
	restClientBuilder.build();
	client = new RestHighLevelClient(restClientBuilder);
	return testClient();
}
 
Example 18
Source File: ElasticSearchRestDAOV6.java    From conductor with Apache License 2.0 4 votes vote down vote up
@Inject
public ElasticSearchRestDAOV6(RestClientBuilder restClientBuilder, ElasticSearchConfiguration config,
    ObjectMapper objectMapper) {

    this.objectMapper = objectMapper;
    this.elasticSearchAdminClient = restClientBuilder.build();
    this.elasticSearchClient = new RestHighLevelClient(restClientBuilder);
    this.clusterHealthColor = config.getClusterHealthColor();
    this.bulkRequests = new ConcurrentHashMap<>();
    this.indexBatchSize = config.getIndexBatchSize();
    this.asyncBufferFlushTimeout = config.getAsyncBufferFlushTimeout();
    this.config = config;

    this.indexPrefix = config.getIndexName();
    if (!config.isElasticSearchAutoIndexManagementEnabled() &&
        StringUtils.isNotBlank(config.getElasticSearchDocumentTypeOverride())) {
        docTypeOverride = config.getElasticSearchDocumentTypeOverride();
    } else {
        docTypeOverride = "";
    }

    this.workflowIndexName = getIndexName(WORKFLOW_DOC_TYPE);
    this.taskIndexName = getIndexName(TASK_DOC_TYPE);
    this.logIndexPrefix = this.indexPrefix + "_" + LOG_DOC_TYPE;
    this.messageIndexPrefix = this.indexPrefix + "_" + MSG_DOC_TYPE;
    this.eventIndexPrefix = this.indexPrefix + "_" + EVENT_DOC_TYPE;
    int workerQueueSize = config.getAsyncWorkerQueueSize();
    int maximumPoolSize = config.getAsyncMaxPoolSize();

    // Set up a workerpool for performing async operations.
    this.executorService = new ThreadPoolExecutor(CORE_POOL_SIZE,
        maximumPoolSize,
        KEEP_ALIVE_TIME,
        TimeUnit.MINUTES,
        new LinkedBlockingQueue<>(workerQueueSize),
        (runnable, executor) -> {
            logger.warn("Request  {} to async dao discarded in executor {}", runnable, executor);
            Monitors.recordDiscardedIndexingCount("indexQueue");
        });

    // Set up a workerpool for performing async operations for task_logs, event_executions, message
    int corePoolSize = 1;
    maximumPoolSize = 2;
    long keepAliveTime = 30L;
    this.logExecutorService = new ThreadPoolExecutor(corePoolSize,
        maximumPoolSize,
        keepAliveTime,
        TimeUnit.SECONDS,
        new LinkedBlockingQueue<>(workerQueueSize),
        (runnable, executor) -> {
            logger.warn("Request {} to async log dao discarded in executor {}", runnable, executor);
            Monitors.recordDiscardedIndexingCount("logQueue");
        });

    Executors.newSingleThreadScheduledExecutor()
        .scheduleAtFixedRate(this::flushBulkRequests, 60, 30, TimeUnit.SECONDS);
}
 
Example 19
Source File: ElasticEndpoint.java    From sagacity-sqltoy with Apache License 2.0 4 votes vote down vote up
/**
 * @param restClient the restClient to set
 */
public void initRestClient() {
	if (StringUtil.isBlank(this.getUrl()))
		return;
	if (restClient == null) {
		// 替换全角字符
		String[] urls = this.getUrl().replaceAll("\\;", ";").replaceAll("\\,", ",").replaceAll("\\;", ",")
				.split("\\,");
		// 当为单一地址时使用httpclient直接调用
		if (urls.length < 2)
			return;
		List<HttpHost> hosts = new ArrayList<HttpHost>();
		for (String urlStr : urls) {
			try {
				if (StringUtil.isNotBlank(urlStr)) {
					URL url = new java.net.URL(urlStr.trim());
					hosts.add(new HttpHost(url.getHost(), url.getPort(), url.getProtocol()));
				}
			} catch (MalformedURLException e) {
				e.printStackTrace();
			}
		}
		if (!hosts.isEmpty()) {
			HttpHost[] hostAry = new HttpHost[hosts.size()];
			hosts.toArray(hostAry);
			RestClientBuilder builder = RestClient.builder(hostAry);
			final ConnectionConfig connectionConfig = ConnectionConfig.custom()
					.setCharset(Charset.forName(this.charset == null ? "UTF-8" : this.charset)).build();
			RequestConfig requestConfig = RequestConfig.custom().setConnectionRequestTimeout(this.requestTimeout)
					.setConnectTimeout(this.connectTimeout).setSocketTimeout(this.socketTimeout).build();
			final CredentialsProvider credsProvider = new BasicCredentialsProvider();
			final boolean hasCrede = (StringUtil.isNotBlank(this.getUsername())
					&& StringUtil.isNotBlank(getPassword())) ? true : false;
			// 凭据提供器
			if (hasCrede) {
				credsProvider.setCredentials(AuthScope.ANY,
						// 认证用户名和密码
						new UsernamePasswordCredentials(getUsername(), getPassword()));
			}
			builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
				@Override
				public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
					httpClientBuilder.setDefaultConnectionConfig(connectionConfig)
							.setDefaultRequestConfig(requestConfig);
					if (hasCrede) {
						httpClientBuilder.setDefaultCredentialsProvider(credsProvider);
					}
					return httpClientBuilder;
				}
			});
			restClient = builder.build();
		}
	}
}
 
Example 20
Source File: ElasticsearchIO.java    From beam with Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
RestClient createClient() throws IOException {
  HttpHost[] hosts = new HttpHost[getAddresses().size()];
  int i = 0;
  for (String address : getAddresses()) {
    URL url = new URL(address);
    hosts[i] = new HttpHost(url.getHost(), url.getPort(), url.getProtocol());
    i++;
  }
  RestClientBuilder restClientBuilder = RestClient.builder(hosts);
  if (getUsername() != null) {
    final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    credentialsProvider.setCredentials(
        AuthScope.ANY, new UsernamePasswordCredentials(getUsername(), getPassword()));
    restClientBuilder.setHttpClientConfigCallback(
        httpAsyncClientBuilder ->
            httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider));
  }
  if (getKeystorePath() != null && !getKeystorePath().isEmpty()) {
    try {
      KeyStore keyStore = KeyStore.getInstance("jks");
      try (InputStream is = new FileInputStream(new File(getKeystorePath()))) {
        String keystorePassword = getKeystorePassword();
        keyStore.load(is, (keystorePassword == null) ? null : keystorePassword.toCharArray());
      }
      final TrustStrategy trustStrategy =
          isTrustSelfSignedCerts() ? new TrustSelfSignedStrategy() : null;
      final SSLContext sslContext =
          SSLContexts.custom().loadTrustMaterial(keyStore, trustStrategy).build();
      final SSLIOSessionStrategy sessionStrategy = new SSLIOSessionStrategy(sslContext);
      restClientBuilder.setHttpClientConfigCallback(
          httpClientBuilder ->
              httpClientBuilder.setSSLContext(sslContext).setSSLStrategy(sessionStrategy));
    } catch (Exception e) {
      throw new IOException("Can't load the client certificate from the keystore", e);
    }
  }
  restClientBuilder.setRequestConfigCallback(
      new RestClientBuilder.RequestConfigCallback() {
        @Override
        public RequestConfig.Builder customizeRequestConfig(
            RequestConfig.Builder requestConfigBuilder) {
          if (getConnectTimeout() != null) {
            requestConfigBuilder.setConnectTimeout(getConnectTimeout());
          }
          if (getSocketTimeout() != null) {
            requestConfigBuilder.setSocketTimeout(getSocketTimeout());
          }
          return requestConfigBuilder;
        }
      });
  return restClientBuilder.build();
}