Java Code Examples for org.elasticsearch.common.unit.TimeValue

The following examples show how to use org.elasticsearch.common.unit.TimeValue. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: SpringBootLearn   Source File: ElasticsearchUtil.java    License: Apache License 2.0 6 votes vote down vote up
/**
     * 处理scroll结果
     * @Author lihaodong
     * @Description
     * @Date 20:18 2018/12/21
     * @Param [response, highlightField]
     * @return java.util.List<java.util.Map<java.lang.String,java.lang.Object>>
     **/
    private static   List<Map<String, Object>>   disposeScrollResult(SearchResponse response ,String highlightField){
        List<Map<String, Object>> sourceList = new ArrayList<Map<String, Object>>();
        //使用scrollId迭代查询
        while (response.getHits().getHits().length > 0) {
            String scrollId = response.getScrollId();
            response = client.prepareSearchScroll(scrollId)
                    .setScroll(TimeValue.timeValueMinutes(1))//设置查询context的存活时间
                    .execute()
                    .actionGet();
            SearchHits hits = response.getHits();
            for (SearchHit hit : hits.getHits()) {
                Map<String, Object> resultMap =getResultMap(hit, highlightField);
                sourceList.add(resultMap);
//                System.out.println(JSON.toJSONString(resultMap));
            }
        }
        ClearScrollRequest request = new ClearScrollRequest();
        request.addScrollId(response.getScrollId());
        client.clearScroll(request);
        return sourceList;
    }
 
Example 2
Source Project: Elasticsearch   Source File: SearchSlowLog.java    License: Apache License 2.0 6 votes vote down vote up
SearchSlowLog(Settings indexSettings) {

        this.reformat = indexSettings.getAsBoolean(INDEX_SEARCH_SLOWLOG_REFORMAT, true);

        this.queryWarnThreshold = indexSettings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_WARN, TimeValue.timeValueNanos(-1)).nanos();
        this.queryInfoThreshold = indexSettings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_INFO, TimeValue.timeValueNanos(-1)).nanos();
        this.queryDebugThreshold = indexSettings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_DEBUG, TimeValue.timeValueNanos(-1)).nanos();
        this.queryTraceThreshold = indexSettings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_QUERY_TRACE, TimeValue.timeValueNanos(-1)).nanos();

        this.fetchWarnThreshold = indexSettings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_WARN, TimeValue.timeValueNanos(-1)).nanos();
        this.fetchInfoThreshold = indexSettings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_INFO, TimeValue.timeValueNanos(-1)).nanos();
        this.fetchDebugThreshold = indexSettings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG, TimeValue.timeValueNanos(-1)).nanos();
        this.fetchTraceThreshold = indexSettings.getAsTime(INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_TRACE, TimeValue.timeValueNanos(-1)).nanos();

        this.level = indexSettings.get(INDEX_SEARCH_SLOWLOG_LEVEL, "TRACE").toUpperCase(Locale.ROOT);

        this.queryLogger = Loggers.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".query");
        this.fetchLogger = Loggers.getLogger(INDEX_SEARCH_SLOWLOG_PREFIX + ".fetch");

        queryLogger.setLevel(level);
        fetchLogger.setLevel(level);
    }
 
Example 3
Source Project: elasticsearch-helper   Source File: NodeTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void startNodes() {
    try {
        logger.info("starting");
        setClusterName();
        startNode("1");
        findNodeAddress();
        try {
            ClusterHealthResponse healthResponse = client("1").execute(ClusterHealthAction.INSTANCE,
                            new ClusterHealthRequest().waitForStatus(ClusterHealthStatus.GREEN).timeout(TimeValue.timeValueSeconds(30))).actionGet();
            if (healthResponse != null && healthResponse.isTimedOut()) {
                throw new IOException("cluster state is " + healthResponse.getStatus().name()
                        + ", from here on, everything will fail!");
            }
        } catch (ElasticsearchTimeoutException e) {
            throw new IOException("timeout, cluster does not respond to health request, cowardly refusing to continue with operations");
        }
    } catch (Throwable t) {
        logger.error("startNodes failed", t);
    }
}
 
Example 4
Source Project: anomaly-detection   Source File: ADStateManagerTests.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@Before
public void setUp() throws Exception {
    super.setUp();
    modelManager = mock(ModelManager.class);
    when(modelManager.getPartitionedForestSizes(any(AnomalyDetector.class))).thenReturn(new SimpleImmutableEntry<>(2, 20));
    client = mock(Client.class);
    clientUtil = mock(ClientUtil.class);
    settings = Settings
        .builder()
        .put("opendistro.anomaly_detection.max_retry_for_unresponsive_node", 3)
        .put("opendistro.anomaly_detection.ad_mute_minutes", TimeValue.timeValueMinutes(10))
        .build();
    clock = mock(Clock.class);
    duration = Duration.ofHours(1);
    context = TestHelpers.createThreadPool();
    throttler = new Throttler(clock);

    stateManager = new ADStateManager(client, xContentRegistry(), modelManager, settings, clientUtil, clock, duration);

}
 
Example 5
Source Project: mongolastic   Source File: ElasticToMongoProvider.java    License: MIT License 6 votes vote down vote up
@Override
public List buildJSONContent(int skip, int limit) {

    if (Objects.isNull(response)) {
        response = elastic.getClient().prepareSearch(config.getMisc().getDindex().getName())
                .setTypes(config.getMisc().getCtype().getName())
                .setSearchType(SearchType.QUERY_THEN_FETCH)
                .setScroll(new TimeValue(60000))
                .setSize(limit)
                .execute().actionGet();
    }
    else {
        response = elastic.getClient()
                .prepareSearchScroll(response.getScrollId())
                .setScroll(new TimeValue(60000))
                .execute().actionGet();
    }

    return Arrays.stream(response.getHits().getHits())
            .map(hit -> new Document(hit.getSourceAsMap()))
            .collect(Collectors.toList());
}
 
Example 6
Source Project: jframe   Source File: TestTransportClient.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testScrollSearch() {
    QueryBuilder qb = QueryBuilders.termQuery("multi", "test");

    // 100 hits per shard will be returned for each scroll
    SearchResponse scrollResp = client.prepareSearch("index1").addSort(SortParseElement.DOC_FIELD_NAME, SortOrder.ASC)
            .setScroll(new TimeValue(60000)).setQuery(qb).setSize(100).execute().actionGet();
    // Scroll until no hits are returned
    while (true) {

        for (SearchHit hit : scrollResp.getHits().getHits()) {
            // Handle the hit...
        }
        scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet();
        // Break condition: No hits are returned
        if (scrollResp.getHits().getHits().length == 0) {
            break;
        }
    }
}
 
Example 7
Source Project: BioSolr   Source File: OntologyMapper.java    License: Apache License 2.0 6 votes vote down vote up
public static OntologyHelper getHelper(OntologySettings settings,
		ThreadPool threadPool) throws OntologyHelperException {
	String helperKey = buildHelperKey(settings);
	OntologyHelper helper = helpers.get(helperKey);

	if (helper == null) {
		helper = new ElasticOntologyHelperFactory(settings).buildOntologyHelper();
		OntologyCheckRunnable checker = new OntologyCheckRunnable(helperKey);
		ScheduledFuture checkFuture = threadPool.scheduleWithFixedDelay(checker, TimeValue.timeValueMillis(DELETE_CHECK_DELAY_MS));
		helpers.put(helperKey, helper);
		checkers.put(helperKey, checkFuture);
		helper.updateLastCallTime();
	}

	return helper;
}
 
Example 8
private void createBulkProcessorsAndExecutors(final int bulkCapacity,
                                              final int concurrentRequests,
                                              final int flushInterval,
                                              final int batchingThreads)
{
  Map<Integer, Entry<BulkProcessor, ExecutorService>> bulkProcessorAndThreadPools = new HashMap<>();
  for (int count = 0; count < batchingThreads; ++count) {
    final BulkIndexUpdateListener updateListener = new BulkIndexUpdateListener();
    updateListeners.add(updateListener);
    bulkProcessorAndThreadPools.put(count, new SimpleImmutableEntry<>(BulkProcessor
        .builder(this.client.get(), updateListener)
        .setBulkActions(bulkCapacity)
        .setBulkSize(new ByteSizeValue(-1)) // turn off automatic flush based on size in bytes
        .setConcurrentRequests(concurrentRequests)
        .setFlushInterval(periodicFlush ? TimeValue.timeValueMillis(flushInterval) : null)
        .build(), createThreadPool(count)));
  }
  this.bulkProcessorToExecutors = unmodifiableMap(bulkProcessorAndThreadPools);
}
 
Example 9
Source Project: Elasticsearch   Source File: DefaultSearchContext.java    License: Apache License 2.0 6 votes vote down vote up
public DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget,
                            Engine.Searcher engineSearcher, IndexService indexService, IndexShard indexShard,
                            ScriptService scriptService, PageCacheRecycler pageCacheRecycler,
                            BigArrays bigArrays, Counter timeEstimateCounter, ParseFieldMatcher parseFieldMatcher,
                            TimeValue timeout
) {
    super(parseFieldMatcher, request);
    this.id = id;
    this.request = request;
    this.searchType = request.searchType();
    this.shardTarget = shardTarget;
    this.engineSearcher = engineSearcher;
    this.scriptService = scriptService;
    this.pageCacheRecycler = pageCacheRecycler;
    // SearchContexts use a BigArrays that can circuit break
    this.bigArrays = bigArrays.withCircuitBreaking();
    this.dfsResult = new DfsSearchResult(id, shardTarget);
    this.queryResult = new QuerySearchResult(id, shardTarget);
    this.fetchResult = new FetchSearchResult(id, shardTarget);
    this.indexShard = indexShard;
    this.indexService = indexService;
    this.searcher = new ContextIndexSearcher(engineSearcher, indexService.cache().query(), indexShard.getQueryCachingPolicy());
    this.timeEstimateCounter = timeEstimateCounter;
    this.timeoutInMillis = timeout.millis();
}
 
Example 10
Source Project: soundwave   Source File: EsStore.java    License: Apache License 2.0 6 votes vote down vote up
protected <E extends EsDocument> ScrollableResponse<List<E>> retrieveScrollByField(
    String field, Object value, String[] includeFields, int size,
    ThrowingFunction<String, E> createFunc)
    throws Exception {

  Preconditions.checkArgument(size > 0);

  SearchRequestBuilder builder = esClient.prepareSearch()
      .setIndices(getIndexName()).setTypes(getDocTypeName())
      .setScroll(new TimeValue(SCROLLDEFAULTTIMEOUT))
      .setSize(size)
      .setQuery(QueryBuilders
          .filteredQuery(QueryBuilders.matchAllQuery(), FilterBuilders.termsFilter(field, value)))
      .setFetchSource(includeFields, null).setVersion(true);

  SearchResponse response = builder.execute().actionGet();
  return convertToScrollableResponse(response, createFunc);
}
 
Example 11
Source Project: foxtrot   Source File: ElasticsearchQueryStore.java    License: Apache License 2.0 6 votes vote down vote up
private void deleteIndices(List<String> indicesToDelete) {
    logger.warn("Deleting Indexes - Indexes - {}", indicesToDelete);
    if(!indicesToDelete.isEmpty()) {
        List<List<String>> subLists = Lists.partition(indicesToDelete, 5);
        for(List<String> subList : subLists) {
            try {
                connection.getClient()
                        .admin()
                        .indices()
                        .prepareDelete(subList.toArray(new String[0]))
                        .execute()
                        .actionGet(TimeValue.timeValueMinutes(5));
                logger.warn("Deleted Indexes - Indexes - {}", subList);
            } catch (Exception e) {
                logger.error("Index deletion failed - Indexes - {}", subList, e);
            }
        }
    }
}
 
Example 12
/**
 * Scroll 获取多个对象
 *
 * @param transportClient
 * @throws IOException
 */
private static void queryByScroll(TransportClient transportClient) throws IOException {

	//setSize 是设置每批查询多少条数据
	SearchResponse searchResponse = transportClient.prepareSearch("product_index").setTypes("product")
			.setQuery(QueryBuilders.termQuery("product_name", "飞利浦"))
			.setScroll(new TimeValue(60000))
			.setSize(3)
			.get();

	int count = 0;

	do {
		for (SearchHit searchHit : searchResponse.getHits().getHits()) {
			//打印查询结果,或者做其他处理
			logger.info("count=" + ++count);
			logger.info(searchHit.getSourceAsString());
		}

		searchResponse = transportClient.prepareSearchScroll(searchResponse.getScrollId()).setScroll(new TimeValue(60000))
				.execute()
				.actionGet();
	} while (searchResponse.getHits().getHits().length != 0);
}
 
Example 13
@BadApple(bugUrl = "https://github.com/o19s/elasticsearch-learning-to-rank/issues/75")
public void testExpirationOnWrite() throws IOException, InterruptedException {
    Caches caches = new Caches(TimeValue.timeValueMillis(100), TimeValue.timeValueHours(1), new ByteSizeValue(1000000));
    CachedFeatureStore store = new CachedFeatureStore(memStore, caches);
    CompiledLtrModel model = LtrTestUtils.buildRandomModel();
    memStore.add(model);
    store.loadModel(model.name());
    assertNotNull(store.getCachedModel(model.name()));
    assertFalse(caches.getCachedStoreNames().isEmpty());
    assertEquals(1, caches.getPerStoreStats(memStore.getStoreName()).totalCount());
    Thread.sleep(500);
    caches.modelCache().refresh();
    assertNull(store.getCachedModel(model.name()));
    assertTrue(caches.getCachedStoreNames().isEmpty());
    assertEquals(0, caches.getPerStoreStats(memStore.getStoreName()).modelRam());
    assertEquals(0, caches.getPerStoreStats(memStore.getStoreName()).totalCount());
}
 
Example 14
Source Project: BioSolr   Source File: OntologyMapper.java    License: Apache License 2.0 5 votes vote down vote up
public static OntologyHelper getHelper(OntologySettings settings, ThreadPool threadPool) throws OntologyHelperException {
	String helperKey = buildHelperKey(settings);
	OntologyHelper helper = helpers.get(helperKey);

	if (helper == null) {
		helper = new ElasticOntologyHelperFactory(settings).buildOntologyHelper();
		OntologyCheckRunnable checker = new OntologyCheckRunnable(helperKey);
		threadPool.scheduleWithFixedDelay(checker, TimeValue.timeValueMillis(DELETE_CHECK_DELAY_MS));
		helpers.put(helperKey, helper);
		helper.updateLastCallTime();
	}

	return helper;
}
 
Example 15
Source Project: Elasticsearch   Source File: IndicesStore.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    clusterName = ClusterName.readClusterName(in);
    indexUUID = in.readString();
    shardId = ShardId.readShardId(in);
    timeout = new TimeValue(in.readLong(), TimeUnit.MILLISECONDS);
}
 
Example 16
Source Project: Elasticsearch   Source File: IndicesFieldDataCache.java    License: Apache License 2.0 5 votes vote down vote up
@Inject
public IndicesFieldDataCache(Settings settings, IndicesFieldDataCacheListener indicesFieldDataCacheListener, ThreadPool threadPool) {
    super(settings);
    this.threadPool = threadPool;
    this.indicesFieldDataCacheListener = indicesFieldDataCacheListener;
    final String size = settings.get(INDICES_FIELDDATA_CACHE_SIZE_KEY, "-1");
    final long sizeInBytes = settings.getAsMemory(INDICES_FIELDDATA_CACHE_SIZE_KEY, "-1").bytes();
    CacheBuilder<Key, Accountable> cacheBuilder = CacheBuilder.newBuilder()
            .removalListener(this);
    if (sizeInBytes > 0) {
        cacheBuilder.maximumWeight(sizeInBytes).weigher(new FieldDataWeigher());
    }
    // defaults to 4, but this is a busy map for all indices, increase it a bit by default
    final int concurrencyLevel =  settings.getAsInt(FIELDDATA_CACHE_CONCURRENCY_LEVEL, 16);
    if (concurrencyLevel <= 0) {
        throw new IllegalArgumentException("concurrency_level must be > 0 but was: " + concurrencyLevel);
    }
    cacheBuilder.concurrencyLevel(concurrencyLevel);

    logger.debug("using size [{}] [{}]", size, new ByteSizeValue(sizeInBytes));
    cache = cacheBuilder.build();

    this.cleanInterval = settings.getAsTime(FIELDDATA_CLEAN_INTERVAL_SETTING, TimeValue.timeValueMinutes(1));
    // Start thread that will manage cleaning the field data cache periodically
    threadPool.schedule(this.cleanInterval, ThreadPool.Names.SAME,
            new FieldDataCacheCleaner(this.cache, this.logger, this.threadPool, this.cleanInterval));
}
 
Example 17
Source Project: Elasticsearch   Source File: ThreadPool.java    License: Apache License 2.0 5 votes vote down vote up
EstimatedTimeThread(String name, long interval) {
    super(name);
    this.interval = interval;
    this.estimatedTimeInMillis = TimeValue.nsecToMSec(System.nanoTime());
    this.counter = new TimeCounter();
    setDaemon(true);
}
 
Example 18
Source Project: elasticsearch-helper   Source File: GcMonitor.java    License: Apache License 2.0 5 votes vote down vote up
public GcMonitor(Settings settings) {
    this.enabled = settings.getAsBoolean("monitor.gc.enabled", false);
    TimeValue interval = settings.getAsTime("monitor.gc.interval", timeValueSeconds(1));
    this.gcThresholds = new HashMap<>();
    Map<String, Settings> gcThresholdGroups = settings.getGroups("monitor.gc.level");
    for (Map.Entry<String, Settings> entry : gcThresholdGroups.entrySet()) {
        String name = entry.getKey();
        TimeValue warn = entry.getValue().getAsTime("warn", null);
        TimeValue info = entry.getValue().getAsTime("info", null);
        TimeValue debug = entry.getValue().getAsTime("debug", null);
        if (warn == null || info == null || debug == null) {
            logger.warn("ignoring gc_threshold for [{}], missing warn/info/debug values", name);
        } else {
            gcThresholds.put(name, new GcThreshold(name, warn.millis(), info.millis(), debug.millis()));
        }
    }
    if (!gcThresholds.containsKey(JvmInfo.YOUNG)) {
        gcThresholds.put(JvmInfo.YOUNG, new GcThreshold(JvmInfo.YOUNG, 1000, 700, 400));
    }
    if (!gcThresholds.containsKey(JvmInfo.OLD)) {
        gcThresholds.put(JvmInfo.OLD, new GcThreshold(JvmInfo.OLD, 10000, 5000, 2000));
    }
    if (!gcThresholds.containsKey("default")) {
        gcThresholds.put("default", new GcThreshold("default", 10000, 5000, 2000));
    }
    logger.debug("enabled [{}], interval [{}], gc_threshold [{}]", enabled, interval, this.gcThresholds);
    if (enabled) {
        scheduledFuture = Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(new GcMonitorThread(), 0L, interval.seconds(), TimeUnit.SECONDS);
    }
}
 
Example 19
Source Project: Elasticsearch   Source File: StopWatch.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Return the time taken by the last task.
 */
public TimeValue lastTaskTime() throws IllegalStateException {
    if (this.lastTaskInfo == null) {
        throw new IllegalStateException("No tests run: can't get last interval");
    }
    return this.lastTaskInfo.getTime();
}
 
Example 20
Source Project: Elasticsearch   Source File: Retry.java    License: Apache License 2.0 5 votes vote down vote up
private BulkResponse getAccumulatedResponse() {
    BulkItemResponse[] itemResponses;
    synchronized (responses) {
        itemResponses = responses.toArray(new BulkItemResponse[1]);
    }
    long stopTimestamp = System.nanoTime();
    long totalLatencyMs = TimeValue.timeValueNanos(stopTimestamp - startTimestampNanos).millis();
    return new BulkResponse(itemResponses, totalLatencyMs);
}
 
Example 21
Source Project: Elasticsearch   Source File: IndicesTTLService.java    License: Apache License 2.0 5 votes vote down vote up
@Inject
public IndicesTTLService(Settings settings, ClusterService clusterService, IndicesService indicesService, NodeSettingsService nodeSettingsService, TransportBulkAction bulkAction) {
    super(settings);
    this.clusterService = clusterService;
    this.indicesService = indicesService;
    TimeValue interval = this.settings.getAsTime("indices.ttl.interval", DEFAULT_TTL_INTERVAL);
    this.bulkAction = bulkAction;
    this.bulkSize = this.settings.getAsInt("indices.ttl.bulk_size", 10000);
    this.purgerThread = new PurgerThread(EsExecutors.threadName(settings, "[ttl_expire]"), interval);

    nodeSettingsService.addListener(new ApplySettings());
}
 
Example 22
Source Project: elasticsearch-sql   Source File: ElasticJoinExecutor.java    License: Apache License 2.0 5 votes vote down vote up
protected SearchResponse scrollOneTimeWithMax(Client client,TableInJoinRequestBuilder tableRequest) {
        SearchResponse responseWithHits;SearchRequestBuilder scrollRequest = tableRequest.getRequestBuilder()
                .setScroll(new TimeValue(60000))
                .setSize(MAX_RESULTS_ON_ONE_FETCH);
        boolean ordered = tableRequest.getOriginalSelect().isOrderdSelect();
        if(!ordered) scrollRequest.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC);
        responseWithHits = scrollRequest.get();
        //on ordered select - not using SCAN , elastic returns hits on first scroll
        //es5.0 elastic always return docs on scan
//        if(!ordered)
//            responseWithHits = client.prepareSearchScroll(responseWithHits.getScrollId()).setScroll(new TimeValue(600000)).get();
        return responseWithHits;
    }
 
Example 23
Source Project: Elasticsearch   Source File: IndicesService.java    License: Apache License 2.0 5 votes vote down vote up
@Inject
public IndicesService(Settings settings, IndicesLifecycle indicesLifecycle, IndicesAnalysisService indicesAnalysisService, Injector injector, NodeEnvironment nodeEnv) {
    super(settings);
    this.indicesLifecycle = (InternalIndicesLifecycle) indicesLifecycle;
    this.indicesAnalysisService = indicesAnalysisService;
    this.injector = injector;
    this.pluginsService = injector.getInstance(PluginsService.class);
    this.indicesLifecycle.addListener(oldShardsStats);
    this.nodeEnv = nodeEnv;
    this.shardsClosedTimeout = settings.getAsTime(INDICES_SHARDS_CLOSED_TIMEOUT, new TimeValue(1, TimeUnit.DAYS));
}
 
Example 24
Source Project: Elasticsearch   Source File: TranslogService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void onRefreshSettings(Settings settings) {
    int flushThresholdOperations = settings.getAsInt(INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS, TranslogService.this.flushThresholdOperations);
    if (flushThresholdOperations != TranslogService.this.flushThresholdOperations) {
        logger.info("updating flush_threshold_ops from [{}] to [{}]", TranslogService.this.flushThresholdOperations, flushThresholdOperations);
        TranslogService.this.flushThresholdOperations = flushThresholdOperations;
    }
    ByteSizeValue flushThresholdSize = settings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, TranslogService.this.flushThresholdSize);
    if (!flushThresholdSize.equals(TranslogService.this.flushThresholdSize)) {
        logger.info("updating flush_threshold_size from [{}] to [{}]", TranslogService.this.flushThresholdSize, flushThresholdSize);
        TranslogService.this.flushThresholdSize = flushThresholdSize;
    }
    TimeValue flushThresholdPeriod = settings.getAsTime(INDEX_TRANSLOG_FLUSH_THRESHOLD_PERIOD, TranslogService.this.flushThresholdPeriod);
    if (!flushThresholdPeriod.equals(TranslogService.this.flushThresholdPeriod)) {
        logger.info("updating flush_threshold_period from [{}] to [{}]", TranslogService.this.flushThresholdPeriod, flushThresholdPeriod);
        TranslogService.this.flushThresholdPeriod = flushThresholdPeriod;
    }
    TimeValue interval = settings.getAsTime(INDEX_TRANSLOG_FLUSH_INTERVAL, TranslogService.this.interval);
    if (!interval.equals(TranslogService.this.interval)) {
        logger.info("updating interval from [{}] to [{}]", TranslogService.this.interval, interval);
        TranslogService.this.interval = interval;
    }
    boolean disableFlush = settings.getAsBoolean(INDEX_TRANSLOG_DISABLE_FLUSH, TranslogService.this.disableFlush);
    if (disableFlush != TranslogService.this.disableFlush) {
        logger.info("updating disable_flush from [{}] to [{}]", TranslogService.this.disableFlush, disableFlush);
        TranslogService.this.disableFlush = disableFlush;
    }
}
 
Example 25
Source Project: elasticsearch-helper   Source File: BulkNodeClient.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public BulkNodeClient waitForResponses(TimeValue maxWaitTime) throws InterruptedException, ExecutionException {
    if (closed) {
        throw new ElasticsearchException("client is closed");
    }
    while (!bulkProcessor.awaitClose(maxWaitTime.getMillis(), TimeUnit.MILLISECONDS)) {
        logger.warn("still waiting for responses");
    }
    return this;
}
 
Example 26
Source Project: summerframework   Source File: ElasticsearchTemplate.java    License: Apache License 2.0 5 votes vote down vote up
public List<Map<String, Object>> highLightResultSet(ESBasicInfo esBasicInfo, QueryCondition queryCondition,
    HighLight highLight) {
    SearchResponse response = esClient.prepareSearch(esBasicInfo.getIndex()).setTypes(esBasicInfo.getType())
        .setSearchType(queryCondition.getSearchType()).setScroll(new TimeValue(queryCondition.getMillis()))
        .setQuery(queryCondition.getQueryBuilder()).setSize(queryCondition.getSize())
        .highlighter(highLight.getBuilder()).execute().actionGet();

    String highlightField = highLight.getField();
    List<Map<String, Object>> sourceList = new ArrayList<>();

    for (SearchHit searchHit : response.getHits()) {
        Map<String, Object> element = new HashMap<>();
        StringBuilder stringBuilder = new StringBuilder();
        if (StringUtils.isNotEmpty(highlightField)) {
            Text[] text = searchHit.getHighlightFields().get(highlightField).getFragments();

            if (text != null) {
                for (Text str : text) {
                    stringBuilder.append(str.string());
                }

                log.info("遍历 高亮结果集{}", stringBuilder.toString());
                element.put(highlightField, stringBuilder.toString());
            }
        }
        sourceList.add(element);
    }

    return sourceList;
}
 
Example 27
Source Project: log4j2-elasticsearch   Source File: BulkProcessorFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public BatchEmitter createInstance(int batchSize, int deliveryInterval, ClientObjectFactory clientObjectFactory, FailoverPolicy failoverPolicy) {

    Function<BulkRequest, Boolean> failureHandler = clientObjectFactory.createFailureHandler(failoverPolicy);

    BulkProcessor.Listener listener = new BulkExecutionListener(failureHandler);

    BulkProcessor.Builder builder = BulkProcessor.builder((Client) clientObjectFactory.createClient(), listener)
            .setBulkActions(batchSize)
            .setFlushInterval(TimeValue.timeValueMillis(deliveryInterval));
    return new BulkProcessorDelegate(builder.build());
}
 
Example 28
Source Project: storm-crawler   Source File: ElasticSearchConnection.java    License: Apache License 2.0 5 votes vote down vote up
public static ElasticSearchConnection getConnection(Map stormConf,
        String boltType, BulkProcessor.Listener listener) {

    String flushIntervalString = ConfUtils.getString(stormConf,
            "es." + boltType + ".flushInterval", "5s");

    TimeValue flushInterval = TimeValue.parseTimeValue(flushIntervalString,
            TimeValue.timeValueSeconds(5), "flushInterval");

    int bulkActions = ConfUtils.getInt(stormConf,
            "es." + boltType + ".bulkActions", 50);

    int concurrentRequests = ConfUtils.getInt(stormConf,
            "es." + boltType + ".concurrentRequests", 1);

    RestHighLevelClient client = getClient(stormConf, boltType);

    boolean sniff = ConfUtils.getBoolean(stormConf,
            "es." + boltType + ".sniff", true);
    Sniffer sniffer = null;
    if (sniff) {
        sniffer = Sniffer.builder(client.getLowLevelClient()).build();
    }

    BulkProcessor bulkProcessor = BulkProcessor
            .builder((request, bulkListener) -> client.bulkAsync(request,
                    RequestOptions.DEFAULT, bulkListener), listener)
            .setFlushInterval(flushInterval).setBulkActions(bulkActions)
            .setConcurrentRequests(concurrentRequests).build();

    return new ElasticSearchConnection(client, bulkProcessor, sniffer);
}
 
Example 29
Source Project: Elasticsearch   Source File: HttpDownloadHelper.java    License: Apache License 2.0 5 votes vote down vote up
public boolean download(URL source, Path dest, @Nullable DownloadProgress progress, TimeValue timeout) throws Exception {
    if (Files.exists(dest) && skipExisting) {
        return true;
    }

    //don't do any progress, unless asked
    if (progress == null) {
        progress = new NullProgress();
    }

    //set the timestamp to the file date.
    long timestamp = 0;

    boolean hasTimestamp = false;
    if (useTimestamp && Files.exists(dest) ) {
        timestamp = Files.getLastModifiedTime(dest).toMillis();
        hasTimestamp = true;
    }

    GetThread getThread = new GetThread(source, dest, hasTimestamp, timestamp, progress);

    try {
        getThread.setDaemon(true);
        getThread.start();
        getThread.join(timeout.millis());

        if (getThread.isAlive()) {
            throw new ElasticsearchTimeoutException("The GET operation took longer than " + timeout + ", stopping it.");
        }
    }
    catch (InterruptedException ie) {
        return false;
    } finally {
        getThread.closeStreams();
    }

    return getThread.wasSuccessful();
}
 
Example 30
Source Project: elasticsearch-helper   Source File: HttpBulkNodeClient.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public HttpBulkNodeClient waitForResponses(TimeValue maxWaitTime) throws InterruptedException, ExecutionException {
    if (closed) {
        throw new ElasticsearchException("client is closed");
    }
    while (!bulkProcessor.awaitClose(maxWaitTime.getMillis(), TimeUnit.MILLISECONDS)) {
        logger.warn("still waiting for responses");
    }
    return this;
}