org.elasticsearch.action.search.SearchScrollRequestBuilder Java Examples

The following examples show how to use org.elasticsearch.action.search.SearchScrollRequestBuilder. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ScanAndScrollStreamProducer.java From sfs with Apache License 2.0

6 votes

protected Observable<SearchResponse> scroll() {
    return defer(() -> {
        if (scrollId == null) {
            return just(null);
        }

        SearchScrollRequestBuilder request = elasticsearch.get()
                .prepareSearchScroll(scrollId)
                .setScroll(timeValueMillis(elasticsearch.getDefaultScrollTimeout()));

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("Request = " + Jsonify.toString(request));
        }

        return elasticsearch.execute(vertxContext, request, elasticsearch.getDefaultSearchTimeout())
                .map(Optional::get)
                .flatMap(searchResponse -> just(searchResponse)
                        .map(searchResponse1 -> {
                            if (LOGGER.isDebugEnabled()) {
                                LOGGER.debug("Response = " + Jsonify.toString(searchResponse1));
                            }
                            return searchResponse1;
                        }));
    });
}

Example #2

Source File: DefaultElasticSearchService.java From vertx-elasticsearch-service with Apache License 2.0

6 votes

@Override
public void searchScroll(String scrollId, SearchScrollOptions options, Handler<AsyncResult<com.hubrick.vertx.elasticsearch.model.SearchResponse>> resultHandler) {
    final SearchScrollRequestBuilder builder = client.prepareSearchScroll(scrollId);

    if (options != null) {
        if (options.getScroll() != null) builder.setScroll(options.getScroll());
    }

    builder.execute(new ActionListener<SearchResponse>() {
        @Override
        public void onResponse(SearchResponse searchResponse) {
            resultHandler.handle(Future.succeededFuture(mapToSearchResponse(searchResponse)));
        }

        @Override
        public void onFailure(Exception t) {
            handleFailure(resultHandler, t);
        }
    });
}

Example #3

Source File: EsStore.java From soundwave with Apache License 2.0

5 votes

protected <E extends EsDocument> ScrollableResponse<List<E>> scrollNext(
    String scrollId, ThrowingFunction<String, E> createFunc)
    throws Exception {
  SearchScrollRequestBuilder
      builder =
      esClient.prepareSearchScroll(scrollId)
          .setScroll(TimeValue.timeValueMillis(SCROLLDEFAULTTIMEOUT));
  SearchResponse response = builder.execute().actionGet();
  ArrayList<E> list = new ArrayList<>();
  ScrollableResponse<List<E>> ret = new ScrollableResponse<>();
  ret.setValue(list);
  ret.setContinousToken(response.getScrollId());
  if (response.getHits().totalHits() == 0) {
    //Clear the scroll as early as possible to save resource
    ClearScrollRequestBuilder
        clearRequestBuilder =
        esClient.prepareClearScroll().addScrollId(scrollId);
    clearRequestBuilder.execute();
    ret.setScrollToEnd(true);
  } else {
    for (int i = 0; i < response.getHits().getHits().length; i++) {
      String str = response.getHits().getAt(i).getSourceAsString();
      E element = createFunc.apply(str);
      element.setId(response.getHits().getAt(i).getId());
      list.add(element);
    }
  }
  return ret;
}

Example #4

Source File: Elasticsearch.java From jlogstash-input-plugin with Apache License 2.0

5 votes

public void emit() {
    // 构建查询
    Scroll esScroll = new Scroll(new TimeValue(scroll, TimeUnit.MINUTES));
    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index).setScroll(esScroll).setQuery(query).setSize(size);
    if (StringUtils.isNotEmpty(type)) {
        searchRequestBuilder.setTypes(type);
    }

    if (logger.isDebugEnabled()) {
        logger.debug(searchRequestBuilder.toString());
    }

    String scrollId = null;
    try {
        SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
        scrollId = searchResponse.getScrollId();

        long total = searchResponse.getHits().getTotalHits();
        // 处理第一次search查询
        int messageCount = submitMessage(searchResponse);

        // 计算剩余的批次
        long remainTotal = total - messageCount;
        long batchNum = remainTotal % size == 0 ? remainTotal / size : (remainTotal / size) + 1;

        for (long i = 0; !stop && i < batchNum; i++) {
            // 按批查询数据
            SearchScrollRequestBuilder scrollRequestBuilder = client.prepareSearchScroll(scrollId).setScroll(esScroll);
            searchResponse = scrollRequestBuilder.execute().actionGet();
            submitMessage(searchResponse);
        }
    } catch (Exception e) {
        logger.error("query error", e);
    } finally {
        if (StringUtils.isNotEmpty(scrollId)) {
            client.prepareClearScroll().addScrollId(scrollId).execute().actionGet();
        }
    }
}

Example #5

Source File: ElasticClient.java From Stargraph with MIT License

4 votes

public SearchScrollRequestBuilder prepareSearchScroll(String scrollId) {
    return client.prepareSearchScroll(scrollId);
}

Example #6

Source File: ClientWithStats.java From rdf4j with BSD 3-Clause "New" or "Revised" License

4 votes

@Override
public SearchScrollRequestBuilder prepareSearchScroll(String scrollId) {
	return wrapped.prepareSearchScroll(scrollId);
}

Example #7

Source File: ListContainers.java From sfs with Apache License 2.0

4 votes

protected Observable<Void> scroll(
        final PersistentAccount account,
        final String prefix,
        final String delimiter,
        final String marker,
        final String endMarker,
        final int limit,
        final Elasticsearch elasticsearch,
        final String scrollId,
        final NavigableMap<String, SparseContainer> listedContainers) {
    SearchScrollRequestBuilder scrollRequest =
            elasticsearch.get().prepareSearchScroll(scrollId)
                    .setScroll(timeValueMillis(elasticsearch.getDefaultScrollTimeout()));

    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug(format("Search Request = %s", Jsonify.toString(scrollRequest)));
    }

    return elasticsearch.execute(vertxContext, scrollRequest, elasticsearch.getDefaultSearchTimeout())
            .map(Optional::get)
            .flatMap(searchResponse -> {

                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug(format("Search Response = %s", Jsonify.toString(searchResponse)));
                }
                SearchHits hits = searchResponse.getHits();
                int numberOfHits = hits.getHits().length;
                if (numberOfHits > 0) {
                    return toIterable(account, prefix, delimiter, marker, endMarker, hits)
                            .doOnNext(sparseContainer -> {
                                String id = sparseContainer.getContainerName();
                                SparseContainer existing = listedContainers.get(id);
                                if (existing == null) {
                                    listedContainers.put(id, sparseContainer);
                                } else {
                                    existing.setByteCount(existing.getByteCount() + sparseContainer.getByteCount());
                                    existing.setObjectCount(existing.getObjectCount() + sparseContainer.getObjectCount());
                                }
                                if (listedContainers.size() > limit) {
                                    listedContainers.pollLastEntry();
                                }
                            })
                            .count()
                            .map(new ToVoid<>())
                            .flatMap(aVoid -> scroll(account, prefix, delimiter, marker, endMarker, limit, elasticsearch, searchResponse.getScrollId(), listedContainers));
                } else {
                    return clearScroll(elasticsearch, searchResponse.getScrollId());
                }
            });
}

Example #8

Source File: ListObjects.java From sfs with Apache License 2.0

4 votes

protected Observable<Void> scroll(
        final PersistentContainer container,
        final String prefix,
        final String delimiter,
        final String marker,
        final String endMarker,
        final int limit,
        final Elasticsearch elasticsearch,
        final String scrollId,
        final NavigableMap<String, ListedObject> listedObjects) {
    SearchScrollRequestBuilder scrollRequest =
            elasticsearch.get().prepareSearchScroll(scrollId)
                    .setScroll(timeValueMillis(elasticsearch.getDefaultScrollTimeout()));

    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug(format("Search Request = %s", Jsonify.toString(scrollRequest)));
    }

    return elasticsearch.execute(vertxContext, scrollRequest, elasticsearch.getDefaultSearchTimeout())
            .map(Optional::get)
            .flatMap(searchResponse -> {

                if (LOGGER.isDebugEnabled()) {
                    LOGGER.debug(format("Search Response = %s", Jsonify.toString(searchResponse)));
                }
                SearchHits hits = searchResponse.getHits();
                int numberOfHits = hits.getHits().length;
                if (numberOfHits > 0) {
                    for (ListedObject listedObject : toIterable(container, prefix, delimiter, marker, endMarker, hits)) {
                        String id = listedObject.getName();
                        ListedObject existing = listedObjects.get(id);
                        if (existing == null) {
                            listedObjects.put(id, listedObject);
                        } else {
                            existing.setLength(existing.getLength() + listedObject.getLength());
                        }
                        if (listedObjects.size() > limit) {
                            listedObjects.pollLastEntry();
                        }
                    }
                    return scroll(container, prefix, delimiter, marker, endMarker, limit, elasticsearch, searchResponse.getScrollId(), listedObjects);
                } else {
                    return clearScroll(elasticsearch, searchResponse.getScrollId());
                }
            });
}

Example #9

Source File: DefaultQueryAction.java From elasticsearch-sql with Apache License 2.0

4 votes

@Override
	public SqlElasticSearchRequestBuilder explain() throws SqlParseException {
        Hint scrollHint = null;
        for (Hint hint : select.getHints()) {
            if (hint.getType() == HintType.USE_SCROLL) {
                scrollHint = hint;
                break;
            }
        }
        if (scrollHint != null && scrollHint.getParams()[0] instanceof String) {
            return new SqlElasticSearchRequestBuilder(new SearchScrollRequestBuilder(client, SearchScrollAction.INSTANCE, (String) scrollHint.getParams()[0]).setScroll(new TimeValue((Integer) scrollHint.getParams()[1])));
        }

		/*
		zhongshu-comment 6.1.1.5这个版本和elastic6.1.1这个分支用的是这一行代码
		但是在本地调试时我的client没有实例化，并没有去连es，所以这行代码会报空指针
		那就将这行注释掉吧，以后就用下面那行
		 */
//		this.request = client.prepareSearch();

		/*
		zhongshu-comment  6.2.4.1这个版本和master_zhongshu_dev_01用的是这一行代码，虽然client为null，但是下面这行代码并不会报空指针
							为了在本地调试、执行下文的那些代码获得es的dsl，所以就使用这行代码，暂时将上面哪一行注释掉，上线的时候记得替换掉
		变量request是es搜索请求对象，调用的是es的api，SearchRequestBuilder是es的原生api
		 */
        this.request = new SearchRequestBuilder(client, SearchAction.INSTANCE);
		setIndicesAndTypes();

		//zhongshu-comment 将Select对象中封装的sql token信息转换并传到成员变量es搜索请求对象request中
		setFields(select.getFields());

		setWhere(select.getWhere());
		setSorts(select.getOrderBys());
		setLimit(select.getOffset(), select.getRowCount());

        //
        if (scrollHint != null) {
            if (!select.isOrderdSelect())
                request.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC);
            request.setSize((Integer) scrollHint.getParams()[0]).setScroll(new TimeValue((Integer) scrollHint.getParams()[1]));
        } else {
            request.setSearchType(SearchType.DFS_QUERY_THEN_FETCH);
        }
        updateRequestWithIndexAndRoutingOptions(select, request);
		updateRequestWithHighlight(select, request);
		updateRequestWithCollapse(select, request);
		updateRequestWithPostFilter(select, request);
		updateRequestWithStats(select, request);
		updateRequestWithPreference(select, request);
		updateRequestWithTrackTotalHits(select, request);
		updateRequestWithTimeout(select, request);
		updateRequestWithIndicesOptions(select, request);
		updateRequestWithMinScore(select, request);
		SqlElasticSearchRequestBuilder sqlElasticRequestBuilder = new SqlElasticSearchRequestBuilder(request);

		return sqlElasticRequestBuilder;
	}

Example #10

Source File: FessEsClient.java From fess with Apache License 2.0

4 votes

@Override
public SearchScrollRequestBuilder prepareSearchScroll(final String scrollId) {
    return client.prepareSearchScroll(scrollId);
}

Example #11

Source File: SearchRequestBuilderStrategyV2.java From usergrid with Apache License 2.0

4 votes

public SearchScrollRequestBuilder getScrollBuilder(String scrollId){
    return esProvider.getClient().prepareSearchScroll( scrollId );
}