org.elasticsearch.action.search.ShardSearchFailure Java Examples

public static SearchResponse createEmptySearchResponse() throws IOException {
    return new SearchResponse(
        new InternalSearchResponse(
            new SearchHits(new SearchHit[0], new TotalHits(0, TotalHits.Relation.EQUAL_TO), 1.0f),
            new InternalAggregations(Collections.emptyList()),
            new Suggest(Collections.emptyList()),
            new SearchProfileShardResults(Collections.emptyMap()),
            false,
            false,
            1
        ),
        "",
        5,
        5,
        0,
        100,
        ShardSearchFailure.EMPTY_ARRAY,
        SearchResponse.Clusters.EMPTY
    );
}
 

public static SearchResponse createSearchResponse(ToXContentObject o) throws IOException {
    XContentBuilder content = o.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS);

    SearchHit[] hits = new SearchHit[1];
    hits[0] = new SearchHit(0).sourceRef(BytesReference.bytes(content));

    return new SearchResponse(
        new InternalSearchResponse(
            new SearchHits(hits, new TotalHits(1, TotalHits.Relation.EQUAL_TO), 1.0f),
            new InternalAggregations(Collections.emptyList()),
            new Suggest(Collections.emptyList()),
            new SearchProfileShardResults(Collections.emptyMap()),
            false,
            false,
            1
        ),
        "",
        5,
        5,
        0,
        100,
        ShardSearchFailure.EMPTY_ARRAY,
        SearchResponse.Clusters.EMPTY
    );
}
 

protected void validateRespose(final SearchResponse response) {
    final int totalShards = response.getTotalShards();
    final int successfulShards = response.getSuccessfulShards();
    if (totalShards != successfulShards) {
        throw new MissingShardsException(totalShards - successfulShards
                + " shards are failed.");
    }
    final ShardSearchFailure[] failures = response.getShardFailures();
    if (failures.length > 0) {
        final StringBuilder buf = new StringBuilder();
        for (final ShardOperationFailedException failure : failures) {
            buf.append('\n').append(failure.toString());
        }
        throw new OperationFailedException("Search Operation Failed: "
                + buf.toString());
    }
}
 

@Override
public List<T> findAll(SearchRequest searchRequest) throws Exception {
	SearchResponse searchResp = this.restHighLevelClient.search(searchRequest);
	for (ShardSearchFailure failure : searchResp.getShardFailures()) {
		listener.onFailure(failure);
	}
	SearchHits hits = searchResp.getHits();
	SearchHit[] searchHits = hits.getHits();
	List<T> list = new ArrayList<>();
	for (SearchHit hit : searchHits) {
		String sourceAsString = hit.getSourceAsString();
		T t = JacksonUtils.parseJSON(sourceAsString, clazzP);
		list.add(t);
	}
	Collections.reverse(list);
	return list;
}
 

/**
 * Perform a test search request to validate the element prior to storing it.
 *
 * @param validation validation info
 * @param element the element stored
 * @param task the parent task
 * @param listener the action listener to write to
 * @param onSuccess action ro run when the validation is successfull
 */
private void validate(FeatureValidation validation,
                      StorableElement element,
                      Task task,
                      ActionListener<FeatureStoreResponse> listener,
                      Runnable onSuccess) {
    ValidatingLtrQueryBuilder ltrBuilder = new ValidatingLtrQueryBuilder(element,
            validation, factory);
    SearchRequestBuilder builder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
    builder.setIndices(validation.getIndex());
    builder.setQuery(ltrBuilder);
    builder.setFrom(0);
    builder.setSize(20);
    // Bail out early and don't score the whole index.
    builder.setTerminateAfter(1000);
    builder.request().setParentTask(clusterService.localNode().getId(), task.getId());
    builder.execute(wrap((r) -> {
            if (r.getFailedShards() > 0) {
                ShardSearchFailure failure = r.getShardFailures()[0];
                throw new IllegalArgumentException("Validating the element caused " + r.getFailedShards() +
                        " shard failures, see root cause: " + failure.reason(), failure.getCause());
            }
            onSuccess.run();
        },
        (e) -> listener.onFailure(new IllegalArgumentException("Cannot store element, validation failed.", e))));
}
 

@Override
public SearchResponse search(String searchTerms, List<String> references, List<String> siteIds, int start, int end, Map<String,String> additionalSearchInfromation) {
    return new SearchResponse(
            new InternalSearchResponse(new InternalSearchHits(new InternalSearchHit[0], 0, 0.0f), new InternalFacets(Collections.EMPTY_LIST), new InternalAggregations(Collections.EMPTY_LIST), new Suggest(), false, false),
            "no-op",
            1,
            1,
            1,
            new ShardSearchFailure[0]
    );
}
 

@Test
public void searchShouldHandleShardFailure() throws InvalidSearchException, IOException {
  // mocks
  SearchResponse response = mock(SearchResponse.class);
  SearchRequest request = new SearchRequest();
  ShardSearchFailure fail = mock(ShardSearchFailure.class);
  SearchShardTarget target = new SearchShardTarget("node1", mock(Index.class), 1, "metron");

  // response will have status of OK
  when(response.status()).thenReturn(RestStatus.OK);

  // response will indicate 1 search hit
  SearchHits hits = mock(SearchHits.class);
  when(hits.getTotalHits()).thenReturn(1L);

  // the response will report shard failures
  when(response.getFailedShards()).thenReturn(1);
  when(response.getTotalShards()).thenReturn(2);
  when(response.getHits()).thenReturn(hits);

  // the response will return the failures
  ShardSearchFailure[] failures = { fail };
  when(response.getShardFailures()).thenReturn(failures);

  // shard failure needs to report the node
  when(fail.shard()).thenReturn(target);

  // shard failure needs to report details of failure
  when(fail.index()).thenReturn("bro_index_2017-10-11");
  when(fail.shardId()).thenReturn(1);

  // search should succeed, even with failed shards
  ElasticsearchRequestSubmitter submitter = setup(response);
  SearchResponse actual = submitter.submitSearch(request);
  assertNotNull(actual);
}
 

/**
 * Handle individual shard failures that can occur even when the response is OK.  These
 * can indicate misconfiguration of the search indices.
 * @param request The search request.
 * @param response  The search response.
 */
private void handleShardFailures(
        org.elasticsearch.action.search.SearchRequest request,
        org.elasticsearch.action.search.SearchResponse response) {
  /*
   * shard failures are only logged.  the search itself is not failed.  this approach
   * assumes that a user is interested in partial search results, even if the
   * entire search result set cannot be produced.
   *
   * for example, assume the user adds an additional sensor and the telemetry
   * is indexed into a new search index.  if that search index is misconfigured,
   * it can result in partial shard failures.  rather than failing the entire search,
   * we log the error and allow the results to be returned from shards that
   * are correctly configured.
   */
  int errors = ArrayUtils.getLength(response.getShardFailures());
  LOG.error("Search resulted in {}/{} shards failing; errors={}, search={}",
          response.getFailedShards(),
          response.getTotalShards(),
          errors,
          ElasticsearchUtils.toJSON(request).orElse("???"));

  // log each reported failure
  int failureCount=1;
  for(ShardSearchFailure fail: response.getShardFailures()) {
    String msg = String.format(
            "Shard search failure [%s/%s]; reason=%s, index=%s, shard=%s, status=%s, nodeId=%s",
            failureCount,
            errors,
            ExceptionUtils.getRootCauseMessage(fail.getCause()),
            fail.index(),
            fail.shardId(),
            fail.status(),
            fail.shard().getNodeId());
    LOG.error(msg, fail.getCause());
  }
}
 

@Override
public SearchResponse search(String searchTerms, List<String> references, List<String> siteIds, int start, int end, Map<String,String> additionalSearchInfromation) {
    return new SearchResponse(
            new InternalSearchResponse(new InternalSearchHits(new InternalSearchHit[0], 0, 0.0f), new InternalFacets(Collections.EMPTY_LIST), new InternalAggregations(Collections.EMPTY_LIST), new Suggest(), false, false),
            "no-op",
            1,
            1,
            1,
            new ShardSearchFailure[0]
    );
}
 

public static SearchResponse checkForFailures(SearchResponse searchResponse) {
    ShardSearchFailure[] shardFailures = searchResponse.getShardFailures();
    if (shardFailures.length > 0) {
        for (ShardSearchFailure shardFailure : shardFailures) {
            LOGGER.error("search response shard failure", shardFailure.getCause());
        }
        throw new VertexiumException("search response shard failures", shardFailures[0].getCause());
    }
    return searchResponse;
}
 

public static SearchResponse checkForFailures(SearchResponse searchResponse) {
    ShardSearchFailure[] shardFailures = searchResponse.getShardFailures();
    if (shardFailures.length > 0) {
        for (ShardSearchFailure shardFailure : shardFailures) {
            LOGGER.error("search response shard failure", shardFailure.getCause());
        }
        throw new VertexiumException("search response shard failures", shardFailures[0].getCause());
    }
    return searchResponse;
}
 

public static void givenSearchResultForDocuments(PluginClient client, String indexPattern, Map<String, BytesReference> docs) {
    List<SearchHit> hits = new ArrayList<>(docs.size());
    for (Map.Entry<String, BytesReference> entry : docs.entrySet()) {
        SearchHit hit = new SearchHit(1, entry.getKey(), null, null);
        hit.sourceRef(entry.getValue());
        hits.add(hit);
    }
    SearchHits searchHits = new SearchHits(hits.toArray(new SearchHit[hits.size()]), hits.size(), 1.0f);
    SearchResponseSections sections = new SearchResponseSections(searchHits, null, null, false, Boolean.FALSE, null,
            0);
    ShardSearchFailure[] failures = null;
    SearchResponse response = new SearchResponse(sections, "", 0, 0, 0, 0L, failures);

    when(client.search(anyString(), anyString(),anyInt(), anyBoolean())).thenReturn(response);
}
 

private void givenSearchResultToIncludePattern(String indexPattern) {
    SearchHit[] hits = new SearchHit[1];
    hits[0] = new SearchHit(1, indexPattern, null, null);
    int totHits = 1;
    if (indexPattern == null) {
        totHits = 0;
    }
    SearchHits searchHits = new SearchHits(hits, totHits, 1.0f);
    SearchResponseSections sections = new SearchResponseSections(searchHits, null, null, false, Boolean.FALSE, null,
            0);
    ShardSearchFailure[] failures = null;
    SearchResponse response = new SearchResponse(sections, "", 0, 0, 0, 0L, failures);

    when(client.search(anyString(), anyString(),anyInt(), anyBoolean())).thenReturn(response);
}
 

/**
 * Finalizes the shard in repository and then removes it from cluster state
 * <p>
 * This is non-blocking method that runs on a thread from SNAPSHOT thread pool
 *
 * @param entry   snapshot
 * @param failure failure reason or null if snapshot was successful
 */
private void endSnapshot(final SnapshotsInProgress.Entry entry, final String failure) {
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
        @Override
        public void run() {
            SnapshotId snapshotId = entry.snapshotId();
            try {
                final Repository repository = repositoriesService.repository(snapshotId.getRepository());
                logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshotId, entry.state(), failure);
                ArrayList<ShardSearchFailure> failures = new ArrayList<>();
                ArrayList<SnapshotShardFailure> shardFailures = new ArrayList<>();
                for (Map.Entry<ShardId, ShardSnapshotStatus> shardStatus : entry.shards().entrySet()) {
                    ShardId shardId = shardStatus.getKey();
                    ShardSnapshotStatus status = shardStatus.getValue();
                    if (status.state().failed()) {
                        failures.add(new ShardSearchFailure(status.reason(), new SearchShardTarget(status.nodeId(), shardId.getIndex(), shardId.id())));
                        shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId.getIndex(), shardId.id(), status.reason()));
                    }
                }
                Snapshot snapshot = repository.finalizeSnapshot(snapshotId, entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures));
                removeSnapshotFromClusterState(snapshotId, new SnapshotInfo(snapshot), null);
            } catch (Throwable t) {
                logger.warn("[{}] failed to finalize snapshot", t, snapshotId);
                removeSnapshotFromClusterState(snapshotId, null, t);
            }
        }
    });
}
 

@Test
public void extractAggregationValue() {
    int docId = 0, docId1 = 1, docId2 = 2, docId3 = 3, docId4 = 400, docId5 = 5000;
    SearchHit[] hit = new SearchHit[6];
    hit[0] = new SearchHit(docId);
    hit[1] = new SearchHit(docId1);
    hit[2] = new SearchHit(docId2);
    hit[3] = new SearchHit(docId3);
    hit[4] = new SearchHit(docId4);
    hit[5] = new SearchHit(docId5);
    SearchHits hits = new SearchHits(hit, 30000, 450);

    Aggregations aggregations = new Aggregations(new ArrayList());

    List<Suggest.Suggestion<? extends Suggest.Suggestion.Entry<? extends Suggest.Suggestion.Entry.Option>>> suggestions = new ArrayList();
    suggestions.add(new Suggest.Suggestion("sug1", 1));
    suggestions.add(new Suggest.Suggestion("sug2", 2));
    suggestions.add(new Suggest.Suggestion("sug3", 3));
    suggestions.add(new Suggest.Suggestion("sug4", 4));
    suggestions.add(new Suggest.Suggestion("sug5", 50));

    Suggest suggest = new Suggest(suggestions);

    SearchProfileShardResults profileResults = new SearchProfileShardResults(Collections.emptyMap());

    SearchResponseSections internalResponse = new SearchResponseSections(hits, aggregations, suggest, false, false, profileResults, 0);

    ShardSearchFailure[] shardFailures = new ShardSearchFailure[0];

    SearchResponse.Clusters clusters = SearchResponse.Clusters.EMPTY;

    SearchResponse sRes = new SearchResponse(internalResponse, "id1", 1, 1, 1, 1, shardFailures, clusters);

    JSONObject res = QueryUtil.extractAggregationValue(sRes);

    assertNotNull(res.get("suggest"));
    assertNotNull(res.get("hits"));
}
 

@Override
public ShardSearchFailure[] getShardFailures() {
  return searchResponse.getShardFailures();
}
 

private SearchResponse createSearchResponseWithScrollId(String scrollId) {
  return new SearchResponse(InternalSearchResponse.empty(), scrollId, 1, 1, 1, new ShardSearchFailure[0]);
}
 

/**
     * @return void
     * @Author pancm
     * @Description 普通查询
     * @Date 2019/9/12
     * @Param []
     **/
    private static void genSearch() throws IOException {
        String type = "_doc";
        String index = "test1";
        // 查询指定的索引库
        SearchRequest searchRequest = new SearchRequest(index);
        searchRequest.types(type);
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        // 设置查询条件
        sourceBuilder.query(QueryBuilders.termQuery("uid", "1234"));
        // 设置起止和结束
        sourceBuilder.from(0);
        sourceBuilder.size(5);
        sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        // 设置路由
//		searchRequest.routing("routing");
        // 设置索引库表达式
        searchRequest.indicesOptions(IndicesOptions.lenientExpandOpen());
        // 查询选择本地分片，默认是集群分片
        searchRequest.preference("_local");

        // 排序
        // 根据默认值进行降序排序
//		sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC));
        // 根据字段进行升序排序
//		sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC));

        // 关闭suorce查询
//		sourceBuilder.fetchSource(false);

        String[] includeFields = new String[]{"title", "user", "innerObject.*"};
        String[] excludeFields = new String[]{"_type"};
        // 包含或排除字段
//		sourceBuilder.fetchSource(includeFields, excludeFields);

        searchRequest.source(sourceBuilder);
        System.out.println("普通查询的DSL语句:"+sourceBuilder.toString());
        // 同步查询
        SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);

        // HTTP状态代码、执行时间或请求是否提前终止或超时
        RestStatus status = searchResponse.status();
        TimeValue took = searchResponse.getTook();
        Boolean terminatedEarly = searchResponse.isTerminatedEarly();
        boolean timedOut = searchResponse.isTimedOut();

        // 供关于受搜索影响的切分总数的统计信息，以及成功和失败的切分
        int totalShards = searchResponse.getTotalShards();
        int successfulShards = searchResponse.getSuccessfulShards();
        int failedShards = searchResponse.getFailedShards();
        // 失败的原因
        for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
            // failures should be handled here
        }
        // 结果
        searchResponse.getHits().forEach(hit -> {
            Map<String, Object> map = hit.getSourceAsMap();
            System.out.println("普通查询的结果:" + map);
        });
        System.out.println("\n=================\n");
    }
 

/**
 * shardFailures.
 * @return ShardSearchFailure[]
 */
@Deprecated
public ShardSearchFailure[] shardFailures() {
    return getShardFailures();
}
 

@Override
@Deprecated
public ShardSearchFailure[] getShardFailures() {
    return super.getShardFailures();
}
 

/**
 * shardFailures.
 * @return ShardSearchFailure[]
 */
@Deprecated
public ShardSearchFailure[] shardFailures() {
    return getShardFailures();
}
 

@Override
@Deprecated
public ShardSearchFailure[] getShardFailures() {
    return super.getShardFailures();
}
 

private void treatError(SearchResponse searchResponse) {
    log.error("Pwoblem when load configuration From ES");
    for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
        log.error(failure.toString());
    }
}
 

@Override
public void onFailure(ShardSearchFailure failure) {

}
 

void onFailure(ShardSearchFailure failure);