org.elasticsearch.search.SearchShardTarget Java Examples

public CrateSearchContext(long id,
                          final long nowInMillis,
                          SearchShardTarget shardTarget,
                          Engine.Searcher engineSearcher,
                          IndexService indexService,
                          final IndexShard indexShard,
                          ScriptService scriptService,
                          PageCacheRecycler pageCacheRecycler,
                          BigArrays bigArrays,
                          Counter timeEstimateCounter,
                          Optional<Scroll> scroll) {
    super(id, new CrateSearchShardRequest(nowInMillis, scroll, indexShard),
            shardTarget, engineSearcher, indexService,
            indexShard, scriptService, pageCacheRecycler,
            bigArrays, timeEstimateCounter, ParseFieldMatcher.STRICT, SearchService.NO_TIMEOUT);
    this.engineSearcher = engineSearcher;
}
 

void executeFetch(final int shardIndex, final SearchShardTarget shardTarget, final AtomicInteger counter,
                  final ShardFetchSearchRequest fetchSearchRequest, DiscoveryNode node) {
    searchService.sendExecuteFetch(node, fetchSearchRequest, new ActionListener<FetchSearchResult>() {
        @Override
        public void onResponse(FetchSearchResult result) {
            result.shardTarget(shardTarget);
            fetchResults.set(shardIndex, result);
            if (counter.decrementAndGet() == 0) {
                finishHim();
            }
        }

        @Override
        public void onFailure(Throwable t) {
            // the search context might not be cleared on the node where the fetch was executed for example
            // because the action was rejected by the thread pool. in this case we need to send a dedicated
            // request to clear the search context. by setting docIdsToLoad to null, the context will be cleared
            // in TransportSearchTypeAction.releaseIrrelevantSearchContexts() after the search request is done.
            docIdsToLoad.set(shardIndex, null);
            onFetchFailure(t, fetchSearchRequest, shardIndex, shardTarget, counter);
        }
    });
}
 

void executeFetch(final int shardIndex, final SearchShardTarget shardTarget, final AtomicInteger counter,
                  final ShardFetchSearchRequest fetchSearchRequest, DiscoveryNode node) {
    searchService.sendExecuteFetch(node, fetchSearchRequest, new ActionListener<FetchSearchResult>() {
        @Override
        public void onResponse(FetchSearchResult result) {
            result.shardTarget(shardTarget);
            fetchResults.set(shardIndex, result);
            if (counter.decrementAndGet() == 0) {
                finishHim();
            }
        }

        @Override
        public void onFailure(Throwable t) {
            // the search context might not be cleared on the node where the fetch was executed for example
            // because the action was rejected by the thread pool. in this case we need to send a dedicated
            // request to clear the search context. by setting docIdsToLoad to null, the context will be cleared
            // in TransportSearchTypeAction.releaseIrrelevantSearchContexts() after the search request is done.
            docIdsToLoad.set(shardIndex, null);
            onFetchFailure(t, fetchSearchRequest, shardIndex, shardTarget, counter);
        }
    });
}
 

protected final void addShardFailure(final int shardIndex, @Nullable SearchShardTarget shardTarget, Throwable t) {
    // we don't aggregate shard failures on non active shards (but do keep the header counts right)
    if (TransportActions.isShardNotAvailableException(t)) {
        return;
    }

    // lazily create shard failures, so we can early build the empty shard failure list in most cases (no failures)
    if (shardFailures == null) {
        synchronized (shardFailuresMutex) {
            if (shardFailures == null) {
                shardFailures = new AtomicArray<>(shardsIts.size());
            }
        }
    }
    ShardSearchFailure failure = shardFailures.get(shardIndex);
    if (failure == null) {
        shardFailures.set(shardIndex, new ShardSearchFailure(t, shardTarget));
    } else {
        // the failure is already present, try and not override it with an exception that is less meaningless
        // for example, getting illegal shard state
        if (TransportActions.isReadOverrideException(t)) {
            shardFailures.set(shardIndex, new ShardSearchFailure(t, shardTarget));
        }
    }
}
 

void onFirstPhaseResult(int shardIndex, ShardRouting shard, FirstResult result, ShardIterator shardIt) {
    result.shardTarget(new SearchShardTarget(shard.currentNodeId(), shard.index(), shard.id()));
    processFirstPhaseResult(shardIndex, result);
    // we need to increment successful ops first before we compare the exit condition otherwise if we
    // are fast we could concurrently update totalOps but then preempt one of the threads which can
    // cause the successor to read a wrong value from successfulOps if second phase is very fast ie. count etc.
    successfulOps.incrementAndGet();
    // increment all the "future" shards to update the total ops since we some may work and some may not...
    // and when that happens, we break on total ops, so we must maintain them
    final int xTotalOps = totalOps.addAndGet(shardIt.remaining() + 1);
    if (xTotalOps == expectedTotalOps) {
        try {
            innerMoveToSecondPhase();
        } catch (Throwable e) {
            if (logger.isDebugEnabled()) {
                logger.debug(shardIt.shardId() + ": Failed to execute [" + request + "] while moving to second phase", e);
            }
            raiseEarlyFailure(new ReduceSearchPhaseException(firstPhaseName(), "", e, buildShardFailures()));
        }
    } else if (xTotalOps > expectedTotalOps) {
        raiseEarlyFailure(new IllegalStateException("unexpected higher total ops [" + xTotalOps + "] compared to expected [" + expectedTotalOps + "]"));
    }
}
 

public DefaultSearchContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget,
                            Engine.Searcher engineSearcher, IndexService indexService, IndexShard indexShard,
                            ScriptService scriptService, PageCacheRecycler pageCacheRecycler,
                            BigArrays bigArrays, Counter timeEstimateCounter, ParseFieldMatcher parseFieldMatcher,
                            TimeValue timeout
) {
    super(parseFieldMatcher, request);
    this.id = id;
    this.request = request;
    this.searchType = request.searchType();
    this.shardTarget = shardTarget;
    this.engineSearcher = engineSearcher;
    this.scriptService = scriptService;
    this.pageCacheRecycler = pageCacheRecycler;
    // SearchContexts use a BigArrays that can circuit break
    this.bigArrays = bigArrays.withCircuitBreaking();
    this.dfsResult = new DfsSearchResult(id, shardTarget);
    this.queryResult = new QuerySearchResult(id, shardTarget);
    this.fetchResult = new FetchSearchResult(id, shardTarget);
    this.indexShard = indexShard;
    this.indexService = indexService;
    this.searcher = new ContextIndexSearcher(engineSearcher, indexService.cache().query(), indexShard.getQueryCachingPolicy());
    this.timeEstimateCounter = timeEstimateCounter;
    this.timeoutInMillis = timeout.millis();
}
 

public PercolateContext(PercolateShardRequest request, SearchShardTarget searchShardTarget, IndexShard indexShard,
                        IndexService indexService, PageCacheRecycler pageCacheRecycler,
                        BigArrays bigArrays, ScriptService scriptService, Query aliasFilter, ParseFieldMatcher parseFieldMatcher) {
    super(parseFieldMatcher, request);
    this.indexShard = indexShard;
    this.indexService = indexService;
    this.fieldDataService = indexService.fieldData();
    this.searchShardTarget = searchShardTarget;
    this.percolateQueries = indexShard.percolateRegistry().percolateQueries();
    this.types = new String[]{request.documentType()};
    this.pageCacheRecycler = pageCacheRecycler;
    this.bigArrays = bigArrays.withCircuitBreaking();
    this.querySearchResult = new QuerySearchResult(0, searchShardTarget);
    this.engineSearcher = indexShard.acquireSearcher("percolate");
    this.searcher = new ContextIndexSearcher(engineSearcher, indexService.cache().query(), indexShard.getQueryCachingPolicy());
    this.scriptService = scriptService;
    this.numberOfShards = request.getNumberOfShards();
    this.aliasFilter = aliasFilter;
    this.startTime = request.getStartTime();
}
 

/**
 * Finalizes the shard in repository and then removes it from cluster state
 * <p>
 * This is non-blocking method that runs on a thread from SNAPSHOT thread pool
 *
 * @param entry   snapshot
 * @param failure failure reason or null if snapshot was successful
 */
private void endSnapshot(final SnapshotsInProgress.Entry entry, final String failure) {
    threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(new Runnable() {
        @Override
        public void run() {
            SnapshotId snapshotId = entry.snapshotId();
            try {
                final Repository repository = repositoriesService.repository(snapshotId.getRepository());
                logger.trace("[{}] finalizing snapshot in repository, state: [{}], failure[{}]", snapshotId, entry.state(), failure);
                ArrayList<ShardSearchFailure> failures = new ArrayList<>();
                ArrayList<SnapshotShardFailure> shardFailures = new ArrayList<>();
                for (Map.Entry<ShardId, ShardSnapshotStatus> shardStatus : entry.shards().entrySet()) {
                    ShardId shardId = shardStatus.getKey();
                    ShardSnapshotStatus status = shardStatus.getValue();
                    if (status.state().failed()) {
                        failures.add(new ShardSearchFailure(status.reason(), new SearchShardTarget(status.nodeId(), shardId.getIndex(), shardId.id())));
                        shardFailures.add(new SnapshotShardFailure(status.nodeId(), shardId.getIndex(), shardId.id(), status.reason()));
                    }
                }
                Snapshot snapshot = repository.finalizeSnapshot(snapshotId, entry.indices(), entry.startTime(), failure, entry.shards().size(), Collections.unmodifiableList(shardFailures));
                removeSnapshotFromClusterState(snapshotId, new SnapshotInfo(snapshot), null);
            } catch (Throwable t) {
                logger.warn("[{}] failed to finalize snapshot", t, snapshotId);
                removeSnapshotFromClusterState(snapshotId, null, t);
            }
        }
    });
}
 

@Override
protected ShardExportResponse shardOperation(ShardExportRequest request) throws ElasticSearchException {


    IndexService indexService = indicesService.indexServiceSafe(request.index());
    IndexShard indexShard = indexService.shardSafe(request.shardId());

    SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().id(), request.index(), request.shardId());
    ExportContext context = new ExportContext(0,
        new ShardSearchRequest().types(request.types()).filteringAliases(request.filteringAliases()),
        shardTarget, indexShard.searcher(), indexService, indexShard, scriptService, cacheRecycler, nodePath);
    ExportContext.setCurrent(context);

    try {
        BytesReference source = request.source();
        exportParser.parseSource(context, source);
        context.preProcess();
        exporter.check(context);
        try {
            if (context.explain()) {
                return new ShardExportResponse(shardTarget.nodeIdText(), request.index(), request.shardId(), context.outputCmd(), context.outputCmdArray(), context.outputFile());
            } else {
                Exporter.Result res = exporter.execute(context);
                return new ShardExportResponse(shardTarget.nodeIdText(), request.index(), request.shardId(), context.outputCmd(), context.outputCmdArray(), context.outputFile(), res.outputResult.stdErr, res.outputResult.stdOut, res.outputResult.exit, res.numExported);
            }

        } catch (Exception e) {
            throw new QueryPhaseExecutionException(context, "failed to execute export", e);
        }
    } finally {
        // this will also release the index searcher
        context.release();
        SearchContext.removeCurrent();
    }
}
 

public SearchIntoContext(long id, ShardSearchRequest request,
        SearchShardTarget shardTarget, Engine.Searcher engineSearcher,
        IndexService indexService, IndexShard indexShard,
        ScriptService scriptService, CacheRecycler cacheRecycler) {
    super(id, request, shardTarget, engineSearcher, indexService,
            indexShard, scriptService, cacheRecycler);
}
 

@Test
public void searchShouldHandleShardFailure() throws InvalidSearchException, IOException {
  // mocks
  SearchResponse response = mock(SearchResponse.class);
  SearchRequest request = new SearchRequest();
  ShardSearchFailure fail = mock(ShardSearchFailure.class);
  SearchShardTarget target = new SearchShardTarget("node1", mock(Index.class), 1, "metron");

  // response will have status of OK
  when(response.status()).thenReturn(RestStatus.OK);

  // response will indicate 1 search hit
  SearchHits hits = mock(SearchHits.class);
  when(hits.getTotalHits()).thenReturn(1L);

  // the response will report shard failures
  when(response.getFailedShards()).thenReturn(1);
  when(response.getTotalShards()).thenReturn(2);
  when(response.getHits()).thenReturn(hits);

  // the response will return the failures
  ShardSearchFailure[] failures = { fail };
  when(response.getShardFailures()).thenReturn(failures);

  // shard failure needs to report the node
  when(fail.shard()).thenReturn(target);

  // shard failure needs to report details of failure
  when(fail.index()).thenReturn("bro_index_2017-10-11");
  when(fail.shardId()).thenReturn(1);

  // search should succeed, even with failed shards
  ElasticsearchRequestSubmitter submitter = setup(response);
  SearchResponse actual = submitter.submitSearch(request);
  assertNotNull(actual);
}
 

void onFetchFailure(Throwable t, ShardFetchSearchRequest fetchSearchRequest, int shardIndex, SearchShardTarget shardTarget,
                    AtomicInteger counter) {
    if (logger.isDebugEnabled()) {
        logger.debug("[{}] Failed to execute fetch phase", t, fetchSearchRequest.id());
    }
    this.addShardFailure(shardIndex, shardTarget, t);
    successfulOps.decrementAndGet();
    if (counter.decrementAndGet() == 0) {
        finishHim();
    }
}
 

public CrateSearchContext createContext(
        int jobSearchContextId,
        IndexShard indexshard,
        Engine.Searcher engineSearcher,
        WhereClause whereClause) {

    ShardId shardId = indexshard.shardId();
    SearchShardTarget searchShardTarget = new SearchShardTarget(
            clusterService.state().nodes().localNodeId(),
            shardId.getIndex(),
            shardId.id()
    );
    IndexService indexService = indexshard.indexService();
    CrateSearchContext searchContext = new CrateSearchContext(
            jobSearchContextId,
            System.currentTimeMillis(),
            searchShardTarget,
            engineSearcher,
            indexService,
            indexshard,
            scriptService,
            pageCacheRecycler,
            bigArrays,
            threadPool.estimatedTimeInMillisCounter(),
            Optional.<Scroll>absent()
    );
    LuceneQueryBuilder.Context context = luceneQueryBuilder.convert(
            whereClause,  indexService.mapperService(), indexService.fieldData(), indexService.cache());
    searchContext.parsedQuery(new ParsedQuery(context.query(), EMPTY_NAMED_FILTERS));

    Float minScore = context.minScore();
    if (minScore != null) {
        searchContext.minimumScore(minScore);
    }

    return searchContext;
}
 

void onFetchFailure(Throwable t, ShardFetchSearchRequest fetchSearchRequest, int shardIndex,
                    SearchShardTarget shardTarget, AtomicInteger counter) {
    if (logger.isDebugEnabled()) {
        logger.debug("[{}] Failed to execute fetch phase", t, fetchSearchRequest.id());
    }
    this.addShardFailure(shardIndex, shardTarget, t);
    successfulOps.decrementAndGet();
    if (counter.decrementAndGet() == 0) {
        finishHim();
    }
}
 

public ShardSearchFailure(Throwable t, @Nullable SearchShardTarget shardTarget) {
    Throwable actual = ExceptionsHelper.unwrapCause(t);
    if (actual != null && actual instanceof SearchException) {
        this.shardTarget = ((SearchException) actual).shard();
    } else if (shardTarget != null) {
        this.shardTarget = shardTarget;
    }
    status = ExceptionsHelper.status(actual);
    this.reason = ExceptionsHelper.detailedMessage(t);
    this.cause = actual;
}
 

@Override
public SearchShardTarget shardTarget() {
    return shardTarget;
}
 

@Override
public SearchShardTarget shardTarget() {
    return shardTarget;
}
 

public ExportContext(long id, ShardSearchRequest request, SearchShardTarget shardTarget,
                     Engine.Searcher engineSearcher, IndexService indexService, IndexShard indexShard,
                     ScriptService scriptService, CacheRecycler cacheRecycler, String nodePath) {
    super(id, request, shardTarget, engineSearcher, indexService, indexShard, scriptService, cacheRecycler);
    this.nodePath = nodePath;
}
 

@Override
public SearchShardTarget shardTarget() {
    return queryResult.shardTarget();
}
 

@Override
protected ShardSearchIntoResponse shardOperation(ShardSearchIntoRequest
        request) throws ElasticSearchException {

    IndexService indexService = indicesService.indexServiceSafe(
            request.index());
    IndexShard indexShard = indexService.shardSafe(request.shardId());

    SearchShardTarget shardTarget = new SearchShardTarget(
            clusterService.localNode().id(), request.index(),
            request.shardId());
    SearchIntoContext context = new SearchIntoContext(0,
        new ShardSearchRequest().types(request.types()).filteringAliases(request.filteringAliases()),
        shardTarget, indexShard.searcher(), indexService, indexShard, scriptService, cacheRecycler
    );
    SearchIntoContext.setCurrent(context);

    try {
        BytesReference source = request.source();
        parser.parseSource(context, source);
        context.preProcess();
        try {
            if (context.explain()) {
                return new ShardSearchIntoResponse(
                        shardTarget.nodeIdText(), request.index(),
                        request.shardId());
            } else {
                WriterResult res = writer.execute(context);
                return new ShardSearchIntoResponse(
                        shardTarget.nodeIdText(), request.index(),
                        request.shardId(), res);
            }

        } catch (Exception e) {
            throw new QueryPhaseExecutionException(context,
                    "failed to execute inout", e);
        }
    } finally {
        // this will also release the index searcher
        context.release();
        SearchContext.removeCurrent();
    }
}
 

public ScrollQueryFetchSearchResult(QueryFetchSearchResult result, SearchShardTarget shardTarget) {
    this.result = result;
    this.shardTarget = shardTarget;
}
 

/**
 * The operation that executes the query and generates a {@link TermsByQueryShardResponse} for each shard.
 */
@Override
protected TermsByQueryShardResponse shardOperation(TermsByQueryShardRequest shardRequest) throws ElasticsearchException {
  IndexService indexService = indicesService.indexServiceSafe(shardRequest.shardId().getIndex());
  IndexShard indexShard = indexService.shardSafe(shardRequest.shardId().id());
  TermsByQueryRequest request = shardRequest.request();
  OrderByShardOperation orderByOperation = OrderByShardOperation.get(request.getOrderBy(), request.maxTermsPerShard());

  SearchShardTarget shardTarget = new SearchShardTarget(clusterService.localNode().id(),
                                                        shardRequest.shardId().getIndex(),
                                                        shardRequest.shardId().id());

  ShardSearchRequest shardSearchRequest = new ShardSearchLocalRequest(request.types(), request.nowInMillis(),
                                                                      shardRequest.filteringAliases());

  SearchContext context = new DefaultSearchContext(0, shardSearchRequest, shardTarget,
    indexShard.acquireSearcher("termsByQuery"), indexService, indexShard, scriptService,
    pageCacheRecycler, bigArrays, threadPool.estimatedTimeInMillisCounter(), parseFieldMatcher,
    SearchService.NO_TIMEOUT);
  SearchContext.setCurrent(context);

  try {
    MappedFieldType fieldType = context.smartNameFieldType(request.field());
    if (fieldType == null) {
      throw new SearchContextException(context, "[termsByQuery] field '" + request.field() +
              "' not found for types " + Arrays.toString(request.types()));
    }

    IndexFieldData indexFieldData = context.fieldData().getForField(fieldType);

    BytesReference querySource = request.querySource();
    if (querySource != null && querySource.length() > 0) {
      XContentParser queryParser = null;
      try {
        queryParser = XContentFactory.xContent(querySource).createParser(querySource);
        QueryParseContext.setTypes(request.types());
        ParsedQuery parsedQuery = orderByOperation.getParsedQuery(queryParser, indexService);
        if (parsedQuery != null) {
          context.parsedQuery(parsedQuery);
        }
      }
      finally {
        QueryParseContext.removeTypes();
        if (queryParser != null) {
          queryParser.close();
        }
      }
    }

    context.preProcess();

    // execute the search only gathering the hit count and bitset for each segment
    logger.debug("{}: Executes search for collecting terms {}", Thread.currentThread().getName(),
      shardRequest.shardId());

    TermsCollector termsCollector = this.getTermsCollector(request.termsEncoding(), indexFieldData, context);
    if (request.expectedTerms() != null) termsCollector.setExpectedTerms(request.expectedTerms());
    if (request.maxTermsPerShard() != null) termsCollector.setMaxTerms(request.maxTermsPerShard());
    HitStream hitStream = orderByOperation.getHitStream(context);
    TermsSet terms = termsCollector.collect(hitStream);

    logger.debug("{}: Returns terms response with {} terms for shard {}", Thread.currentThread().getName(),
      terms.size(), shardRequest.shardId());

    return new TermsByQueryShardResponse(shardRequest.shardId(), terms);
  }
  catch (Throwable e) {
    logger.error("[termsByQuery] Error executing shard operation", e);
    throw new QueryPhaseExecutionException(context, "[termsByQuery] Failed to execute query", e);
  }
  finally {
    // this will also release the index searcher
    context.close();
    SearchContext.removeCurrent();
  }
}
 

public SearchShardTarget shardTarget() {
    return shardTarget;
}
 

public FetchSearchResult(long id, SearchShardTarget shardTarget) {
    this.id = id;
    this.shardTarget = shardTarget;
}
 

@Override
public SearchShardTarget shardTarget() {
    return this.shardTarget;
}
 

public DfsSearchResult(long id, SearchShardTarget shardTarget) {
    this.id = id;
    this.shardTarget = shardTarget;
}
 

/**
 * The search shard target the failure occurred on.
 */
@Nullable
public SearchShardTarget shard() {
    return this.shardTarget;
}
 

public ShardSearchFailure(String reason, SearchShardTarget shardTarget, RestStatus status) {
    this.shardTarget = shardTarget;
    this.reason = reason;
    this.status = status;
}
 

public ShardSearchFailure(String reason, SearchShardTarget shardTarget) {
    this(reason, shardTarget, RestStatus.INTERNAL_SERVER_ERROR);
}
 

public IdentityHashMap<SearchShardTarget, Integer> shardHandleLookup() {
    return shardHandleLookup;
}