org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException Java Examples

/**
 * translog読み込み時にUncategorizedExecutionExceptionが発生した場合にflushが実行されること.
 * @throws Exception 実行時例外
 */
@Test
public void translogのflush時にBroadcastShardOperationFailedExceptionが発生した場合にflushのリトライをしないこと()
        throws Exception {
    Constructor<FlushTranslogRetryableRequest> constructor = FlushTranslogRetryableRequest.class
            .getDeclaredConstructor(new Class[] {EsTranslogHandler.class, Integer.TYPE, Long.TYPE });
    constructor.setAccessible(true);
    EsTranslogHandler handler = new EsTranslogHandler(RETRY_COUNT, 0, null, INDEX_FOR_TEST);
    FlushTranslogRetryableRequest flushMock = Mockito.spy((FlushTranslogRetryableRequest) constructor.newInstance(
            handler, 5, 0L));

    BroadcastShardOperationFailedException toBeThrown = Mockito.mock(BroadcastShardOperationFailedException.class);
    Mockito.doThrow(toBeThrown) // 初回リクエストの例外投入
            .doReturn(null)
            .when(flushMock)
            .doProcess();

    flushMock.doRequest();
    Mockito.verify(flushMock, Mockito.times(1)).doProcess(); // ParticularErrorのためリトライしないこと
    Mockito.verify(flushMock, Mockito.times(0)).flushTransLog();
}
 

private void finishAndNotifyListener(ActionListener listener, CopyOnWriteArrayList<ShardResponse> shardsResponses) {
    logger.trace("{}: got all shard responses", actionName);
    int successfulShards = 0;
    int failedShards = 0;
    int totalNumCopies = 0;
    List<DefaultShardOperationFailedException> shardFailures = null;
    for (int i = 0; i < shardsResponses.size(); i++) {
        ReplicationResponse shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            // non active shard, ignore
        } else {
            failedShards += shardResponse.getShardInfo().getFailed();
            successfulShards += shardResponse.getShardInfo().getSuccessful();
            totalNumCopies += shardResponse.getShardInfo().getTotal();
            if (shardFailures == null) {
                shardFailures = new ArrayList<>();
            }
            for (ReplicationResponse.ShardInfo.Failure failure : shardResponse.getShardInfo().getFailures()) {
                shardFailures.add(new DefaultShardOperationFailedException(new BroadcastShardOperationFailedException(failure.fullShardId(), failure.getCause())));
            }
        }
    }
    listener.onResponse(newResponse(successfulShards, failedShards, totalNumCopies, shardFailures));
}
 

@Override
protected ExportResponse newResponse(ExportRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
    int successfulShards = 0;
    int failedShards = 0;
    List<ShardOperationFailedException> shardFailures = null;
    List<ShardExportResponse> responses = new ArrayList<ShardExportResponse>();
    for (int i = 0; i < shardsResponses.length(); i++) {
        Object shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            failedShards++;
        } else if (shardResponse instanceof BroadcastShardOperationFailedException) {
            failedShards++;
            if (shardFailures == null) {
                shardFailures = newArrayList();
            }
            shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
        } else {
            responses.add((ShardExportResponse) shardResponse);
            successfulShards++;
        }
    }
    return new ExportResponse(responses, shardsResponses.length(), successfulShards, failedShards, shardFailures);
}
 

/**
 * translog読み込み時にUncategorizedExecutionExceptionが発生した場合にflushが実行されること.
 * @throws Exception 実行時例外
 */
@Test
public void translogのflush時にBroadcastShardOperationFailedExceptionが発生した場合にflushのリトライをしないこと()
        throws Exception {
    Constructor<FlushTranslogRetryableRequest> constructor = FlushTranslogRetryableRequest.class
            .getDeclaredConstructor(new Class[] {EsTranslogHandler.class, Integer.TYPE, Long.TYPE });
    constructor.setAccessible(true);
    EsTranslogHandler handler = new EsTranslogHandler(RETRY_COUNT, 0, null, INDEX_FOR_TEST);
    FlushTranslogRetryableRequest flushMock = Mockito.spy((FlushTranslogRetryableRequest) constructor.newInstance(
            handler, 5, 0L));

    BroadcastShardOperationFailedException toBeThrown = Mockito.mock(BroadcastShardOperationFailedException.class);
    Mockito.doThrow(toBeThrown) // 初回リクエストの例外投入
            .doReturn(null)
            .when(flushMock)
            .doProcess();

    flushMock.doRequest();
    Mockito.verify(flushMock, Mockito.times(1)).doProcess(); // ParticularErrorのためリトライしないこと
    Mockito.verify(flushMock, Mockito.times(0)).flushTransLog();
}
 

@Override
protected DfsOnlyResponse newResponse(DfsOnlyRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
    int successfulShards = 0;
    int failedShards = 0;
    List<ShardOperationFailedException> shardFailures = null;
    AtomicArray<DfsSearchResult> dfsResults = new AtomicArray<>(shardsResponses.length());
    for (int i = 0; i < shardsResponses.length(); i++) {
        Object shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            // simply ignore non active shards
        } else if (shardResponse instanceof BroadcastShardOperationFailedException) {
            failedShards++;
            if (shardFailures == null) {
                shardFailures = new ArrayList<>();
            }
            shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
        } else {
            dfsResults.set(i, ((ShardDfsOnlyResponse) shardResponse).getDfsSearchResult());
            successfulShards++;
        }
    }
    AggregatedDfs dfs = searchPhaseController.aggregateDfs(dfsResults);
    return new DfsOnlyResponse(dfs, shardsResponses.length(), successfulShards, failedShards, shardFailures, buildTookInMillis(request));
}
 

private void finishAndNotifyListener(ActionListener listener, CopyOnWriteArrayList<ShardResponse> shardsResponses) {
    logger.trace("{}: got all shard responses", actionName);
    int successfulShards = 0;
    int failedShards = 0;
    int totalNumCopies = 0;
    List<ShardOperationFailedException> shardFailures = null;
    for (int i = 0; i < shardsResponses.size(); i++) {
        ActionWriteResponse shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            // non active shard, ignore
        } else {
            failedShards += shardResponse.getShardInfo().getFailed();
            successfulShards += shardResponse.getShardInfo().getSuccessful();
            totalNumCopies += shardResponse.getShardInfo().getTotal();
            if (shardFailures == null) {
                shardFailures = new ArrayList<>();
            }
            for (ActionWriteResponse.ShardInfo.Failure failure : shardResponse.getShardInfo().getFailures()) {
                shardFailures.add(new DefaultShardOperationFailedException(new BroadcastShardOperationFailedException(new ShardId(failure.index(), failure.shardId()), failure.getCause())));
            }
        }
    }
    listener.onResponse(newResponse(successfulShards, failedShards, totalNumCopies, shardFailures));
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    nodeId = in.readString();
    totalShards = in.readVInt();
    int resultsSize = in.readVInt();
    results = new ArrayList<>(resultsSize);
    for (; resultsSize > 0; resultsSize--) {
        final ShardOperationResult result = in.readBoolean() ? readShardResult(in) : null;
        results.add(result);
    }
    if (in.readBoolean()) {
        int failureShards = in.readVInt();
        exceptions = new ArrayList<>(failureShards);
        for (int i = 0; i < failureShards; i++) {
            exceptions.add(new BroadcastShardOperationFailedException(in));
        }
    } else {
        exceptions = null;
    }
}
 

private void onShardOperation(final NodeRequest request, final Object[] shardResults, final int shardIndex, final ShardRouting shardRouting) {
    try {
        if (logger.isTraceEnabled()) {
            logger.trace("[{}]  executing operation for shard [{}]", actionName, shardRouting.shortSummary());
        }
        ShardOperationResult result = shardOperation(request.indicesLevelRequest, shardRouting);
        shardResults[shardIndex] = result;
        if (logger.isTraceEnabled()) {
            logger.trace("[{}]  completed operation for shard [{}]", actionName, shardRouting.shortSummary());
        }
    } catch (Throwable t) {
        BroadcastShardOperationFailedException e = new BroadcastShardOperationFailedException(shardRouting.shardId(), "operation " + actionName + " failed", t);
        e.setIndex(shardRouting.getIndex());
        e.setShard(shardRouting.shardId());
        shardResults[shardIndex] = e;
        if (TransportActions.isShardNotAvailableException(t)) {
            if (logger.isTraceEnabled()) {
                logger.trace("[{}] failed to execute operation for shard [{}]", t, actionName, shardRouting.shortSummary());
            }
        } else {
            if (logger.isDebugEnabled()) {
                logger.debug("[{}] failed to execute operation for shard [{}]", t, actionName, shardRouting.shortSummary());
            }
        }
    }
}
 

@Override
protected SearchIntoResponse newResponse(SearchIntoRequest request,
        AtomicReferenceArray shardsResponses, ClusterState clusterState) {
    int successfulShards = 0;
    int failedShards = 0;
    List<ShardOperationFailedException> shardFailures = null;
    List<ShardSearchIntoResponse> responses = new
            ArrayList<ShardSearchIntoResponse>();
    for (int i = 0; i < shardsResponses.length(); i++) {
        Object shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            failedShards++;
        } else if (shardResponse instanceof
                BroadcastShardOperationFailedException) {
            failedShards++;
            if (shardFailures == null) {
                shardFailures = newArrayList();
            }
            shardFailures.add(new DefaultShardOperationFailedException(
                    (BroadcastShardOperationFailedException)
                            shardResponse));
        } else {
            responses.add((ShardSearchIntoResponse) shardResponse);
            successfulShards++;
        }
    }
    return new SearchIntoResponse(responses, shardsResponses.length(),
            successfulShards, failedShards, shardFailures);
}
 

@Override
protected ValidateQueryResponse newResponse(ValidateQueryRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
    int successfulShards = 0;
    int failedShards = 0;
    boolean valid = true;
    List<ShardOperationFailedException> shardFailures = null;
    List<QueryExplanation> queryExplanations = null;
    for (int i = 0; i < shardsResponses.length(); i++) {
        Object shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            // simply ignore non active shards
        } else if (shardResponse instanceof BroadcastShardOperationFailedException) {
            failedShards++;
            if (shardFailures == null) {
                shardFailures = new ArrayList<>();
            }
            shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
        } else {
            ShardValidateQueryResponse validateQueryResponse = (ShardValidateQueryResponse) shardResponse;
            valid = valid && validateQueryResponse.isValid();
            if (request.explain() || request.rewrite()) {
                if (queryExplanations == null) {
                    queryExplanations = new ArrayList<>();
                }
                queryExplanations.add(new QueryExplanation(
                        validateQueryResponse.getIndex(),
                        validateQueryResponse.isValid(),
                        validateQueryResponse.getExplanation(),
                        validateQueryResponse.getError()
                ));
            }
            successfulShards++;
        }
    }
    return new ValidateQueryResponse(valid, queryExplanations, shardsResponses.length(), successfulShards, failedShards, shardFailures);
}
 

public NodeResponse(StreamInput in) throws IOException {
    nodeId = in.readString();
    totalShards = in.readVInt();
    results = in.readList((stream) -> stream.readBoolean() ? readShardResult(stream) : null);
    if (in.readBoolean()) {
        exceptions = in.readList(BroadcastShardOperationFailedException::new);
    } else {
        exceptions = null;
    }
}
 

NodeResponse(String nodeId,
                    int totalShards,
                    List<ShardOperationResult> results,
                    List<BroadcastShardOperationFailedException> exceptions) {
    this.nodeId = nodeId;
    this.totalShards = totalShards;
    this.results = results;
    this.exceptions = exceptions;
}
 

private void onShardOperation(final NodeRequest request, final Object[] shardResults, final int shardIndex, final ShardRouting shardRouting) {
    try {
        if (logger.isTraceEnabled()) {
            logger.trace("[{}]  executing operation for shard [{}]", actionName, shardRouting.shortSummary());
        }
        ShardOperationResult result = shardOperation(request.indicesLevelRequest, shardRouting);
        shardResults[shardIndex] = result;
        if (logger.isTraceEnabled()) {
            logger.trace("[{}]  completed operation for shard [{}]", actionName, shardRouting.shortSummary());
        }
    } catch (Exception e) {
        BroadcastShardOperationFailedException failure =
            new BroadcastShardOperationFailedException(shardRouting.shardId(), "operation " + actionName + " failed", e);
        failure.setShard(shardRouting.shardId());
        shardResults[shardIndex] = failure;
        if (TransportActions.isShardNotAvailableException(e)) {
            if (logger.isTraceEnabled()) {
                logger.trace(new ParameterizedMessage(
                    "[{}] failed to execute operation for shard [{}]", actionName, shardRouting.shortSummary()), e);
            }
        } else {
            if (logger.isDebugEnabled()) {
                logger.debug(new ParameterizedMessage(
                    "[{}] failed to execute operation for shard [{}]", actionName, shardRouting.shortSummary()), e);
            }
        }
    }
}
 

@Override
public void messageReceived(final NodeRequest request, TransportChannel channel, Task task) throws Exception {
    List<ShardRouting> shards = request.getShards();
    final int totalShards = shards.size();
    if (logger.isTraceEnabled()) {
        logger.trace("[{}] executing operation on [{}] shards", actionName, totalShards);
    }
    final Object[] shardResultOrExceptions = new Object[totalShards];

    int shardIndex = -1;
    for (final ShardRouting shardRouting : shards) {
        shardIndex++;
        onShardOperation(request, shardResultOrExceptions, shardIndex, shardRouting);
    }

    List<BroadcastShardOperationFailedException> accumulatedExceptions = new ArrayList<>();
    List<ShardOperationResult> results = new ArrayList<>();
    for (int i = 0; i < totalShards; i++) {
        if (shardResultOrExceptions[i] instanceof BroadcastShardOperationFailedException) {
            accumulatedExceptions.add((BroadcastShardOperationFailedException) shardResultOrExceptions[i]);
        } else {
            results.add((ShardOperationResult) shardResultOrExceptions[i]);
        }
    }

    channel.sendResponse(new NodeResponse(request.getNodeId(), totalShards, results, accumulatedExceptions));
}
 

private Response newResponse(
        Request request,
        AtomicReferenceArray responses,
        List<NoShardAvailableActionException> unavailableShardExceptions,
        Map<String, List<ShardRouting>> nodes,
        ClusterState clusterState) {
    int totalShards = 0;
    int successfulShards = 0;
    List<ShardOperationResult> broadcastByNodeResponses = new ArrayList<>();
    List<DefaultShardOperationFailedException> exceptions = new ArrayList<>();
    for (int i = 0; i < responses.length(); i++) {
        if (responses.get(i) instanceof FailedNodeException) {
            FailedNodeException exception = (FailedNodeException) responses.get(i);
            totalShards += nodes.get(exception.nodeId()).size();
            for (ShardRouting shard : nodes.get(exception.nodeId())) {
                exceptions.add(new DefaultShardOperationFailedException(shard.getIndexName(), shard.getId(), exception));
            }
        } else {
            NodeResponse response = (NodeResponse) responses.get(i);
            broadcastByNodeResponses.addAll(response.results);
            totalShards += response.getTotalShards();
            successfulShards += response.getSuccessfulShards();
            for (BroadcastShardOperationFailedException throwable : response.getExceptions()) {
                if (!TransportActions.isShardNotAvailableException(throwable)) {
                    exceptions.add(new DefaultShardOperationFailedException(throwable.getShardId().getIndexName(), throwable.getShardId().getId(), throwable));
                }
            }
        }
    }
    totalShards += unavailableShardExceptions.size();
    int failedShards = exceptions.size();
    return newResponse(request, totalShards, successfulShards, failedShards, broadcastByNodeResponses, exceptions, clusterState);
}
 

private final Response newResponse(
        Request request,
        AtomicReferenceArray responses,
        List<NoShardAvailableActionException> unavailableShardExceptions,
        Map<String, List<ShardRouting>> nodes,
        ClusterState clusterState) {
    int totalShards = 0;
    int successfulShards = 0;
    List<ShardOperationResult> broadcastByNodeResponses = new ArrayList<>();
    List<ShardOperationFailedException> exceptions = new ArrayList<>();
    for (int i = 0; i < responses.length(); i++) {
        if (responses.get(i) instanceof FailedNodeException) {
            FailedNodeException exception = (FailedNodeException) responses.get(i);
            totalShards += nodes.get(exception.nodeId()).size();
            for (ShardRouting shard : nodes.get(exception.nodeId())) {
                exceptions.add(new DefaultShardOperationFailedException(shard.getIndex(), shard.getId(), exception));
            }
        } else {
            NodeResponse response = (NodeResponse) responses.get(i);
            broadcastByNodeResponses.addAll(response.results);
            totalShards += response.getTotalShards();
            successfulShards += response.getSuccessfulShards();
            for (BroadcastShardOperationFailedException throwable : response.getExceptions()) {
                if (!TransportActions.isShardNotAvailableException(throwable)) {
                    exceptions.add(new DefaultShardOperationFailedException(throwable.getIndex(), throwable.getShardId().getId(), throwable));
                }
            }
        }
    }
    totalShards += unavailableShardExceptions.size();
    int failedShards = exceptions.size();
    return newResponse(request, totalShards, successfulShards, failedShards, broadcastByNodeResponses, exceptions, clusterState);
}
 

@Override
public void messageReceived(final NodeRequest request, TransportChannel channel) throws Exception {
    List<ShardRouting> shards = request.getShards();
    final int totalShards = shards.size();
    if (logger.isTraceEnabled()) {
        logger.trace("[{}] executing operation on [{}] shards", actionName, totalShards);
    }
    final Object[] shardResultOrExceptions = new Object[totalShards];

    int shardIndex = -1;
    for (final ShardRouting shardRouting : shards) {
        shardIndex++;
        onShardOperation(request, shardResultOrExceptions, shardIndex, shardRouting);
    }

    List<BroadcastShardOperationFailedException> accumulatedExceptions = new ArrayList<>();
    List<ShardOperationResult> results = new ArrayList<>();
    for (int i = 0; i < totalShards; i++) {
        if (shardResultOrExceptions[i] instanceof BroadcastShardOperationFailedException) {
            accumulatedExceptions.add((BroadcastShardOperationFailedException) shardResultOrExceptions[i]);
        } else {
            results.add((ShardOperationResult) shardResultOrExceptions[i]);
        }
    }

    channel.sendResponse(new NodeResponse(request.getNodeId(), totalShards, results, accumulatedExceptions));
}
 

@Override
protected ExistsResponse newResponse(ExistsRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
    int successfulShards = 0;
    int failedShards = 0;
    boolean exists = false;
    List<ShardOperationFailedException> shardFailures = null;

    // if docs do exist, the last response will have exists = true (since we early terminate the shard requests)
    for (int i = shardsResponses.length() - 1; i >= 0 ; i--) {
        Object shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            // simply ignore non active shards
        } else if (shardResponse instanceof BroadcastShardOperationFailedException) {
            failedShards++;
            if (shardFailures == null) {
                shardFailures = new ArrayList<>();
            }
            shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
        } else {
            successfulShards++;
            if ((exists = ((ShardExistsResponse) shardResponse).exists())) {
                successfulShards = shardsResponses.length() - failedShards;
                break;
            }
        }
    }
    return new ExistsResponse(exists, shardsResponses.length(), successfulShards, failedShards, shardFailures);
}
 

@Override
protected SuggestResponse newResponse(SuggestRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) {
    int successfulShards = 0;
    int failedShards = 0;

    final Map<String, List<Suggest.Suggestion>> groupedSuggestions = new HashMap<>();

    List<ShardOperationFailedException> shardFailures = null;
    for (int i = 0; i < shardsResponses.length(); i++) {
        Object shardResponse = shardsResponses.get(i);
        if (shardResponse == null) {
            // simply ignore non active shards
        } else if (shardResponse instanceof BroadcastShardOperationFailedException) {
            failedShards++;
            if (shardFailures == null) {
                shardFailures = new ArrayList<>();
            }
            shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse));
        } else {
            Suggest suggest = ((ShardSuggestResponse) shardResponse).getSuggest();
            Suggest.group(groupedSuggestions, suggest);
            successfulShards++;
        }
    }

    return new SuggestResponse(new Suggest(Suggest.reduce(groupedSuggestions)), shardsResponses.length(), successfulShards, failedShards, shardFailures);
}
 

public NodeResponse(String nodeId,
                    int totalShards,
                    List<ShardOperationResult> results,
                    List<BroadcastShardOperationFailedException> exceptions) {
    this.nodeId = nodeId;
    this.totalShards = totalShards;
    this.results = results;
    this.exceptions = exceptions;
}
 

public List<BroadcastShardOperationFailedException> getExceptions() {
    return exceptions;
}
 

public List<BroadcastShardOperationFailedException> getExceptions() {
    return exceptions;
}
 

/**
 * flushの失敗時のエラーの場合分け.< br/>
 * 以下の場合はすでに他のスレッドからflushが依頼されたとみなし、リトライは行わない。< br/>
 * <ul>
 * <li>BroadcastShardOperationFailedException</li>
 * <li>FlushNotAllowedEngineException</li>
 * </ul>
 * @param e 検査対象の例外
 * @return true: 正常終了として扱う場合, false: 左記以外の場合
 */
@Override
boolean isParticularError(ElasticsearchException e) {
    return e instanceof BroadcastShardOperationFailedException
            || e instanceof FlushNotAllowedEngineException;
}
 

/**
 * flushの失敗時のエラーの場合分け.< br/>
 * 以下の場合はすでに他のスレッドからflushが依頼されたとみなし、リトライは行わない。< br/>
 * <ul>
 * <li>BroadcastShardOperationFailedException</li>
 * <li>FlushNotAllowedEngineException</li>
 * </ul>
 * @param e 検査対象の例外
 * @return true: 正常終了として扱う場合, false: 左記以外の場合
 */
@Override
boolean isParticularError(ElasticsearchException e) {
    return e instanceof BroadcastShardOperationFailedException
            || e instanceof FlushNotAllowedEngineException;
}