org.elasticsearch.action.FailedNodeException Java Examples

@Override
public void run() {
    DiscoveryNode[] dataNodes = nodeFilter.getEligibleDataNodes();

    // we also add the cancel query function here based on query text from the negative cache.

    CronRequest modelDeleteRequest = new CronRequest(dataNodes);
    client.execute(CronAction.INSTANCE, modelDeleteRequest, ActionListener.wrap(response -> {
        if (response.hasFailures()) {
            for (FailedNodeException failedNodeException : response.failures()) {
                LOG.warn(NODE_EXCEPTION_LOG_MSG, failedNodeException);
            }
        } else {
            LOG.info(SUCCEEDS_LOG_MSG);
        }
    }, exception -> { LOG.error(EXCEPTION_LOG_MSG, exception); }));
}
 

@Override
protected NodesStoreFilesMetaData newResponse(Request request, AtomicReferenceArray responses) {
    final List<NodeStoreFilesMetaData> nodeStoreFilesMetaDatas = new ArrayList<>();
    final List<FailedNodeException> failures = new ArrayList<>();
    for (int i = 0; i < responses.length(); i++) {
        Object resp = responses.get(i);
        if (resp instanceof NodeStoreFilesMetaData) { // will also filter out null response for unallocated ones
            nodeStoreFilesMetaDatas.add((NodeStoreFilesMetaData) resp);
        } else if (resp instanceof FailedNodeException) {
            failures.add((FailedNodeException) resp);
        } else {
            logger.warn("unknown response type [{}], expected NodeStoreFilesMetaData or FailedNodeException", resp);
        }
    }
    return new NodesStoreFilesMetaData(clusterName, nodeStoreFilesMetaDatas.toArray(new NodeStoreFilesMetaData[nodeStoreFilesMetaDatas.size()]),
            failures.toArray(new FailedNodeException[failures.size()]));
}
 

@Test
public void testToXContent() throws IOException {
    ADStatsResponse adStatsResponse = new ADStatsResponse();
    Map<String, Object> testClusterStats = new HashMap<>();
    testClusterStats.put("test_stat", 1);
    adStatsResponse.setClusterStats(testClusterStats);
    List<ADStatsNodeResponse> responses = Collections.emptyList();
    List<FailedNodeException> failures = Collections.emptyList();
    ADStatsNodesResponse adStatsNodesResponse = new ADStatsNodesResponse(ClusterName.DEFAULT, responses, failures);
    adStatsResponse.setADStatsNodesResponse(adStatsNodesResponse);

    XContentBuilder builder = XContentFactory.jsonBuilder();
    adStatsResponse.toXContent(builder);
    XContentParser parser = createParser(builder);
    assertEquals(1, parser.map().get("test_stat"));
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int responsesCount = in.readInt();
    this.responses = new ArrayList<NodeImportResponse>(responsesCount);
    for (int i = 0; i < responsesCount; i++) {
        responses.add(NodeImportResponse.readNew(in));
    }
    int failuresCount = in.readInt();
    this.nodeFailures = new ArrayList<FailedNodeException>(failuresCount);
    for (int i = 0; i < failuresCount; i++) {
        String nodeId = in.readString();
        String msg = in.readOptionalString();
        FailedNodeException e = new FailedNodeException(nodeId, msg, null);
        nodeFailures.add(e);
    }
}
 

@Override
protected NodesGatewayStartedShards newResponse(Request request, AtomicReferenceArray responses) {
    final List<NodeGatewayStartedShards> nodesList = new ArrayList<>();
    final List<FailedNodeException> failures = new ArrayList<>();
    for (int i = 0; i < responses.length(); i++) {
        Object resp = responses.get(i);
        if (resp instanceof NodeGatewayStartedShards) { // will also filter out null response for unallocated ones
            nodesList.add((NodeGatewayStartedShards) resp);
        } else if (resp instanceof FailedNodeException) {
            failures.add((FailedNodeException) resp);
        } else {
            logger.warn("unknown response type [{}], expected NodeLocalGatewayStartedShards or FailedNodeException", resp);
        }
    }
    return new NodesGatewayStartedShards(clusterName, nodesList.toArray(new NodeGatewayStartedShards[nodesList.size()]),
            failures.toArray(new FailedNodeException[failures.size()]));
}
 

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params)
        throws IOException {
    builder.startObject();
    builder.startArray("imports");
    for (NodeImportResponse r : this.responses) {
        r.toXContent(builder, params);
    }
    builder.endArray();
    if (nodeFailures != null && nodeFailures.size() > 0) {
        builder.startArray("failures");
        for (FailedNodeException failure : nodeFailures) {
            builder.startObject();
            builder.field("node_id", failure.nodeId());
            builder.field("reason", failure.getDetailedMessage());
            builder.endObject();
        }
        builder.endArray();
    }
    builder.endObject();
    return builder;
}
 

@Override
protected NodesSnapshotStatus newResponse(Request request, AtomicReferenceArray responses) {
    final List<NodeSnapshotStatus> nodesList = new ArrayList<>();
    final List<FailedNodeException> failures = new ArrayList<>();
    for (int i = 0; i < responses.length(); i++) {
        Object resp = responses.get(i);
        if (resp instanceof NodeSnapshotStatus) { // will also filter out null response for unallocated ones
            nodesList.add((NodeSnapshotStatus) resp);
        } else if (resp instanceof FailedNodeException) {
            failures.add((FailedNodeException) resp);
        } else {
            logger.warn("unknown response type [{}], expected NodeSnapshotStatus or FailedNodeException", resp);
        }
    }
    return new NodesSnapshotStatus(clusterName, nodesList.toArray(new NodeSnapshotStatus[nodesList.size()]),
            failures.toArray(new FailedNodeException[failures.size()]));
}
 

@Override
protected NodesGatewayMetaState newResponse(Request request, AtomicReferenceArray responses) {
    final List<NodeGatewayMetaState> nodesList = new ArrayList<>();
    final List<FailedNodeException> failures = new ArrayList<>();
    for (int i = 0; i < responses.length(); i++) {
        Object resp = responses.get(i);
        if (resp instanceof NodeGatewayMetaState) { // will also filter out null response for unallocated ones
            nodesList.add((NodeGatewayMetaState) resp);
        } else if (resp instanceof FailedNodeException) {
            failures.add((FailedNodeException) resp);
        } else {
            logger.warn("unknown response type [{}], expected NodeLocalGatewayMetaState or FailedNodeException", resp);
        }
    }
    return new NodesGatewayMetaState(clusterName, nodesList.toArray(new NodeGatewayMetaState[nodesList.size()]),
            failures.toArray(new FailedNodeException[failures.size()]));
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    List<TaskOperationFailure> taskFailures = new ArrayList<>();
    for (int i = 0; i < size; i++) {
        taskFailures.add(new TaskOperationFailure(in));
    }
    size = in.readVInt();
    this.taskFailures = Collections.unmodifiableList(taskFailures);
    List<FailedNodeException> nodeFailures = new ArrayList<>();
    for (int i = 0; i < size; i++) {
        nodeFailures.add(new FailedNodeException(in));
    }
    this.nodeFailures = Collections.unmodifiableList(nodeFailures);
}
 

/**
 * Constructor
 *
 * @param clusterName name of cluster
 * @param nodes List of ProfileNodeResponse from nodes
 * @param failures List of failures from nodes
 */
public ProfileResponse(ClusterName clusterName, List<ProfileNodeResponse> nodes, List<FailedNodeException> failures) {
    super(clusterName, nodes, failures);
    totalSizeInBytes = 0L;
    List<ModelProfile> modelProfileList = new ArrayList<>();
    for (ProfileNodeResponse response : nodes) {
        String curNodeId = response.getNode().getId();
        if (response.getShingleSize() >= 0) {
            coordinatingNode = curNodeId;
            shingleSize = response.getShingleSize();
        }
        for (Map.Entry<String, Long> entry : response.getModelSize().entrySet()) {
            totalSizeInBytes += entry.getValue();
            modelProfileList.add(new ModelProfile(entry.getKey(), entry.getValue(), curNodeId));
        }

    }
    if (coordinatingNode == null) {
        coordinatingNode = "";
    }
    this.modelProfile = modelProfileList.toArray(new ModelProfile[0]);
}
 

/**
 * Map the responses into {@code nodeResponseClass} responses and {@link FailedNodeException}s.
 *
 * @param request The associated request.
 * @param nodesResponses All node-level responses
 * @return Never {@code null}.
 * @throws NullPointerException if {@code nodesResponses} is {@code null}
 * @see #newResponse(BaseNodesRequest, List, List)
 */
protected NodesResponse newResponse(NodesRequest request, AtomicReferenceArray nodesResponses) {
    final List<NodeResponse> responses = new ArrayList<>();
    final List<FailedNodeException> failures = new ArrayList<>();

    for (int i = 0; i < nodesResponses.length(); ++i) {
        Object response = nodesResponses.get(i);

        if (response instanceof FailedNodeException) {
            failures.add((FailedNodeException)response);
        } else {
            responses.add(nodeResponseClass.cast(response));
        }
    }

    return newResponse(request, responses, failures);
}
 

@SuppressWarnings("unchecked")
protected TasksResponse newResponse(TasksRequest request, AtomicReferenceArray responses) {
    List<TaskResponse> tasks = new ArrayList<>();
    List<FailedNodeException> failedNodeExceptions = new ArrayList<>();
    List<TaskOperationFailure> taskOperationFailures = new ArrayList<>();
    for (int i = 0; i < responses.length(); i++) {
        Object response = responses.get(i);
        if (response instanceof FailedNodeException) {
            failedNodeExceptions.add((FailedNodeException) response);
        } else {
            NodeTasksResponse tasksResponse = (NodeTasksResponse) response;
            if (tasksResponse.results != null) {
                tasks.addAll(tasksResponse.results);
            }
            if (tasksResponse.exceptions != null) {
                taskOperationFailures.addAll(tasksResponse.exceptions);
            }
        }
    }
    return newResponse(request, tasks, taskOperationFailures, failedNodeExceptions);
}
 

@Override
protected ADStatsNodesResponse newResponse(
    ADStatsRequest request,
    List<ADStatsNodeResponse> responses,
    List<FailedNodeException> failures
) {
    return new ADStatsNodesResponse(clusterService.getClusterName(), responses, failures);
}
 

private Response newResponse(
        Request request,
        AtomicReferenceArray responses,
        List<NoShardAvailableActionException> unavailableShardExceptions,
        Map<String, List<ShardRouting>> nodes,
        ClusterState clusterState) {
    int totalShards = 0;
    int successfulShards = 0;
    List<ShardOperationResult> broadcastByNodeResponses = new ArrayList<>();
    List<DefaultShardOperationFailedException> exceptions = new ArrayList<>();
    for (int i = 0; i < responses.length(); i++) {
        if (responses.get(i) instanceof FailedNodeException) {
            FailedNodeException exception = (FailedNodeException) responses.get(i);
            totalShards += nodes.get(exception.nodeId()).size();
            for (ShardRouting shard : nodes.get(exception.nodeId())) {
                exceptions.add(new DefaultShardOperationFailedException(shard.getIndexName(), shard.getId(), exception));
            }
        } else {
            NodeResponse response = (NodeResponse) responses.get(i);
            broadcastByNodeResponses.addAll(response.results);
            totalShards += response.getTotalShards();
            successfulShards += response.getSuccessfulShards();
            for (BroadcastShardOperationFailedException throwable : response.getExceptions()) {
                if (!TransportActions.isShardNotAvailableException(throwable)) {
                    exceptions.add(new DefaultShardOperationFailedException(throwable.getShardId().getIndexName(), throwable.getShardId().getId(), throwable));
                }
            }
        }
    }
    totalShards += unavailableShardExceptions.size();
    int failedShards = exceptions.size();
    return newResponse(request, totalShards, successfulShards, failedShards, broadcastByNodeResponses, exceptions, clusterState);
}
 

@Test
public void testMerge() {
    ADStatsResponse adStatsResponse1 = new ADStatsResponse();
    Map<String, Object> testClusterStats = new HashMap<>();
    testClusterStats.put("test_stat", 1L);
    adStatsResponse1.setClusterStats(testClusterStats);

    ADStatsResponse adStatsResponse2 = new ADStatsResponse();
    List<ADStatsNodeResponse> responses = Collections.emptyList();
    List<FailedNodeException> failures = Collections.emptyList();
    ADStatsNodesResponse adStatsNodesResponse = new ADStatsNodesResponse(ClusterName.DEFAULT, responses, failures);
    adStatsResponse2.setADStatsNodesResponse(adStatsNodesResponse);

    adStatsResponse1.merge(adStatsResponse2);
    assertEquals(testClusterStats, adStatsResponse1.getClusterStats());
    assertEquals(adStatsNodesResponse, adStatsResponse1.getADStatsNodesResponse());

    adStatsResponse2.merge(adStatsResponse1);
    assertEquals(testClusterStats, adStatsResponse2.getClusterStats());
    assertEquals(adStatsNodesResponse, adStatsResponse2.getADStatsNodesResponse());

    // Confirm merging with null does nothing
    adStatsResponse1.merge(null);
    assertEquals(testClusterStats, adStatsResponse1.getClusterStats());
    assertEquals(adStatsNodesResponse, adStatsResponse1.getADStatsNodesResponse());

    // Confirm merging with self does nothing
    adStatsResponse1.merge(adStatsResponse1);
    assertEquals(testClusterStats, adStatsResponse1.getClusterStats());
    assertEquals(adStatsNodesResponse, adStatsResponse1.getADStatsNodesResponse());
}
 

@Test
public void testGetAndSetADStatsNodesResponse() {
    ADStatsResponse adStatsResponse = new ADStatsResponse();
    List<ADStatsNodeResponse> responses = Collections.emptyList();
    List<FailedNodeException> failures = Collections.emptyList();
    ADStatsNodesResponse adStatsNodesResponse = new ADStatsNodesResponse(ClusterName.DEFAULT, responses, failures);
    adStatsResponse.setADStatsNodesResponse(adStatsNodesResponse);
    assertEquals(adStatsNodesResponse, adStatsResponse.getADStatsNodesResponse());
}
 

protected void onNodeFailure(DiscoveryNode node, int nodeIndex, Throwable t) {
    String nodeId = node.getId();
    if (logger.isDebugEnabled() && !(t instanceof NodeShouldNotConnectException)) {
        logger.debug(new ParameterizedMessage("failed to execute [{}] on node [{}]", actionName, nodeId), t);
    }

    // this is defensive to protect against the possibility of double invocation
    // the current implementation of TransportService#sendRequest guards against this
    // but concurrency is hard, safety is important, and the small performance loss here does not matter
    if (responses.compareAndSet(nodeIndex, null, new FailedNodeException(nodeId, "Failed node [" + nodeId + "]", t))) {
        if (counter.incrementAndGet() == responses.length()) {
            onCompletion();
        }
    }
}
 

@Test
public void testNewResponse() {
    DiscoveryNode node = clusterService().localNode();
    ProfileRequest profileRequest = new ProfileRequest(detectorId, profilesToRetrieve, node);

    ProfileNodeResponse profileNodeResponse1 = new ProfileNodeResponse(node, new HashMap<>(), shingleSize);
    List<ProfileNodeResponse> profileNodeResponses = Arrays.asList(profileNodeResponse1);
    List<FailedNodeException> failures = new ArrayList<>();

    ProfileResponse profileResponse = action.newResponse(profileRequest, profileNodeResponses, failures);
    assertEquals(node.getId(), profileResponse.getCoordinatingNode());
}
 

@Override
protected ImportResponse newResponse(ImportRequest request,
        AtomicReferenceArray nodesResponses) {
    int total = nodesResponses.length();
    int successfulNodes = 0;
    int failedNodes = 0;
    List<FailedNodeException> nodeFailures = null;
    List<NodeImportResponse> responses = new ArrayList<NodeImportResponse>();
    for (int i=0; i < total; i++) {
        Object nodeResponse = nodesResponses.get(i);
        if (nodeResponse == null) {
            failedNodes++;
        } else if (nodeResponse instanceof FailedNodeException) {
            failedNodes++;
            if (nodeFailures == null) {
                nodeFailures = newArrayList();
            }
            nodeFailures.add((FailedNodeException) nodeResponse);
        } else if (nodeResponse instanceof Exception) {
            ((Exception) nodeResponse).getMessage();
        } else {
            responses.add((NodeImportResponse) nodeResponse);
            successfulNodes++;
        }
    }
    return new ImportResponse(responses, total, successfulNodes, failedNodes, nodeFailures);
}
 

private void onFailure(int idx, String nodeId, Throwable t) {
    if (logger.isDebugEnabled() && !(t instanceof NodeShouldNotConnectException)) {
        logger.debug("failed to execute on node [{}]", t, nodeId);
    }
    if (accumulateExceptions()) {
        responses.set(idx, new FailedNodeException(nodeId, "Failed node [" + nodeId + "]", t));
    }
    if (counter.incrementAndGet() == responses.length()) {
        finishHim();
    }
}
 

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeInt(responses.size());
    for (NodeImportResponse response : responses) {
        response.writeTo(out);
    }
    out.writeInt(nodeFailures.size());
    for (FailedNodeException e : nodeFailures) {
        out.writeString(e.nodeId());
        out.writeOptionalString(e.getMessage());
    }
}
 

@Override
protected synchronized void processAsyncFetch(ShardId shardId, NodeGatewayStartedShards[] responses, FailedNodeException[] failures) {
    fetchResponses.add(new Response(shardId, responses, failures));
    if (expectedOps.countDown()) {
        finish();
    }
}
 

void finish() {
    ImmutableOpenMap.Builder<String, ImmutableOpenIntMap<java.util.List<IndicesShardStoresResponse.StoreStatus>>> indicesStoreStatusesBuilder = ImmutableOpenMap.builder();
    java.util.List<IndicesShardStoresResponse.Failure> failureBuilder = new ArrayList<>();
    for (Response fetchResponse : fetchResponses) {
        ImmutableOpenIntMap<java.util.List<IndicesShardStoresResponse.StoreStatus>> indexStoreStatuses = indicesStoreStatusesBuilder.get(fetchResponse.shardId.getIndex());
        final ImmutableOpenIntMap.Builder<java.util.List<IndicesShardStoresResponse.StoreStatus>> indexShardsBuilder;
        if (indexStoreStatuses == null) {
            indexShardsBuilder = ImmutableOpenIntMap.builder();
        } else {
            indexShardsBuilder = ImmutableOpenIntMap.builder(indexStoreStatuses);
        }
        java.util.List<IndicesShardStoresResponse.StoreStatus> storeStatuses = indexShardsBuilder.get(fetchResponse.shardId.id());
        if (storeStatuses == null) {
            storeStatuses = new ArrayList<>();
        }
        for (NodeGatewayStartedShards response : fetchResponse.responses) {
            if (shardExistsInNode(response)) {
                IndicesShardStoresResponse.StoreStatus.Allocation allocation = getAllocation(fetchResponse.shardId.getIndex(), fetchResponse.shardId.id(), response.getNode());
                storeStatuses.add(new IndicesShardStoresResponse.StoreStatus(response.getNode(), response.version(), allocation, response.storeException()));
            }
        }
        CollectionUtil.timSort(storeStatuses);
        indexShardsBuilder.put(fetchResponse.shardId.id(), storeStatuses);
        indicesStoreStatusesBuilder.put(fetchResponse.shardId.getIndex(), indexShardsBuilder.build());
        for (FailedNodeException failure : fetchResponse.failures) {
            failureBuilder.add(new IndicesShardStoresResponse.Failure(failure.nodeId(), fetchResponse.shardId.getIndex(), fetchResponse.shardId.id(), failure.getCause()));
        }
    }
    listener.onResponse(new IndicesShardStoresResponse(indicesStoreStatusesBuilder.build(), Collections.unmodifiableList(failureBuilder)));
}
 

@Override
protected void doExecute(Task task, ActionRequest actionRequest, ActionListener<StopDetectorResponse> listener) {
    StopDetectorRequest request = StopDetectorRequest.fromActionRequest(actionRequest);
    String adID = request.getAdID();
    try {
        DiscoveryNode[] dataNodes = nodeFilter.getEligibleDataNodes();
        DeleteModelRequest modelDeleteRequest = new DeleteModelRequest(adID, dataNodes);
        client.execute(DeleteModelAction.INSTANCE, modelDeleteRequest, ActionListener.wrap(response -> {
            if (response.hasFailures()) {
                LOG.warn("Cannot delete all models of detector {}", adID);
                for (FailedNodeException failedNodeException : response.failures()) {
                    LOG.warn("Deleting models of node has exception", failedNodeException);
                }
                // if customers are using an updated detector and we haven't deleted old
                // checkpoints, customer would have trouble
                listener.onResponse(new StopDetectorResponse(false));
            } else {
                LOG.info("models of detector {} get deleted", adID);
                listener.onResponse(new StopDetectorResponse(true));
            }
        }, exception -> {
            LOG.error(new ParameterizedMessage("Deletion of detector [{}] has exception.", adID), exception);
            listener.onResponse(new StopDetectorResponse(false));
        }));
    } catch (Exception e) {
        LOG.error("Fail to stop detector " + adID, e);
        Throwable cause = ExceptionsHelper.unwrapCause(e);
        listener.onFailure(new InternalFailure(adID, cause));
    }
}
 

public ListTasksResponse(List<TaskInfo> tasks, List<TaskOperationFailure> taskFailures, List<? extends FailedNodeException> nodeFailures) {
    super(taskFailures, nodeFailures);
    if (tasks == null) {
        this.tasks = Collections.emptyList();
    } else {
        this.tasks = Collections.unmodifiableList(new ArrayList<>(tasks));
    }
}
 

private final Response newResponse(
        Request request,
        AtomicReferenceArray responses,
        List<NoShardAvailableActionException> unavailableShardExceptions,
        Map<String, List<ShardRouting>> nodes,
        ClusterState clusterState) {
    int totalShards = 0;
    int successfulShards = 0;
    List<ShardOperationResult> broadcastByNodeResponses = new ArrayList<>();
    List<ShardOperationFailedException> exceptions = new ArrayList<>();
    for (int i = 0; i < responses.length(); i++) {
        if (responses.get(i) instanceof FailedNodeException) {
            FailedNodeException exception = (FailedNodeException) responses.get(i);
            totalShards += nodes.get(exception.nodeId()).size();
            for (ShardRouting shard : nodes.get(exception.nodeId())) {
                exceptions.add(new DefaultShardOperationFailedException(shard.getIndex(), shard.getId(), exception));
            }
        } else {
            NodeResponse response = (NodeResponse) responses.get(i);
            broadcastByNodeResponses.addAll(response.results);
            totalShards += response.getTotalShards();
            successfulShards += response.getSuccessfulShards();
            for (BroadcastShardOperationFailedException throwable : response.getExceptions()) {
                if (!TransportActions.isShardNotAvailableException(throwable)) {
                    exceptions.add(new DefaultShardOperationFailedException(throwable.getIndex(), throwable.getShardId().getId(), throwable));
                }
            }
        }
    }
    totalShards += unavailableShardExceptions.size();
    int failedShards = exceptions.size();
    return newResponse(request, totalShards, successfulShards, failedShards, broadcastByNodeResponses, exceptions, clusterState);
}
 

protected void onNodeFailure(DiscoveryNode node, int nodeIndex, Throwable t) {
    String nodeId = node.id();
    if (logger.isDebugEnabled() && !(t instanceof NodeShouldNotConnectException)) {
        logger.debug("failed to execute [{}] on node [{}]", t, actionName, nodeId);
    }

    // this is defensive to protect against the possibility of double invocation
    // the current implementation of TransportService#sendRequest guards against this
    // but concurrency is hard, safety is important, and the small performance loss here does not matter
    if (responses.compareAndSet(nodeIndex, null, new FailedNodeException(nodeId, "Failed node [" + nodeId + "]", t))) {
        if (counter.incrementAndGet() == responses.length()) {
            onCompletion();
        }
    }
}
 

public BaseTasksResponse(List<TaskOperationFailure> taskFailures, List<? extends FailedNodeException> nodeFailures) {
    if (taskFailures == null) {
        this.taskFailures = Collections.emptyList();
    } else {
        this.taskFailures = Collections.unmodifiableList(new ArrayList<>(taskFailures));
    }
    if (nodeFailures == null) {
        this.nodeFailures = Collections.emptyList();
    } else {
        this.nodeFailures = Collections.unmodifiableList(new ArrayList<>(nodeFailures));
    }
}
 

public CachesStatsNodesResponse(ClusterName clusterName, List<CachesStatsNodeResponse> nodes, List<FailedNodeException> failures) {
    super(clusterName, nodes, failures);
    allStores = new StatDetails();
    byStore = new HashMap<>();
    nodes.forEach((n) -> {
        allStores.doSum(n.allStores);
        n.byStore.forEach((k, v) -> byStore.merge(k, v, StatDetails::sum));
    });
}
 

private void onFailure(int idx, String nodeId, Throwable t) {
    if (logger.isDebugEnabled() && !(t instanceof NodeShouldNotConnectException)) {
        logger.debug("failed to execute on node [{}]", t, nodeId);
    }
    if (accumulateExceptions()) {
        responses.set(idx, new FailedNodeException(nodeId, "Failed node [" + nodeId + "]", t));
    }
    if (counter.incrementAndGet() == responses.length()) {
        finishHim();
    }
}