org.elasticsearch.transport.ConnectTransportException Java Examples

@Test
public void testRetryOnNodeConnectionError() throws Exception {
    AtomicInteger numRetries = new AtomicInteger(0);
    BaseResultReceiver baseResultReceiver = new BaseResultReceiver();
    ClusterState initialState = clusterService.state();
    RetryOnFailureResultReceiver retryOnFailureResultReceiver = new RetryOnFailureResultReceiver(
        clusterService,
        initialState,
        indexName -> true,
        baseResultReceiver,
        UUID.randomUUID(),
        (newJobId, receiver) -> numRetries.incrementAndGet());

    // Must have a different cluster state then the initial state to trigger a retry
    clusterService.submitStateUpdateTask("dummy", new DummyUpdate());
    assertBusy(() -> assertThat(initialState, Matchers.not(sameInstance(clusterService.state()))));

    retryOnFailureResultReceiver.fail(new ConnectTransportException(null, "node not connected"));

    assertThat(numRetries.get(), is(1));
}
 

@Override
public void onFailure(Throwable e) {
    if (ExceptionsHelper.unwrapCause(e) instanceof ConnectTransportException) {
        int i = ++this.i;
        if (i >= nodes.size()) {
            listener.onFailure(new NoNodeAvailableException("None of the configured nodes were available: " + nodes, e));
        } else {
            try {
                callback.doWithNode(nodes.get((index + i) % nodes.size()), this);
            } catch(final Throwable t) {
                // this exception can't come from the TransportService as it doesn't throw exceptions at all
                listener.onFailure(t);
            }
        }
    } else {
        listener.onFailure(e);
    }
}
 

@Override
public void onFailure(Throwable e) {
    if (ExceptionsHelper.unwrapCause(e) instanceof ConnectTransportException) {
        int n = ++this.n;
        if (n >= nodes.size()) {
            listener.onFailure(new NoNodeAvailableException("none of the configured nodes were available: "
                    + nodes, e));
        } else {
            try {
                logger.warn("retrying on anoher node (n={}, nodes={})", n, nodes.size());
                callback.doWithNode(nodes.get((index + n) % nodes.size()), this);
            } catch (final Throwable t) {
                listener.onFailure(t);
            }
        }
    } else {
        listener.onFailure(e);
    }
}
 

@Override
public Connection openConnection(DiscoveryNode node, ConnectionProfile profile) {
    final Optional<DisruptableMockTransport> matchingTransport = getDisruptableMockTransport(node.getAddress());
    if (matchingTransport.isPresent()) {
        return new CloseableConnection() {
            @Override
            public DiscoveryNode getNode() {
                return node;
            }

            @Override
            public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options)
                throws TransportException {
                onSendRequest(requestId, action, request, matchingTransport.get());
            }

            @Override
            public String toString() {
                return "DisruptableMockTransportConnection{node=" + node + '}';
            }
        };
    } else {
        throw new ConnectTransportException(node, "node " + node + " does not exist");
    }
}
 

@Test
public void testOnFailureOnListenerIsCalledIfNodeIsNotInClusterState() throws Exception {
    Transports transports = new Transports(clusterService, mock(TransportService.class));
    ActionListener actionListener = mock(ActionListener.class);
    transports.sendRequest("actionName",
        "invalid", mock(TransportRequest.class), actionListener, mock(TransportResponseHandler.class));

    verify(actionListener, times(1)).onFailure(any(ConnectTransportException.class));
}
 

/**
 * Adds a rule that will cause ignores each send request, simulating an unresponsive node
 * and failing to connect once the rule was added.
 */
public void addUnresponsiveRule(TransportAddress transportAddress) {
    transport().addConnectBehavior(transportAddress, (transport, discoveryNode, profile) -> {
        throw new ConnectTransportException(discoveryNode, "UNRESPONSIVE: simulated");
    });

    transport().addSendBehavior(
        transportAddress,
        new StubbableTransport.SendRequestBehavior() {
            private Set<Transport.Connection> toClose = ConcurrentHashMap.newKeySet();

            @Override
            public void sendRequest(Transport.Connection connection,
                                    long requestId,
                                    String action,
                                    TransportRequest request,
                                    TransportRequestOptions options) {
                // don't send anything, the receiving node is unresponsive
                toClose.add(connection);
            }

            @Override
            public void clearCallback() {
                // close to simulate that tcp-ip eventually times out and closes connection
                // (necessary to ensure transport eventually responds).
                try {
                    IOUtils.close(toClose);
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
            }
        }
    );
}
 

protected NodeChannels connectToChannelsLight(DiscoveryNode node) {
    InetSocketAddress address = ((InetSocketTransportAddress) node.address()).address();
    ChannelFuture connect = clientBootstrap.connect(address);
    connect.awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
    if (!connect.isSuccess()) {
        throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connect.getCause());
    }
    Channel[] channels = new Channel[1];
    channels[0] = connect.getChannel();
    channels[0].getCloseFuture().addListener(new ChannelCloseListener(node));
    return new NodeChannels(channels, channels, channels, channels, channels);
}
 

/**
 * Adds a rule that will cause every send request to fail, and each new connect since the rule
 * is added to fail as well.
 */
public void addFailToSendNoConnectRule(TransportAddress transportAddress) {
    transport().addConnectBehavior(transportAddress, (transport, discoveryNode, profile) -> {
        throw new ConnectTransportException(discoveryNode, "DISCONNECT: simulated");
    });

    transport().addSendBehavior(transportAddress, (connection, requestId, action, request, options) -> {
        connection.close();
        // send the request, which will blow up
        connection.sendRequest(requestId, action, request, options);
    });
}
 

protected Channel nodeChannel(DiscoveryNode node, TransportRequestOptions options) throws ConnectTransportException {
    NodeChannels nodeChannels = connectedNodes.get(node);
    if (nodeChannels == null) {
        throw new NodeNotConnectedException(node, "Node not connected");
    }
    return nodeChannels.channel(options.type());
}
 

protected Runnable getDisconnectException(long requestId, String action, DiscoveryNode destination) {
    return new Runnable() {
        @Override
        public void run() {
            handleError(requestId, new ConnectTransportException(destination, "disconnected"));
        }

        @Override
        public String toString() {
            return "disconnection response to " + getRequestDescription(requestId, action, destination);
        }
    };
}
 

private void performAction(final DiscoveryNode node, final String action, final boolean isPrimaryAction,
                           final TransportRequest requestToPerform) {
    transportService.sendRequest(node, action, requestToPerform, transportOptions, new TransportResponseHandler<Response>() {

        @Override
        public Response read(StreamInput in) throws IOException {
            return TransportReplicationAction.this.read(in);
        }

        @Override
        public String executor() {
            return ThreadPool.Names.SAME;
        }

        @Override
        public void handleResponse(Response response) {
            finishOnSuccess(response);
        }

        @Override
        public void handleException(TransportException exp) {
            try {
                // if we got disconnected from the node, or the node / shard is not in the right state (being closed)
                final Throwable cause = exp.unwrapCause();
                if (cause instanceof ConnectTransportException || cause instanceof NodeClosedException ||
                    (isPrimaryAction && retryPrimaryException(cause))) {
                    logger.trace(() -> new ParameterizedMessage(
                            "received an error from node [{}] for request [{}], scheduling a retry",
                            node.getId(), requestToPerform), exp);
                    retry(exp);
                } else {
                    finishAsFailed(exp);
                }
            } catch (Exception e) {
                e.addSuppressed(exp);
                finishWithUnexpectedFailure(e);
            }
        }
    });
}
 

void handleWakeUp() {
    if (isClosed.get()) {
        LOGGER.trace("closed check scheduler woken up, doing nothing");
        return;
    }

    LOGGER.trace("checking {} with [{}] = {}", leader, LEADER_CHECK_TIMEOUT_SETTING.getKey(), leaderCheckTimeout);

    transportService.sendRequest(leader, LEADER_CHECK_ACTION_NAME, new LeaderCheckRequest(transportService.getLocalNode()),
        TransportRequestOptions.builder().withTimeout(leaderCheckTimeout).withType(Type.PING).build(),

        new TransportResponseHandler<TransportResponse.Empty>() {

            @Override
            public Empty read(StreamInput in) {
                return Empty.INSTANCE;
            }

            @Override
            public void handleResponse(Empty response) {
                if (isClosed.get()) {
                    LOGGER.debug("closed check scheduler received a response, doing nothing");
                    return;
                }

                failureCountSinceLastSuccess.set(0);
                scheduleNextWakeUp(); // logs trace message indicating success
            }

            @Override
            public void handleException(TransportException exp) {
                if (isClosed.get()) {
                    LOGGER.debug("closed check scheduler received a response, doing nothing");
                    return;
                }

                if (exp instanceof ConnectTransportException || exp.getCause() instanceof ConnectTransportException) {
                    LOGGER.debug(new ParameterizedMessage("leader [{}] disconnected, failing immediately", leader), exp);
                    leaderFailed();
                    return;
                }

                long failureCount = failureCountSinceLastSuccess.incrementAndGet();
                if (failureCount >= leaderCheckRetryCount) {
                    LOGGER.debug(new ParameterizedMessage("{} consecutive failures (limit [{}] is {}) so leader [{}] has failed",
                        failureCount, LEADER_CHECK_RETRY_COUNT_SETTING.getKey(), leaderCheckRetryCount, leader), exp);
                    leaderFailed();
                    return;
                }

                LOGGER.debug(new ParameterizedMessage("{} consecutive failures (limit [{}] is {}) with leader [{}]",
                    failureCount, LEADER_CHECK_RETRY_COUNT_SETTING.getKey(), leaderCheckRetryCount, leader), exp);
                scheduleNextWakeUp();
            }

            @Override
            public String executor() {
                return Names.SAME;
            }
        });
}
 

@Override
public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile,
                          CheckedBiConsumer<Transport.Connection, ConnectionProfile, IOException> connectionValidator)
    throws ConnectTransportException {
    delegate.connectToNode(node, connectionProfile, connectionValidator);
}
 

public void testFailsNodeThatIsDisconnected() {
    testBehaviourOfFailingNode(Settings.EMPTY, () -> {
        throw new ConnectTransportException(null, "simulated exception");
    }, "disconnected", 0);
}
 

private static boolean isTimeoutOrNodeNotReachable(Throwable t) {
    return t instanceof ReceiveTimeoutTransportException
        || t instanceof ConnectTransportException;
}
 

private void handleWakeUp() {
    if (running() == false) {
        LOGGER.trace("handleWakeUp: not running");
        return;
    }

    final FollowerCheckRequest request = new FollowerCheckRequest(fastResponseState.term, transportService.getLocalNode());
    LOGGER.trace("handleWakeUp: checking {} with {}", discoveryNode, request);

    transportService.sendRequest(discoveryNode, FOLLOWER_CHECK_ACTION_NAME, request,
        TransportRequestOptions.builder().withTimeout(followerCheckTimeout).withType(Type.PING).build(),
        new TransportResponseHandler<Empty>() {
            @Override
            public Empty read(StreamInput in) {
                return Empty.INSTANCE;
            }

            @Override
            public void handleResponse(Empty response) {
                if (running() == false) {
                    LOGGER.trace("{} no longer running", FollowerChecker.this);
                    return;
                }

                failureCountSinceLastSuccess = 0;
                LOGGER.trace("{} check successful", FollowerChecker.this);
                scheduleNextWakeUp();
            }

            @Override
            public void handleException(TransportException exp) {
                if (running() == false) {
                    LOGGER.debug(new ParameterizedMessage("{} no longer running", FollowerChecker.this), exp);
                    return;
                }

                failureCountSinceLastSuccess++;

                final String reason;
                if (failureCountSinceLastSuccess >= followerCheckRetryCount) {
                    LOGGER.debug(() -> new ParameterizedMessage("{} failed too many times", FollowerChecker.this), exp);
                    reason = "followers check retry count exceeded";
                } else if (exp instanceof ConnectTransportException
                    || exp.getCause() instanceof ConnectTransportException) {
                    LOGGER.debug(() -> new ParameterizedMessage("{} disconnected", FollowerChecker.this), exp);
                    reason = "disconnected";
                } else {
                    LOGGER.debug(() -> new ParameterizedMessage("{} failed, retrying", FollowerChecker.this), exp);
                    scheduleNextWakeUp();
                    return;
                }

                failNode(reason);
            }


            @Override
            public String executor() {
                return Names.SAME;
            }
        });
}
 

protected void doStart() {
    final ClusterState clusterState = observer.observedState();
    final DiscoveryNodes nodes = clusterState.nodes();
    if (nodes.localNodeMaster() || localExecute(request)) {
        // check for block, if blocked, retry, else, execute locally
        final ClusterBlockException blockException = checkBlock(request, clusterState);
        if (blockException != null) {
            if (!blockException.retryable()) {
                listener.onFailure(blockException);
            } else {
                logger.trace("can't execute due to a cluster block, retrying", blockException);
                retry(blockException, retryableOrNoBlockPredicate);
            }
        } else {
            final ActionListener<Response> delegate = new ActionListener<Response>() {
                @Override
                public void onResponse(Response response) {
                    listener.onResponse(response);
                }

                @Override
                public void onFailure(Throwable t) {
                    if (t instanceof NotMasterException) {
                        logger.debug("master could not publish cluster state or stepped down before publishing action [{}], scheduling a retry", t, actionName);
                        retry(t, masterNodeChangedPredicate);
                    } else {
                        listener.onFailure(t);
                    }
                }
            };
            taskManager.registerChildTask(task, nodes.getLocalNodeId());
            threadPool.executor(executor).execute(new ActionRunnable(delegate) {
                @Override
                protected void doRun() throws Exception {
                    masterOperation(task, request, clusterService.state(), delegate);
                }
            });
        }
    } else {
        if (nodes.masterNode() == null) {
            logger.debug("no known master node, scheduling a retry");
            retry(null, masterNodeChangedPredicate);
        } else {
            taskManager.registerChildTask(task, nodes.masterNode().getId());
            transportService.sendRequest(nodes.masterNode(), actionName, request, new ActionListenerResponseHandler<Response>(listener) {
                @Override
                public Response newInstance() {
                    return newResponse();
                }

                @Override
                public void handleException(final TransportException exp) {
                    Throwable cause = exp.unwrapCause();
                    if (cause instanceof ConnectTransportException) {
                        // we want to retry here a bit to see if a new master is elected
                        logger.debug("connection exception while trying to forward request with action name [{}] to master node [{}], scheduling a retry. Error: [{}]",
                                actionName, nodes.masterNode(), exp.getDetailedMessage());
                        retry(cause, masterNodeChangedPredicate);
                    } else {
                        listener.onFailure(exp);
                    }
                }
            });
        }
    }
}
 

protected void connectToChannels(NodeChannels nodeChannels, DiscoveryNode node) {
    ChannelFuture[] connectRecovery = new ChannelFuture[nodeChannels.recovery.length];
    ChannelFuture[] connectBulk = new ChannelFuture[nodeChannels.bulk.length];
    ChannelFuture[] connectReg = new ChannelFuture[nodeChannels.reg.length];
    ChannelFuture[] connectState = new ChannelFuture[nodeChannels.state.length];
    ChannelFuture[] connectPing = new ChannelFuture[nodeChannels.ping.length];
    InetSocketAddress address = ((InetSocketTransportAddress) node.address()).address();
    for (int i = 0; i < connectRecovery.length; i++) {
        connectRecovery[i] = clientBootstrap.connect(address);
    }
    for (int i = 0; i < connectBulk.length; i++) {
        connectBulk[i] = clientBootstrap.connect(address);
    }
    for (int i = 0; i < connectReg.length; i++) {
        connectReg[i] = clientBootstrap.connect(address);
    }
    for (int i = 0; i < connectState.length; i++) {
        connectState[i] = clientBootstrap.connect(address);
    }
    for (int i = 0; i < connectPing.length; i++) {
        connectPing[i] = clientBootstrap.connect(address);
    }

    try {
        for (int i = 0; i < connectRecovery.length; i++) {
            connectRecovery[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
            if (!connectRecovery[i].isSuccess()) {
                throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectRecovery[i].getCause());
            }
            nodeChannels.recovery[i] = connectRecovery[i].getChannel();
            nodeChannels.recovery[i].getCloseFuture().addListener(new ChannelCloseListener(node));
        }

        for (int i = 0; i < connectBulk.length; i++) {
            connectBulk[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
            if (!connectBulk[i].isSuccess()) {
                throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectBulk[i].getCause());
            }
            nodeChannels.bulk[i] = connectBulk[i].getChannel();
            nodeChannels.bulk[i].getCloseFuture().addListener(new ChannelCloseListener(node));
        }

        for (int i = 0; i < connectReg.length; i++) {
            connectReg[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
            if (!connectReg[i].isSuccess()) {
                throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectReg[i].getCause());
            }
            nodeChannels.reg[i] = connectReg[i].getChannel();
            nodeChannels.reg[i].getCloseFuture().addListener(new ChannelCloseListener(node));
        }

        for (int i = 0; i < connectState.length; i++) {
            connectState[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
            if (!connectState[i].isSuccess()) {
                throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectState[i].getCause());
            }
            nodeChannels.state[i] = connectState[i].getChannel();
            nodeChannels.state[i].getCloseFuture().addListener(new ChannelCloseListener(node));
        }

        for (int i = 0; i < connectPing.length; i++) {
            connectPing[i].awaitUninterruptibly((long) (connectTimeout.millis() * 1.5));
            if (!connectPing[i].isSuccess()) {
                throw new ConnectTransportException(node, "connect_timeout[" + connectTimeout + "]", connectPing[i].getCause());
            }
            nodeChannels.ping[i] = connectPing[i].getChannel();
            nodeChannels.ping[i].getCloseFuture().addListener(new ChannelCloseListener(node));
        }

        if (nodeChannels.recovery.length == 0) {
            if (nodeChannels.bulk.length > 0) {
                nodeChannels.recovery = nodeChannels.bulk;
            } else {
                nodeChannels.recovery = nodeChannels.reg;
            }
        }
        if (nodeChannels.bulk.length == 0) {
            nodeChannels.bulk = nodeChannels.reg;
        }
    } catch (RuntimeException e) {
        // clean the futures
        List<ChannelFuture> futures = new ArrayList<>();
        futures.addAll(Arrays.asList(connectRecovery));
        futures.addAll(Arrays.asList(connectBulk));
        futures.addAll(Arrays.asList(connectReg));
        futures.addAll(Arrays.asList(connectState));
        futures.addAll(Arrays.asList(connectPing));
        for (ChannelFuture future : Collections.unmodifiableList(futures)) {
            future.cancel();
            if (future.getChannel() != null && future.getChannel().isOpen()) {
                try {
                    future.getChannel().close();
                } catch (Exception e1) {
                    // ignore
                }
            }
        }
        throw e;
    }
}
 

@Override
public void connectToNodeLight(DiscoveryNode node) throws ConnectTransportException {
    connectToNode(node, true);
}
 

private void sendPingRequestToNode(final int id, final TimeValue timeout, final UnicastPingRequest pingRequest, final CountDownLatch latch, final DiscoveryNode node, final DiscoveryNode nodeToSend) {
    logger.trace("[{}] sending to {}", id, nodeToSend);
    transportService.sendRequest(nodeToSend, ACTION_NAME, pingRequest, TransportRequestOptions.builder().withTimeout((long) (timeout.millis() * 1.25)).build(), new BaseTransportResponseHandler<UnicastPingResponse>() {

        @Override
        public UnicastPingResponse newInstance() {
            return new UnicastPingResponse();
        }

        @Override
        public String executor() {
            return ThreadPool.Names.SAME;
        }

        @Override
        public void handleResponse(UnicastPingResponse response) {
            logger.trace("[{}] received response from {}: {}", id, nodeToSend, Arrays.toString(response.pingResponses));
            try {
                DiscoveryNodes discoveryNodes = contextProvider.nodes();
                for (PingResponse pingResponse : response.pingResponses) {
                    if (pingResponse.node().id().equals(discoveryNodes.localNodeId())) {
                        // that's us, ignore
                        continue;
                    }
                    if (!pingResponse.clusterName().equals(clusterName)) {
                        // not part of the cluster
                        logger.debug("[{}] filtering out response from {}, not same cluster_name [{}]", id, pingResponse.node(), pingResponse.clusterName().value());
                        continue;
                    }
                    SendPingsHandler sendPingsHandler = receivedResponses.get(response.id);
                    if (sendPingsHandler == null) {
                        if (!closed) {
                            // Only log when we're not closing the node. Having no send ping handler is then expected
                            logger.warn("received ping response {} with no matching handler id [{}]", pingResponse, response.id);
                        }
                    } else {
                        sendPingsHandler.pingCollection().addPing(pingResponse);
                    }
                }
            } finally {
                latch.countDown();
            }
        }

        @Override
        public void handleException(TransportException exp) {
            latch.countDown();
            if (exp instanceof ConnectTransportException) {
                // ok, not connected...
                logger.trace("failed to connect to {}", exp, nodeToSend);
            } else {
                logger.warn("failed to send ping to [{}]", exp, node);
            }
        }
    });
}
 

@Override
public void run() {
    if (!running()) {
        return;
    }
    final PingRequest pingRequest = new PingRequest(node, clusterName, localNode, clusterStateVersion, isDeadNode);
    final TransportRequestOptions options = TransportRequestOptions.builder().withType(TransportRequestOptions.Type.PING).withTimeout(pingTimeout).build();
    transportService.sendRequest(node, PING_ACTION_NAME, pingRequest, options, new BaseTransportResponseHandler<PingResponse>() {
        @Override
        public PingResponse newInstance() {
            return new PingResponse();
        }

        @Override
        public void handleResponse(PingResponse response) {
            if (!running()) {
                return;
            }
            latestDlsn = response.dlsnProcessed;
            retryCount = 0;
            threadPool.schedule(pingInterval, ThreadPool.Names.SAME, NodeFD.this);
        }

        @Override
        public void handleException(TransportException exp) {
            retryCount++;
            try {
                if (!running()) {
                    return;
                }

                if (!isDeadNode && retryCount >= pingRetryCount) {
                    logger.debug("[node  ] failed to ping [{}], tried [{}] times, each with  maximum [{}] timeout", exp, node, pingRetryCount, pingTimeout);
                    handleNodeFailure(NodeFD.this, "failed to ping, tried [" + pingRetryCount + "] times, each with maximum [" + pingTimeout + "] timeout");
                    retryCount = 0;
                }
                
                if (exp instanceof NodeIdNotMatchException || exp.getCause() instanceof NodeIdNotMatchException) {
                    handleNodeFailure(NodeFD.this, "node id not match, move the node and wait for the node to rejoin");
                }
                
                if (exp instanceof ConnectTransportException || exp.getCause() instanceof ConnectTransportException) {
                    transportService.connectToNode(node);
                } else {
                    logger.debug("[node  ] failed to ping [{}], num retry [{}], nodefd detail [{}]", exp, node, retryCount, NodeFD.this);
                }
            } catch (Throwable t) {
                logger.error("errors while handle exceptions when ping node [{}]", NodeFD.this);
            }
            threadPool.schedule(TimeValue.timeValueMillis(Math.min(1000 * retryCount, 5000)), ThreadPool.Names.SAME, NodeFD.this);
        }

        @Override
        public String executor() {
            return ThreadPool.Names.SAME;
        }
    }
    );
}
 

/**
 * Check if the input exception indicates connection issues.
 *
 * @param e exception
 * @return true if we get disconnected from the node or the node is not in the
 *         right state (being closed) or transport request times out (sent from TimeoutHandler.run)
 */
private boolean hasConnectionIssue(Throwable e) {
    return e instanceof ConnectTransportException || e instanceof NodeClosedException || e instanceof ReceiveTimeoutTransportException;
}