Java Code Examples for org.apache.ratis.util.JavaUtils#attempt()

void runTestHandleNotLeaderException(boolean killNewLeader, CLUSTER cluster) throws Exception {
  final RaftPeerId oldLeader = RaftTestUtil.waitForLeader(cluster).getId();
  try(final RaftClient client = cluster.createClient(oldLeader)) {
    sendMessage("m1", client);

    // enforce leader change
    final RaftPeerId newLeader = RaftTestUtil.changeLeader(cluster, oldLeader);

    if (killNewLeader) {
      // kill the new leader
      cluster.killServer(newLeader);
    }

    final RaftClientRpc rpc = client.getClientRpc();
    JavaUtils.attempt(() -> assertNotLeaderException(newLeader, "m2", oldLeader, rpc, cluster),
        10, ONE_SECOND, "assertNotLeaderException", LOG);

    sendMessage("m3", client);
  }
}
 

static void runTestFlushMetric(MiniRaftCluster cluster) throws Exception {
  int numMsg = 2;
  final RaftTestUtil.SimpleMessage[] messages = RaftTestUtil.SimpleMessage.create(numMsg);

  try (final RaftClient client = cluster.createClient()) {
    for (RaftTestUtil.SimpleMessage message : messages) {
      client.send(message);
    }
  }

  // For leader, flush must happen before client can get replies.
  assertFlushCount(cluster.getLeader());

  // For followers, flush can be lagged behind.  Attempt multiple times.
  for(RaftServerImpl f : cluster.getFollowers()) {
    JavaUtils.attempt(() -> assertFlushCount(f), 10, 100, f.getId() + "-assertFlushCount", null);
  }
}
 

static void runTestRaftLogMetrics(MiniRaftCluster cluster) throws Exception {
  int numMsg = 2;
  final RaftTestUtil.SimpleMessage[] messages = RaftTestUtil.SimpleMessage.create(numMsg);

  try (final RaftClient client = cluster.createClient()) {
    for (RaftTestUtil.SimpleMessage message : messages) {
      client.send(message);
    }
  }

  // For leader, flush must happen before client can get replies.
  assertFlushCount(cluster.getLeader());
  assertRaftLogWritePathMetrics(cluster.getLeader());

  // For followers, flush can be lagged behind.  Attempt multiple times.
  for(RaftServerImpl f : cluster.getFollowers()) {
    JavaUtils.attempt(() -> assertFlushCount(f), 10, HUNDRED_MILLIS, f.getId() + "-assertFlushCount", null);
    // We have already waited enough for follower metrics to populate.
    assertRaftLogWritePathMetrics(f);
  }

  // Wait for commits to happen on leader
  JavaUtils.attempt(() -> assertCommitCount(cluster.getLeader(), numMsg), 10, HUNDRED_MILLIS, cluster.getLeader().getId() + "-assertCommitCount", null);
}
 

static RaftPeerId changeLeader(MiniRaftCluster cluster, RaftPeerId oldLeader)
    throws InterruptedException {
  cluster.setBlockRequestsFrom(oldLeader.toString(), true);
  try {
    return JavaUtils.attempt(() -> {
      final RaftPeerId newLeader = waitForLeader(cluster).getId();
      Preconditions.assertTrue(!newLeader.equals(oldLeader),
          () -> "Failed to change leader: newLeader=" + newLeader + " equals oldLeader=" + oldLeader);
      LOG.info("Changed leader from " + oldLeader + " to " + newLeader);
      return newLeader;
    }, 10, 100L, "changeLeader", LOG);
  } finally {
    cluster.setBlockRequestsFrom(oldLeader.toString(), false);
  }
}
 

public static void waitAndCheckNewConf(MiniRaftCluster cluster,
    RaftPeer[] peers, int numOfRemovedPeers, Collection<RaftPeerId> deadPeers)
    throws Exception {
  final TimeDuration sleepTime = cluster.getTimeoutMax().apply(n -> n * (numOfRemovedPeers + 2));
  JavaUtils.attempt(() -> waitAndCheckNewConf(cluster, Arrays.asList(peers), deadPeers),
      10, sleepTime, "waitAndCheckNewConf", LOG);
}
 

static RaftServerImpl waitForLeader(
    MiniRaftCluster cluster, RaftGroupId groupId, boolean expectLeader)
    throws InterruptedException {
  final String name = "waitForLeader-" + groupId + "-(expectLeader? " + expectLeader + ")";
  final int numAttempts = expectLeader? 100: 10;
  final TimeDuration sleepTime = cluster.getTimeoutMax().apply(d -> (d * 3) >> 1);
  LOG.info(cluster.printServers(groupId));

  final AtomicReference<IllegalStateException> exception = new AtomicReference<>();
  final Runnable handleNoLeaders = () -> {
    throw cluster.newIllegalStateExceptionForNoLeaders(groupId);
  };
  final Consumer<List<RaftServerImpl>> handleMultipleLeaders = leaders -> {
    final IllegalStateException ise = cluster.newIllegalStateExceptionForMultipleLeaders(groupId, leaders);
    exception.set(ise);
  };

  final RaftServerImpl leader = JavaUtils.attempt(
      () -> cluster.getLeader(groupId, handleNoLeaders, handleMultipleLeaders),
      numAttempts, sleepTime, name, LOG);

  LOG.info(cluster.printServers(groupId));
  if (expectLeader) {
    return Optional.ofNullable(leader).orElseThrow(exception::get);
  } else {
    if (leader == null) {
      return null;
    } else {
      throw new IllegalStateException("expectLeader = " + expectLeader + " but leader = " + leader);
    }
  }
}
 

private static RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry, RaftProperties properties, Parameters parameters)
    throws IOException {
  final RaftServerProxy proxy;
  try {
    // attempt multiple times to avoid temporary bind exception
    proxy = JavaUtils.attempt(
        () -> new RaftServerProxy(id, stateMachineRegistry, properties, parameters),
        5, 500L, "new RaftServerProxy", RaftServerProxy.LOG);
  } catch (InterruptedException e) {
    throw IOUtils.toInterruptedIOException(
        "Interrupted when creating RaftServer " + id, e);
  }
  return proxy;
}
 

/**
 * Keep generating writing traffic and make sure snapshots are taken.
 * We then restart the whole raft peer and check if it can correctly load
 * snapshots + raft log.
 */
@Test
public void testRestartPeer() throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  final RaftPeerId leaderId = cluster.getLeader().getId();
  int i = 0;
  try(final RaftClient client = cluster.createClient(leaderId)) {
    for (; i < SNAPSHOT_TRIGGER_THRESHOLD * 2 - 1; i++) {
      RaftClientReply reply = client.send(new SimpleMessage("m" + i));
      Assert.assertTrue(reply.isSuccess());
    }
  }

  final long nextIndex = cluster.getLeader().getState().getLog().getNextIndex();
  LOG.info("nextIndex = {}", nextIndex);
  // wait for the snapshot to be done
  final List<File> snapshotFiles = getSnapshotFiles(cluster, nextIndex - SNAPSHOT_TRIGGER_THRESHOLD, nextIndex);
  JavaUtils.attempt(() -> snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists),
      10, 1000, "snapshotFile.exist", LOG);

  // restart the peer and check if it can correctly load snapshot
  cluster.restart(false);
  try {
    // 200 messages + two leader elections --> last committed = 201
    assertLeaderContent(cluster);
  } finally {
    cluster.shutdown();
  }
}
 

@Test
public void testLateServerStart() throws Exception {
  final int numServer = 3;
  LOG.info("Running testLateServerStart");
  final MiniRaftCluster cluster = newCluster(numServer);
  cluster.initServers();

  // start all except one servers
  final Iterator<RaftServerProxy> i = cluster.getServers().iterator();
  for(int j = 1; j < numServer; j++) {
    i.next().start();
  }

  final RaftServerImpl leader = waitForLeader(cluster);
  final TimeDuration sleepTime = TimeDuration.valueOf(3, TimeUnit.SECONDS);
  LOG.info("sleep " + sleepTime);
  sleepTime.sleep();

  // start the last server
  final RaftServerProxy lastServer = i.next();
  lastServer.start();
  final RaftPeerId lastServerLeaderId = JavaUtils.attempt(
      () -> getLeader(lastServer.getImpls().iterator().next().getState()),
      10, 1000, "getLeaderId", LOG);
  LOG.info(cluster.printServers());
  Assert.assertEquals(leader.getId(), lastServerLeaderId);
}
 

public static void waitAndCheckNewConf(MiniRaftCluster cluster,
    RaftPeer[] peers, int numOfRemovedPeers, Collection<RaftPeerId> deadPeers)
    throws Exception {
  final TimeDuration sleepTime = cluster.getTimeoutMax().apply(n -> n * (numOfRemovedPeers + 2));
  JavaUtils.attempt(() -> waitAndCheckNewConf(cluster, peers, deadPeers),
      10, sleepTime, "waitAndCheckNewConf", LOG);
}
 

/**
 * Basic test for install snapshot: start a one node cluster and let it
 * generate a snapshot. Then delete the log and restart the node, and add more
 * nodes as followers.
 */
@Test
public void testBasicInstallSnapshot() throws Exception {
  final List<LogPathAndIndex> logs;
  try {
    RaftTestUtil.waitForLeader(cluster);
    final RaftPeerId leaderId = cluster.getLeader().getId();

    int i = 0;
    try(final RaftClient client = cluster.createClient(leaderId)) {
      for (; i < SNAPSHOT_TRIGGER_THRESHOLD * 2 - 1; i++) {
        RaftClientReply reply = client.send(new SimpleMessage("m" + i));
        Assert.assertTrue(reply.isSuccess());
      }
    }

    // wait for the snapshot to be done
    RaftStorageDirectory storageDirectory = cluster.getLeader().getState()
        .getStorage().getStorageDir();

    final long nextIndex = cluster.getLeader().getState().getLog().getNextIndex();
    LOG.info("nextIndex = {}", nextIndex);
    final List<File> snapshotFiles = getSnapshotFiles(cluster, nextIndex - SNAPSHOT_TRIGGER_THRESHOLD, nextIndex);
    JavaUtils.attempt(() -> snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists),
        10, 1000, "snapshotFile.exist", LOG);
    logs = storageDirectory.getLogSegmentFiles();
  } finally {
    cluster.shutdown();
  }

  // delete the log segments from the leader
  for (LogPathAndIndex path : logs) {
    FileUtils.delete(path.getPath());
  }

  // restart the peer
  LOG.info("Restarting the cluster");
  cluster.restart(false);
  try {
    assertLeaderContent(cluster);

    // generate some more traffic
    try(final RaftClient client = cluster.createClient(cluster.getLeader().getId())) {
      Assert.assertTrue(client.send(new SimpleMessage("test")).isSuccess());
    }

    // add two more peers
    MiniRaftCluster.PeerChanges change = cluster.addNewPeers(
        new String[]{"s3", "s4"}, true);
    // trigger setConfiguration
    cluster.setConfiguration(change.allPeersInNewConf);

    RaftServerTestUtil.waitAndCheckNewConf(cluster, change.allPeersInNewConf, 0, null);
  } finally {
    cluster.shutdown();
  }
}
 

void runTestRetryOnNewLeader(CLUSTER cluster) throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  final RaftPeerId leaderId = cluster.getLeaderAndSendFirstMessage(false).getId();

  final RaftClient client = cluster.createClient(leaderId);
  RaftClientRpc rpc = client.getClientRpc();
  final long callId = 999;
  final long seqNum = 111;
  RaftClientRequest r = cluster.newRaftClientRequest(client.getId(), leaderId,
      callId, seqNum, new SimpleMessage("message"));
  assertReply(rpc.sendRequest(r), client, callId);
  long oldLastApplied = cluster.getLeader().getState().getLastAppliedIndex();

  // trigger the reconfiguration, make sure the original leader is kicked out
  PeerChanges change = cluster.addNewPeers(2, true);
  RaftPeer[] allPeers = cluster.removePeers(2, true,
      asList(change.newPeers)).allPeersInNewConf;
  // trigger setConfiguration
  cluster.setConfiguration(allPeers);

  final RaftPeerId newLeaderId = JavaUtils.attempt(() -> {
    final RaftPeerId id = RaftTestUtil.waitForLeader(cluster).getId();
    Assert.assertNotEquals(leaderId, id);
    return id;
  }, 10, TimeDuration.valueOf(100, TimeUnit.MILLISECONDS), "wait for a leader different than " + leaderId, LOG);
  Assert.assertNotEquals(leaderId, newLeaderId);
  // same clientId and callId in the request
  r = cluster.newRaftClientRequest(client.getId(), newLeaderId,
      callId, seqNum, new SimpleMessage("message"));
  rpc.addServers(Arrays.asList(change.newPeers));
  for (int i = 0; i < 10; i++) {
    try {
      assertReply(rpc.sendRequest(r), client, callId);
      LOG.info("successfully sent out the retry request_" + i);
    } catch (Exception e) {
      LOG.info("hit exception while retrying the same request: " + r, e);
    }
    Thread.sleep(100);
  }

  // check the new leader and make sure the retry did not get committed
  Assert.assertEquals(0, count(cluster.getLeader().getState().getLog(), oldLastApplied + 1));
  client.close();
}
 

void runTestRestartCommitIndex(MiniRaftCluster cluster) throws Exception {
  final TimeDuration sleepTime = TimeDuration.valueOf(100, TimeUnit.MILLISECONDS);
  final SimpleMessage[] messages = SimpleMessage.create(10);
  try (final RaftClient client = cluster.createClient()) {
    for(SimpleMessage m : messages) {
      Assert.assertTrue(client.send(m).isSuccess());
    }
  }

  final List<RaftPeerId> ids = new ArrayList<>();
  final RaftLog leaderLog = cluster.getLeader().getState().getLog();
  final RaftPeerId leaderId = leaderLog.getSelfId();
  ids.add(leaderId);

  // check that the last logged commit index is equal to the index of the last committed StateMachineLogEntry
  final long lastIndex = leaderLog.getLastEntryTermIndex().getIndex();
  LOG.info("{}: leader lastIndex={}", leaderId, lastIndex);
  JavaUtils.attempt(() -> leaderLog.getLastCommittedIndex() == lastIndex,
      10, sleepTime, "leader(commitIndex == lastIndex)", LOG);

  final LogEntryProto lastEntry = leaderLog.get(lastIndex);
  LOG.info("{}: leader lastEntry entry[{}] = {}", leaderId, lastIndex, ServerProtoUtils.toLogEntryString(lastEntry));
  Assert.assertTrue(lastEntry.hasMetadataEntry());
  final long loggedCommitIndex = lastEntry.getMetadataEntry().getCommitIndex();
  for(long i = lastIndex - 1; i > loggedCommitIndex; i--) {
    final LogEntryProto entry = leaderLog.get(i);
    LOG.info("{}: leader entry[{}] =  {}", leaderId, i, ServerProtoUtils.toLogEntryString(entry));
    Assert.assertFalse(entry.hasStateMachineLogEntry());
  }
  final LogEntryProto lastCommittedEntry = leaderLog.get(loggedCommitIndex);
  LOG.info("{}: leader lastCommittedEntry = entry[{}] = {}",
      leaderId, loggedCommitIndex, ServerProtoUtils.toLogEntryString(lastCommittedEntry));
  Assert.assertTrue(lastCommittedEntry.hasStateMachineLogEntry());

  // check follower logs
  for(RaftServerImpl s : cluster.iterateServerImpls()) {
    if (!s.getId().equals(leaderId)) {
      ids.add(s.getId());
      RaftTestUtil.assertSameLog(leaderLog, s.getState().getLog());
    }
  }

  // kill all servers
  ids.forEach(cluster::killServer);

  // Restart and kill servers one by one so that they won't talk to each other.
  for(RaftPeerId id : ids) {
    cluster.restartServer(id, false);
    final RaftServerImpl server = cluster.getRaftServerImpl(id);
    final RaftLog raftLog = server.getState().getLog();
    JavaUtils.attempt(() -> raftLog.getLastCommittedIndex() >= loggedCommitIndex,
        10, sleepTime, id + "(commitIndex >= loggedCommitIndex)", LOG);
    JavaUtils.attempt(() -> server.getState().getLastAppliedIndex() >= loggedCommitIndex,
        10, sleepTime, id + "(lastAppliedIndex >= loggedCommitIndex)", LOG);
    LOG.info("{}: commitIndex={}, lastAppliedIndex={}",
        id, raftLog.getLastCommittedIndex(), server.getState().getLastAppliedIndex());
    cluster.killServer(id);
  }
}
 

void runTestRestartFollower(MiniRaftCluster cluster) throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  final RaftPeerId leaderId = cluster.getLeader().getId();

  // write some messages
  final AtomicInteger messageCount = new AtomicInteger();
  final Supplier<Message> newMessage = () -> new SimpleMessage("m" + messageCount.getAndIncrement());
  writeSomething(newMessage, cluster);

  // restart a follower
  RaftPeerId followerId = cluster.getFollowers().get(0).getId();
  LOG.info("Restart follower {}", followerId);
  cluster.restartServer(followerId, false);

  // write some more messages
  writeSomething(newMessage, cluster);
  final int truncatedMessageIndex = messageCount.get() - 1;

  final long leaderLastIndex = cluster.getLeader().getState().getLog().getLastEntryTermIndex().getIndex();
  // make sure the restarted follower can catchup
  final ServerState followerState = cluster.getRaftServerImpl(followerId).getState();
  JavaUtils.attempt(() -> followerState.getLastAppliedIndex() >= leaderLastIndex,
      10, 500, "follower catchup", LOG);

  // make sure the restarted peer's log segments is correct
  final RaftServerImpl follower = cluster.restartServer(followerId, false);
  final RaftLog followerLog = follower.getState().getLog();
  final long followerLastIndex = followerLog.getLastEntryTermIndex().getIndex();
  Assert.assertTrue(followerLastIndex >= leaderLastIndex);

  final File followerOpenLogFile = getOpenLogFile(follower);
  final File leaderOpenLogFile = getOpenLogFile(cluster.getRaftServerImpl(leaderId));

  // shutdown all servers
  cluster.getServers().forEach(RaftServerProxy::close);

  // truncate log and
  assertTruncatedLog(followerId, followerOpenLogFile, followerLastIndex, cluster);
  assertTruncatedLog(leaderId, leaderOpenLogFile, leaderLastIndex, cluster);

  // restart and write something.
  cluster.restart(false);
  writeSomething(newMessage, cluster);

  // restart again and check messages.
  cluster.restart(false);
  try(final RaftClient client = cluster.createClient()) {
    for(int i = 0; i < messageCount.get(); i++) {
      if (i != truncatedMessageIndex) {
        final Message m = new SimpleMessage("m" + i);
        final RaftClientReply reply = client.sendReadOnly(m);
        Assert.assertTrue(reply.isSuccess());
        LOG.info("query {}: {} {}", m, reply, LogEntryProto.parseFrom(reply.getMessage().getContent()));
      }
    }
  }
}
 

void runTestRevertConfigurationChange(CLUSTER cluster) throws Exception {
  RaftLog log2 = null;
  try {
    RaftTestUtil.waitForLeader(cluster);

    final RaftServerImpl leader = cluster.getLeader();
    final RaftPeerId leaderId = leader.getId();

    final RaftLog log = leader.getState().getLog();
    log2 = log;
    Thread.sleep(1000);

    // we block the incoming msg for the leader and block its requests to
    // followers, so that we force the leader change and the old leader will
    // not know
    LOG.info("start blocking the leader");
    BlockRequestHandlingInjection.getInstance().blockReplier(leaderId.toString());
    cluster.setBlockRequestsFrom(leaderId.toString(), true);

    PeerChanges change = cluster.removePeers(1, false, new ArrayList<>());

    AtomicBoolean gotNotLeader = new AtomicBoolean(false);
    final Thread clientThread = new Thread(() -> {
      try(final RaftClient client = cluster.createClient(leaderId)) {
        LOG.info("client starts to change conf");
        final RaftClientRpc sender = client.getClientRpc();
        RaftClientReply reply = sender.sendRequest(cluster.newSetConfigurationRequest(
            client.getId(), leaderId, change.allPeersInNewConf));
        if (reply.getNotLeaderException() != null) {
          gotNotLeader.set(true);
        }
      } catch (IOException e) {
        LOG.warn("Got unexpected exception when client1 changes conf", e);
      }
    });
    clientThread.start();

    // find ConfigurationEntry
    final long confIndex = JavaUtils.attempt(() -> {
      final long last = log.getLastEntryTermIndex().getIndex();
      for (long i = last; i >= 1; i--) {
        if (log.get(i).hasConfigurationEntry()) {
          return i;
        }
      }
      throw new Exception("ConfigurationEntry not found: last=" + last);
    }, 10, 500, "confIndex", LOG);

    // wait till the old leader persist the new conf
    JavaUtils.attempt(() -> log.getLatestFlushedIndex() >= confIndex,
        10, 500L, "FLUSH", LOG);
    final long committed = log.getLastCommittedIndex();
    Assert.assertTrue(committed < confIndex);

    // unblock the old leader
    BlockRequestHandlingInjection.getInstance().unblockReplier(leaderId.toString());
    cluster.setBlockRequestsFrom(leaderId.toString(), false);

    // the client should get NotLeaderException
    clientThread.join(5000);
    Assert.assertTrue(gotNotLeader.get());

    // the old leader should have truncated the setConf from the log
    JavaUtils.attempt(() -> log.getLastCommittedIndex() >= confIndex,
        10, 500L, "COMMIT", LOG);
    Assert.assertTrue(log.get(confIndex).hasConfigurationEntry());
    log2 = null;
  } finally {
    RaftStorageTestUtils.printLog(log2, s -> LOG.info(s));
  }
}
 

void assertIndicesMultipleAttempts(RaftLog raftLog, long expectedFlushIndex, long expectedNextIndex) throws Exception {
  JavaUtils.attempt(() -> assertIndices(raftLog, expectedFlushIndex, expectedNextIndex),
      10, 100, "assertIndices", LOG);
}
 

private void testRestartFollower(CLUSTER cluster) throws Exception {
  leaderSnapshotInfoRef.set(null);
  int i = 0;
  final RaftServerImpl leader = RaftTestUtil.waitForLeader(cluster);
  final RaftPeerId leaderId = leader.getId();

  try (final RaftClient client = cluster.createClient(leaderId)) {
    for (; i < SNAPSHOT_TRIGGER_THRESHOLD * 2 - 1; i++) {
      final RaftClientReply reply = client.send(new RaftTestUtil.SimpleMessage("m" + i));
      Assert.assertTrue(reply.isSuccess());
    }
  }

  // wait for the snapshot to be done
  final long oldLeaderNextIndex = leader.getState().getLog().getNextIndex();
  {
    LOG.info("{}: oldLeaderNextIndex = {}", leaderId, oldLeaderNextIndex);
    final List<File> snapshotFiles = RaftSnapshotBaseTest.getSnapshotFiles(cluster,
        oldLeaderNextIndex - SNAPSHOT_TRIGGER_THRESHOLD, oldLeaderNextIndex);
    JavaUtils.attemptRepeatedly(() -> {
      Assert.assertTrue(snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists));
      return null;
    }, 10, ONE_SECOND, "snapshotFile.exist", LOG);
  }

  final RaftPeerId followerId = cluster.getFollowers().get(0).getId();
  cluster.killServer(followerId);

  // generate some more traffic
  try (final RaftClient client = cluster.createClient(leader.getId())) {
    Assert.assertTrue(client.send(new RaftTestUtil.SimpleMessage("m" + i)).isSuccess());
  }

  FIVE_SECONDS.sleep();
  cluster.restartServer(followerId, false);
  final RaftServerImpl follower = cluster.getRaftServerImpl(followerId);
  JavaUtils.attempt(() -> {
    final long newLeaderNextIndex = leader.getState().getLog().getNextIndex();
    LOG.info("{}: newLeaderNextIndex = {}", leaderId, newLeaderNextIndex);
    Assert.assertTrue(newLeaderNextIndex > oldLeaderNextIndex);
    Assert.assertEquals(newLeaderNextIndex, follower.getState().getLog().getNextIndex());
  }, 10, ONE_SECOND, "followerNextIndex", LOG);

}
 

void runTestRestartCommitIndex(MiniRaftCluster cluster) throws Exception {
  final SimpleMessage[] messages = SimpleMessage.create(100);
  final List<CompletableFuture<Void>> futures = new ArrayList<>(messages.length);
  for(int i = 0; i < messages.length; i++) {
    final CompletableFuture<Void> f = new CompletableFuture<>();
    futures.add(f);

    final SimpleMessage m = messages[i];
    new Thread(() -> {
      try (final RaftClient client = cluster.createClient()) {
        Assert.assertTrue(client.send(m).isSuccess());
      } catch (IOException e) {
        throw new IllegalStateException("Failed to send " + m, e);
      }
      f.complete(null);
    }).start();
  }
  JavaUtils.allOf(futures).get();

  final List<RaftPeerId> ids = new ArrayList<>();
  final RaftServerImpl leader = cluster.getLeader();
  final RaftLog leaderLog = leader.getState().getLog();
  final RaftPeerId leaderId = leader.getId();
  ids.add(leaderId);

  RaftTestUtil.getStateMachineLogEntries(leaderLog);

  // check that the last metadata entry is written to the log
  JavaUtils.attempt(() -> assertLastLogEntry(leader), 20, HUNDRED_MILLIS, "leader last metadata entry", LOG);

  final long lastIndex = leaderLog.getLastEntryTermIndex().getIndex();
  LOG.info("{}: leader lastIndex={}", leaderId, lastIndex);
  final LogEntryProto lastEntry = leaderLog.get(lastIndex);
  LOG.info("{}: leader lastEntry entry[{}] = {}", leaderId, lastIndex, ServerProtoUtils.toLogEntryString(lastEntry));
  final long loggedCommitIndex = lastEntry.getMetadataEntry().getCommitIndex();
  final LogEntryProto lastCommittedEntry = leaderLog.get(loggedCommitIndex);
  LOG.info("{}: leader lastCommittedEntry = entry[{}] = {}",
      leaderId, loggedCommitIndex, ServerProtoUtils.toLogEntryString(lastCommittedEntry));

  final SimpleStateMachine4Testing leaderStateMachine = SimpleStateMachine4Testing.get(leader);
  final TermIndex lastAppliedTermIndex = leaderStateMachine.getLastAppliedTermIndex();
  LOG.info("{}: leader lastAppliedTermIndex = {}", leaderId, lastAppliedTermIndex);

  // check follower logs
  for(RaftServerImpl s : cluster.iterateServerImpls()) {
    if (!s.getId().equals(leaderId)) {
      ids.add(s.getId());
      RaftTestUtil.assertSameLog(leaderLog, s.getState().getLog());
    }
  }

  // take snapshot and truncate last (metadata) entry
  leaderStateMachine.takeSnapshot();
  leaderLog.truncate(lastIndex);

  // kill all servers
  ids.forEach(cluster::killServer);

  // Restart and kill servers one by one so that they won't talk to each other.
  for(RaftPeerId id : ids) {
    cluster.restartServer(id, false);
    final RaftServerImpl server = cluster.getRaftServerImpl(id);
    final RaftLog raftLog = server.getState().getLog();
    JavaUtils.attemptRepeatedly(() -> {
      Assert.assertTrue(raftLog.getLastCommittedIndex() >= loggedCommitIndex);
      return null;
    }, 10, HUNDRED_MILLIS, id + "(commitIndex >= loggedCommitIndex)", LOG);
    JavaUtils.attemptRepeatedly(() -> {
      Assert.assertTrue(server.getState().getLastAppliedIndex() >= loggedCommitIndex);
      return null;
    }, 10, HUNDRED_MILLIS, id + "(lastAppliedIndex >= loggedCommitIndex)", LOG);
    LOG.info("{}: commitIndex={}, lastAppliedIndex={}",
        id, raftLog.getLastCommittedIndex(), server.getState().getLastAppliedIndex());
    cluster.killServer(id);
  }
}
 

void assertIndicesMultipleAttempts(RaftLog raftLog, long expectedFlushIndex, long expectedNextIndex) throws Exception {
  JavaUtils.attempt(() -> assertIndices(raftLog, expectedFlushIndex, expectedNextIndex),
      10, HUNDRED_MILLIS, "assertIndices", LOG);
}