org.apache.ratis.server.impl.RaftServerImpl Java Examples

@Test
public void testLeaderElectionDetection() throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  final TimeDuration noLeaderTimeout = RaftServerConfigKeys.Notification.noLeaderTimeout(cluster.getProperties());

  RaftServerImpl healthyFollower = cluster.getFollowers().get(1);
  RaftServerImpl failedFollower = cluster.getFollowers().get(0);
  // fail the leader and one of the followers to that quorum is not present
  // for next leader election to succeed.
  cluster.killServer(failedFollower.getId());
  cluster.killServer(cluster.getLeader().getId());

  // Wait to ensure that leader election is triggered and also state machine callback is triggered
  noLeaderTimeout.sleep();
  noLeaderTimeout.sleep();

  RaftProtos.RoleInfoProto roleInfoProto =
      SimpleStateMachine4Testing.get(healthyFollower).getLeaderElectionTimeoutInfo();
  Assert.assertNotNull(roleInfoProto);

  Assert.assertEquals(roleInfoProto.getRole(), RaftProtos.RaftPeerRole.CANDIDATE);
  final long noLeaderTimeoutMs = noLeaderTimeout.toLong(TimeUnit.MILLISECONDS);
  Assert.assertTrue(roleInfoProto.getCandidateInfo().getLastLeaderElapsedTimeMs() > noLeaderTimeoutMs);
}
 

/**
 * @return the list of leaders with the highest term (i.e. leaders with a lower term are not included).
 *         from the given group.
 */
private List<RaftServerImpl> getLeaders(RaftGroupId groupId) {
  final Stream<RaftServerImpl> serverAliveStream = getServerAliveStream(groupId);
  final List<RaftServerImpl> leaders = new ArrayList<>();
  serverAliveStream.filter(RaftServerImpl::isLeader).forEach(s -> {
    if (leaders.isEmpty()) {
      leaders.add(s);
    } else {
      final long leaderTerm = leaders.get(0).getState().getCurrentTerm();
      final long term = s.getState().getCurrentTerm();
      if (term >= leaderTerm) {
        if (term > leaderTerm) {
          leaders.clear();
        }
        leaders.add(s);
      }
    }
  });
  return leaders;
}
 

/**
 * @return the list of leaders with the highest term (i.e. leaders with a lower term are not included).
 *         from the given group.
 */
private List<RaftServerImpl> getLeaders(RaftGroupId groupId) {
  final Stream<RaftServerImpl> serverAliveStream = getServerAliveStream(groupId);
  final List<RaftServerImpl> leaders = new ArrayList<>();
  serverAliveStream.filter(RaftServerImpl::isLeader).forEach(s -> {
    if (leaders.isEmpty()) {
      leaders.add(s);
    } else {
      final long leaderTerm = leaders.get(0).getState().getCurrentTerm();
      final long term = s.getState().getCurrentTerm();
      if (term >= leaderTerm) {
        if (term > leaderTerm) {
          leaders.clear();
        }
        leaders.add(s);
      }
    }
  });
  return leaders;
}
 

static void assertLeaderContent(MiniRaftCluster cluster) throws Exception {
  final RaftServerImpl leader = RaftTestUtil.waitForLeader(cluster);
  final RaftLog leaderLog = leader.getState().getLog();
  final long lastIndex = leaderLog.getLastEntryTermIndex().getIndex();
  final LogEntryProto e = leaderLog.get(lastIndex);
  Assert.assertTrue(e.hasMetadataEntry());
  Assert.assertEquals(leaderLog.getLastCommittedIndex() - 1, e.getMetadataEntry().getCommitIndex());

  final LogEntryProto[] entries = SimpleStateMachine4Testing.get(leader).getContent();
  long message = 0;
  for (int i = 0; i < entries.length; i++) {
    LOG.info("{}) {} {}", i, message, entries[i]);
    if (entries[i].hasStateMachineLogEntry()) {
      final SimpleMessage m = new SimpleMessage("m" + message++);
      Assert.assertArrayEquals(m.getContent().toByteArray(),
          entries[i].getStateMachineLogEntry().getLogData().toByteArray());
    }
  }
}
 

public static void assertLeaderContent(MiniRaftCluster cluster) throws Exception {
  final RaftServerImpl leader = RaftTestUtil.waitForLeader(cluster);
  final RaftLog leaderLog = leader.getState().getLog();
  final long lastIndex = leaderLog.getLastEntryTermIndex().getIndex();
  final LogEntryProto e = leaderLog.get(lastIndex);
  Assert.assertTrue(e.hasMetadataEntry());

  JavaUtils.attemptRepeatedly(() -> {
    Assert.assertEquals(leaderLog.getLastCommittedIndex() - 1, e.getMetadataEntry().getCommitIndex());
    return null;
  }, 50, BaseTest.HUNDRED_MILLIS, "CheckMetadataEntry", LOG);

  SimpleStateMachine4Testing simpleStateMachine = SimpleStateMachine4Testing.get(leader);
  Assert.assertTrue("Is not notified as a leader", simpleStateMachine.isNotifiedAsLeader());
  final LogEntryProto[] entries = simpleStateMachine.getContent();
  long message = 0;
  for (int i = 0; i < entries.length; i++) {
    LOG.info("{}) {} {}", i, message, entries[i]);
    if (entries[i].hasStateMachineLogEntry()) {
      final SimpleMessage m = new SimpleMessage("m" + message++);
      Assert.assertArrayEquals(m.getContent().toByteArray(),
          entries[i].getStateMachineLogEntry().getLogData().toByteArray());
    }
  }
}
 

static void runTestRaftLogMetrics(MiniRaftCluster cluster) throws Exception {
  int numMsg = 2;
  final RaftTestUtil.SimpleMessage[] messages = RaftTestUtil.SimpleMessage.create(numMsg);

  try (final RaftClient client = cluster.createClient()) {
    for (RaftTestUtil.SimpleMessage message : messages) {
      client.send(message);
    }
  }

  // For leader, flush must happen before client can get replies.
  assertFlushCount(cluster.getLeader());
  assertRaftLogWritePathMetrics(cluster.getLeader());

  // For followers, flush can be lagged behind.  Attempt multiple times.
  for(RaftServerImpl f : cluster.getFollowers()) {
    JavaUtils.attempt(() -> assertFlushCount(f), 10, HUNDRED_MILLIS, f.getId() + "-assertFlushCount", null);
    // We have already waited enough for follower metrics to populate.
    assertRaftLogWritePathMetrics(f);
  }

  // Wait for commits to happen on leader
  JavaUtils.attempt(() -> assertCommitCount(cluster.getLeader(), numMsg), 10, HUNDRED_MILLIS, cluster.getLeader().getId() + "-assertCommitCount", null);
}
 

static void assertCorruptedLogHeader(RaftPeerId id, File openLogFile, int partialLength,
    MiniRaftCluster cluster, Logger LOG) throws Exception {
  Preconditions.assertTrue(partialLength < SegmentedRaftLogFormat.getHeaderLength());
  try(final RandomAccessFile raf = new RandomAccessFile(openLogFile, "rw")) {
    SegmentedRaftLogFormat.applyHeaderTo(header -> {
      LOG.info("header    = {}", StringUtils.bytes2HexString(header));
      final byte[] corrupted = new byte[header.length];
      System.arraycopy(header, 0, corrupted, 0, partialLength);
      LOG.info("corrupted = {}", StringUtils.bytes2HexString(corrupted));
      raf.write(corrupted);
      return null;
    });
  }
  final RaftServerImpl server = cluster.restartServer(id, false);
  server.getProxy().close();
}
 

static void runTestFlushMetric(MiniRaftCluster cluster) throws Exception {
  int numMsg = 2;
  final RaftTestUtil.SimpleMessage[] messages = RaftTestUtil.SimpleMessage.create(numMsg);

  try (final RaftClient client = cluster.createClient()) {
    for (RaftTestUtil.SimpleMessage message : messages) {
      client.send(message);
    }
  }

  // For leader, flush must happen before client can get replies.
  assertFlushCount(cluster.getLeader());

  // For followers, flush can be lagged behind.  Attempt multiple times.
  for(RaftServerImpl f : cluster.getFollowers()) {
    JavaUtils.attempt(() -> assertFlushCount(f), 10, 100, f.getId() + "-assertFlushCount", null);
  }
}
 

/**
 * When the server is null, return the dummy instance of {@link ServerLogMethods}.
 * Otherwise, the server is non-null, return the implementation using the given server.
 */
private ServerLogMethods newServerLogMethods(RaftServerImpl impl) {
  if (impl == null) {
    return ServerLogMethods.DUMMY;
  }

  return new ServerLogMethods() {
    @Override
    public boolean shouldEvictCache() {
      return cache.shouldEvict();
    }

    @Override
    public long[] getFollowerNextIndices() {
      return impl.getFollowerNextIndices();
    }

    @Override
    public long getLastAppliedIndex() {
      return impl.getState().getLastAppliedIndex();
    }

    @Override
    public void notifyTruncatedLogEntry(TermIndex ti) {
      try {
        final LogEntryProto entry = get(ti.getIndex());
        impl.notifyTruncatedLogEntry(entry);
      } catch (RaftLogIOException e) {
        LOG.error("{}: Failed to read log {}", getName(), ti, e);
      }
    }
  };
}
 

static void runTestWatchRequestAsyncChangeLeader(TestParameters p) throws Exception {
  final Logger LOG = p.log;
  final MiniRaftCluster cluster = p.cluster;
  final int numMessages = p.numMessages;

  // blockFlushStateMachineData a follower so that no transaction can be ALL_COMMITTED
  final List<RaftServerImpl> followers = cluster.getFollowers();
  final RaftServerImpl blockedFollower = followers.get(ThreadLocalRandom.current().nextInt(followers.size()));
  LOG.info("block follower {}", blockedFollower.getId());
  SimpleStateMachine4Testing.get(blockedFollower).blockFlushStateMachineData();

  final List<CompletableFuture<RaftClientReply>> replies = new ArrayList<>();
  final List<CompletableFuture<WatchReplies>> watches = new ArrayList<>();

  p.sendRequests(replies, watches);

  Assert.assertEquals(numMessages, replies.size());
  Assert.assertEquals(numMessages, watches.size());

  // since only one follower is blocked, requests can be committed MAJORITY but neither ALL nor ALL_COMMITTED.
  checkMajority(replies, watches, LOG);

  TimeUnit.SECONDS.sleep(1);
  assertNotDone(watches.stream().map(CompletableFuture::join).map(w -> w.all));
  assertNotDone(watches.stream().map(CompletableFuture::join).map(w -> w.allCommitted));

  // Now change leader
  RaftTestUtil.changeLeader(cluster, cluster.getLeader().getId());

  // unblock follower so that the transaction can be replicated and committed to all.
  SimpleStateMachine4Testing.get(blockedFollower).unblockFlushStateMachineData();
  LOG.info("unblock follower {}", blockedFollower.getId());
  checkAll(watches, LOG);
}
 

SegmentedRaftLog(RaftGroupMemberId memberId, RaftServerImpl server,
    StateMachine stateMachine, Runnable submitUpdateCommitEvent,
    RaftStorage storage, long lastIndexInSnapshot, RaftProperties properties) {
  super(memberId, lastIndexInSnapshot, properties);
  this.server = newServerLogMethods(server);
  this.storage = storage;
  this.stateMachine = stateMachine;
  segmentMaxSize = RaftServerConfigKeys.Log.segmentSizeMax(properties).getSize();
  this.cache = new SegmentedRaftLogCache(memberId, storage, properties, getRaftLogMetrics());
  this.fileLogWorker = new SegmentedRaftLogWorker(memberId, stateMachine,
      submitUpdateCommitEvent, server, storage, properties, getRaftLogMetrics());
  stateMachineCachingEnabled = RaftServerConfigKeys.Log.StateMachineData.cachingEnabled(properties);
}
 

void testRaftClientRequestMetrics(MiniRaftClusterWithGrpc cluster) throws IOException,
    ExecutionException, InterruptedException {
  final RaftServerImpl leader = RaftTestUtil.waitForLeader(cluster);
  RaftServerMetrics raftServerMetrics = leader.getRaftServerMetrics();

  try (final RaftClient client = cluster.createClient()) {
    final CompletableFuture<RaftClientReply> f1 = client.sendAsync(new SimpleMessage("testing"));
    Assert.assertTrue(f1.get().isSuccess());
    Assert.assertTrue(raftServerMetrics.getTimer(RAFT_CLIENT_WRITE_REQUEST).getCount() > 0);

    final CompletableFuture<RaftClientReply> f2 = client.sendReadOnlyAsync(new SimpleMessage("testing"));
    Assert.assertTrue(f2.get().isSuccess());
    Assert.assertTrue(raftServerMetrics.getTimer(RAFT_CLIENT_READ_REQUEST).getCount() > 0);

    final CompletableFuture<RaftClientReply> f3 = client.sendStaleReadAsync(new SimpleMessage("testing"),
        0, leader.getId());
    Assert.assertTrue(f3.get().isSuccess());
    Assert.assertTrue(raftServerMetrics.getTimer(RAFT_CLIENT_STALE_READ_REQUEST).getCount() > 0);

    final CompletableFuture<RaftClientReply> f4 = client.sendWatchAsync(0, RaftProtos.ReplicationLevel.ALL);
    Assert.assertTrue(f4.get().isSuccess());
    Assert.assertTrue(raftServerMetrics.getTimer(String.format(RAFT_CLIENT_WATCH_REQUEST, "-ALL")).getCount() > 0);

    final CompletableFuture<RaftClientReply> f5 = client.sendWatchAsync(0, RaftProtos.ReplicationLevel.MAJORITY);
    Assert.assertTrue(f5.get().isSuccess());
    Assert.assertTrue(raftServerMetrics.getTimer(String.format(RAFT_CLIENT_WATCH_REQUEST, "")).getCount() > 0);
  }
}
 

SegmentedRaftLogWorker(RaftGroupMemberId memberId, StateMachine stateMachine, Runnable submitUpdateCommitEvent,
                       RaftServerImpl server, RaftStorage storage, RaftProperties properties,
                       RaftLogMetrics metricRegistry) {
  this.name = memberId + "-" + getClass().getSimpleName();
  LOG.info("new {} for {}", name, storage);

  this.submitUpdateCommitEvent = submitUpdateCommitEvent;
  this.stateMachine = stateMachine;
  this.raftLogMetrics = metricRegistry;
  this.storage = storage;
  this.server = server;
  final SizeInBytes queueByteLimit = RaftServerConfigKeys.Log.queueByteLimit(properties);
  final int queueElementLimit = RaftServerConfigKeys.Log.queueElementLimit(properties);
  this.queue =
      new DataBlockingQueue<>(name, queueByteLimit, queueElementLimit, Task::getSerializedSize);

  this.segmentMaxSize = RaftServerConfigKeys.Log.segmentSizeMax(properties).getSize();
  this.preallocatedSize = RaftServerConfigKeys.Log.preallocatedSize(properties).getSize();
  this.bufferSize = RaftServerConfigKeys.Log.writeBufferSize(properties).getSizeInt();
  this.forceSyncNum = RaftServerConfigKeys.Log.forceSyncNum(properties);
  this.flushBatchSize = 0;

  this.stateMachineDataPolicy = new StateMachineDataPolicy(properties);

  this.workerThread = new Thread(this, name);

  // Server Id can be null in unit tests
  metricRegistry.addDataQueueSizeGauge(queue);
  metricRegistry.addLogWorkerQueueSizeGauge(writeTasks.q);
  metricRegistry.addFlushBatchSizeGauge(() -> (Gauge<Integer>) () -> flushBatchSize);
  this.logFlushTimer = metricRegistry.getFlushTimer();
  this.raftLogSyncTimer = metricRegistry.getRaftLogSyncTimer();
  this.raftLogQueueingTimer = metricRegistry.getRaftLogQueueTimer();
  this.raftLogEnqueueingDelayTimer = metricRegistry.getRaftLogEnqueueDelayTimer();

  this.writeBuffer = ByteBuffer.allocateDirect(bufferSize);
}
 

public RaftServerImpl restartServer(RaftPeerId serverId, RaftGroup group, boolean format) throws IOException {
  killServer(serverId);
  servers.remove(serverId);

  final RaftServerProxy proxy = putNewServer(serverId, group, format);
  proxy.start();
  return group == null? null: proxy.getImpl(group.getGroupId());
}
 

static void assertLogEntries(RaftServerImpl server, long expectedTerm, SimpleMessage... expectedMessages) {
  LOG.info("checking raft log for " + server.getId());
  final RaftLog log = server.getState().getLog();
  try {
    RaftTestUtil.assertLogEntries(log, expectedTerm, expectedMessages);
  } catch (AssertionError e) {
    LOG.error(server.getId() + ": Unexpected raft log", e);
    throw e;
  }
}
 

public RaftServerImpl restartServer(RaftPeerId newId, RaftGroup group, boolean format) throws IOException {
  killServer(newId);
  servers.remove(newId);

  final RaftServerProxy proxy = putNewServer(newId, group, format);
  proxy.start();
  return group == null? null: proxy.getImpl(group.getGroupId());
}
 

@Before
public void setup() {
  path = GenericTestUtils
          .getTempPath(TestOzoneClientKeyGenerator.class.getSimpleName());
  GenericTestUtils.setLogLevel(RaftLog.LOG, Level.DEBUG);
  GenericTestUtils.setLogLevel(RaftServerImpl.LOG, Level.DEBUG);
  File baseDir = new File(path);
  baseDir.mkdirs();
}
 

public String printAllLogs() {
  StringBuilder b = new StringBuilder("\n#servers = " + servers.size() + "\n");
  for (RaftServerImpl s : iterateServerImpls()) {
    b.append("  ");
    b.append(s).append("\n");

    final RaftLog log = s.getState().getLog();
    if (log instanceof MemoryRaftLog) {
      b.append("    ");
      b.append(((MemoryRaftLog) log).getEntryString());
    }
  }
  return b.toString();
}
 

static void assertTruncatedLog(RaftPeerId id, File openLogFile, long lastIndex, MiniRaftCluster cluster) throws Exception {
  // truncate log
  FileUtils.truncateFile(openLogFile, openLogFile.length() - 1);
  final RaftServerImpl server = cluster.restartServer(id, false);
  // the last index should be one less than before
  Assert.assertEquals(lastIndex - 1, server.getState().getLog().getLastEntryTermIndex().getIndex());
  server.getProxy().close();
}
 

private void runTestWriteAndFlush(CLUSTER cluster) throws Exception {
  final int bufferSize = ByteValue.BUFFERSIZE;
  RaftServerImpl leader = waitForLeader(cluster);
  OutputStream out = newOutputStream(cluster, bufferSize);

  int[] lengths = new int[]{1, 500, 1023, 1024, 1025, 2048, 3000, 3072};
  ByteValue[] values = new ByteValue[lengths.length];
  for (int i = 0; i < values.length; i++) {
    values[i] = new ByteValue(lengths[i], (byte) 9);
  }

  List<byte[]> expectedTxs = new ArrayList<>();
  for (ByteValue v : values) {
    byte[] data = v.genData();
    expectedTxs.addAll(v.getTransactions());
    out.write(data);
    out.flush();

    // make sure after the flush the data has been committed
    assertRaftLog(expectedTxs.size(), leader);
  }
  out.close();

  try {
    out.write(0);
    fail("The OutputStream has been closed");
  } catch (IOException ignored) {
  }

  LOG.info("Start to check leader's log");
  final AtomicInteger index = new AtomicInteger(0);
  checkLog(leader.getState().getLog(), expectedTxs.size(),
      () -> expectedTxs.get(index.getAndIncrement()));
}
 

private RaftLog assertRaftLog(int expectedEntries, RaftServerImpl server) throws Exception {
  final RaftLog raftLog = server.getState().getLog();
  final EnumMap<LogEntryBodyCase, AtomicLong> counts = RaftTestUtil.countEntries(raftLog);
  Assert.assertEquals(expectedEntries, counts.get(LogEntryBodyCase.STATEMACHINELOGENTRY).get());

  final LogEntryProto last = RaftTestUtil.getLastEntry(LogEntryBodyCase.STATEMACHINELOGENTRY, raftLog);
  Assert.assertNotNull(last);
  Assert.assertTrue(raftLog.getLastCommittedIndex() >= last.getIndex());
  Assert.assertTrue(server.getState().getLastAppliedIndex() >= last.getIndex());
  return raftLog;
}
 

public static List<File> getSnapshotFiles(MiniRaftCluster cluster, long startIndex, long endIndex) {
  final RaftServerImpl leader = cluster.getLeader();
  final SimpleStateMachineStorage storage = SimpleStateMachine4Testing.get(leader).getStateMachineStorage();
  final long term = leader.getState().getCurrentTerm();
  return LongStream.range(startIndex, endIndex)
      .mapToObj(i -> storage.getSnapshotFile(term, i))
      .collect(Collectors.toList());
}
 

SegmentedRaftLog(RaftPeerId selfId, RaftServerImpl server,
    StateMachine stateMachine, Runnable submitUpdateCommitEvent,
    RaftStorage storage, long lastIndexInSnapshot, RaftProperties properties) {
  super(selfId, lastIndexInSnapshot, RaftServerConfigKeys.Log.Appender.bufferByteLimit(properties).getSizeInt());
  this.server = Optional.ofNullable(server);
  this.storage = storage;
  segmentMaxSize = RaftServerConfigKeys.Log.segmentSizeMax(properties).getSize();
  cache = new RaftLogCache(selfId, storage, properties);
  this.fileLogWorker = new RaftLogWorker(selfId, stateMachine, submitUpdateCommitEvent, storage, properties);
  stateMachineCachingEnabled = RaftServerConfigKeys.Log.StateMachineData.cachingEnabled(properties);
}
 

@BeforeClass
public static void setUp() throws Exception {
  RaftServerImpl raftServer = mock(RaftServerImpl.class);
  ServerState serverStateMock = mock(ServerState.class);
  when(raftServer.getState()).thenReturn(serverStateMock);
  when(serverStateMock.getLastLeaderElapsedTimeMs()).thenReturn(1000L);
  RaftGroupId raftGroupId = RaftGroupId.randomId();
  RaftPeerId raftPeerId = RaftPeerId.valueOf("TestId");
  RaftGroupMemberId raftGroupMemberId = RaftGroupMemberId.valueOf(raftPeerId, raftGroupId);
  when(raftServer.getMemberId()).thenReturn(raftGroupMemberId);
  leaderElectionMetrics = LeaderElectionMetrics.getLeaderElectionMetrics(raftServer);
  ratisMetricRegistry = leaderElectionMetrics.getRegistry();
}
 

static List<File> getSnapshotFiles(MiniRaftCluster cluster, long startIndex, long endIndex) {
  final RaftServerImpl leader = cluster.getLeader();
  final SimpleStateMachineStorage storage = SimpleStateMachine4Testing.get(leader).getStateMachineStorage();
  final long term = leader.getState().getCurrentTerm();
  return LongStream.range(startIndex, endIndex)
      .mapToObj(i -> storage.getSnapshotFile(term, i))
      .collect(Collectors.toList());
}
 

static RaftServerImpl waitForLeader(
    MiniRaftCluster cluster, RaftGroupId groupId, boolean expectLeader)
    throws InterruptedException {
  final String name = "waitForLeader-" + groupId + "-(expectLeader? " + expectLeader + ")";
  final int numAttempts = expectLeader? 100: 10;
  final TimeDuration sleepTime = cluster.getTimeoutMax().apply(d -> (d * 3) >> 1);
  LOG.info(cluster.printServers(groupId));

  final AtomicReference<IllegalStateException> exception = new AtomicReference<>();
  final Runnable handleNoLeaders = () -> {
    throw cluster.newIllegalStateExceptionForNoLeaders(groupId);
  };
  final Consumer<List<RaftServerImpl>> handleMultipleLeaders = leaders -> {
    final IllegalStateException ise = cluster.newIllegalStateExceptionForMultipleLeaders(groupId, leaders);
    exception.set(ise);
  };

  final RaftServerImpl leader = JavaUtils.attemptRepeatedly(() -> {
    RaftServerImpl l = cluster.getLeader(groupId, handleNoLeaders, handleMultipleLeaders);
    if (l != null && !l.isLeaderReady()) {
      throw new IllegalStateException("Leader: "+ l.getMemberId() +  " not ready");
    }
    return l;
  }, numAttempts, sleepTime, name, LOG);

  LOG.info(cluster.printServers(groupId));
  if (expectLeader) {
    return Optional.ofNullable(leader).orElseThrow(exception::get);
  } else {
    if (leader == null) {
      return null;
    } else {
      throw new IllegalStateException("expectLeader = " + expectLeader + " but leader = " + leader);
    }
  }
}
 

static RaftPeerId waitAndKillLeader(MiniRaftCluster cluster) throws InterruptedException {
  final RaftServerImpl leader = waitForLeader(cluster);
  Assert.assertNotNull(leader);

  LOG.info("killing leader = " + leader);
  cluster.killServer(leader.getId());
  return leader.getId();
}
 

public void assertServer(MiniRaftCluster cluster, ClientId clientId, long callId, long oldLastApplied) throws Exception {
  long leaderApplied = cluster.getLeader().getState().getLastAppliedIndex();
  // make sure retry cache has the entry
  for (RaftServerImpl server : cluster.iterateServerImpls()) {
    LOG.info("check server " + server.getId());
    if (server.getState().getLastAppliedIndex() < leaderApplied) {
      Thread.sleep(1000);
    }
    Assert.assertEquals(2, RaftServerTestUtil.getRetryCacheSize(server));
    Assert.assertNotNull(RaftServerTestUtil.getRetryEntry(server, clientId, callId));
    // make sure there is only one log entry committed
    Assert.assertEquals(1, count(server.getState().getLog(), oldLastApplied + 1));
  }
}
 

@Test
public void testSlownessDetection() throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  long slownessTimeout = RaftServerConfigKeys.Rpc
      .slownessTimeout(cluster.getProperties()).toIntExact(TimeUnit.MILLISECONDS);
  RaftServerImpl failedFollower = cluster.getFollowers().get(0);

  // fail the node and wait for the callback to be triggered
  cluster.killServer(failedFollower.getId());
  Thread.sleep( slownessTimeout * 2);

  // Followers should not get any failed not notification
  for (RaftServerImpl followerServer : cluster.getFollowers()) {
    Assert.assertNull(SimpleStateMachine4Testing.get(followerServer).getSlownessInfo());
  }
  // the leader should get notification that the follower has failed now
  RaftProtos.RoleInfoProto roleInfoProto =
      SimpleStateMachine4Testing.get(cluster.getLeader()).getSlownessInfo();
  Assert.assertNotNull(roleInfoProto);

  List<RaftProtos.ServerRpcProto> followers =
      roleInfoProto.getLeaderInfo().getFollowerInfoList();
  //Assert that the node shutdown is lagging behind
  for (RaftProtos.ServerRpcProto serverProto : followers) {
    if (RaftPeerId.valueOf(serverProto.getId().getId()).equals(failedFollower.getId())) {
      Assert.assertTrue(serverProto.getLastRpcElapsedTimeMs() > slownessTimeout);
    }
  }
}
 

@Test
public void testStateMachineRegistry() throws Throwable {
  final Map<RaftGroupId, StateMachine> registry = new ConcurrentHashMap<>();
  registry.put(RaftGroupId.randomId(), new SimpleStateMachine4Testing());
  registry.put(RaftGroupId.randomId(), new SMTransactionContext());

  try(MiniRaftClusterWithSimulatedRpc cluster = newCluster(0)) {
    cluster.setStateMachineRegistry(registry::get);

    final RaftPeerId id = RaftPeerId.valueOf("s0");
    cluster.putNewServer(id, null, true);
    cluster.start();

    for(RaftGroupId gid : registry.keySet()) {
      final RaftGroup newGroup = RaftGroup.valueOf(gid, cluster.getPeers());
      LOG.info("add new group: " + newGroup);
      final RaftClient client = cluster.createClient(newGroup);
      for(RaftPeer p : newGroup.getPeers()) {
        client.groupAdd(newGroup, p.getId());
      }
    }

    final RaftServerProxy proxy = cluster.getServer(id);
    for(Map.Entry<RaftGroupId, StateMachine> e: registry.entrySet()) {
      final RaftServerImpl impl = RaftServerTestUtil.getRaftServerImpl(proxy, e.getKey());
      Assert.assertSame(e.getValue(), impl.getStateMachine());
    }
  }
}