org.apache.ratis.client.RaftClient Java Examples

@Test
public void testAsyncConfiguration() throws IOException {
  LOG.info("Running testAsyncConfiguration");
  final RaftProperties properties = new RaftProperties();
  RaftClient.Builder clientBuilder = RaftClient.newBuilder()
      .setRaftGroup(RaftGroup.emptyGroup())
      .setProperties(properties);
  int numThreads = RaftClientConfigKeys.Async.SCHEDULER_THREADS_DEFAULT;
  int maxOutstandingRequests = RaftClientConfigKeys.Async.MAX_OUTSTANDING_REQUESTS_DEFAULT;
  try(RaftClient client = clientBuilder.build()) {
    RaftClientTestUtil.assertScheduler(client, numThreads);
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, maxOutstandingRequests, 0);
  }

  numThreads = 200;
  maxOutstandingRequests = 5;
  RaftClientConfigKeys.Async.setMaxOutstandingRequests(properties, maxOutstandingRequests);
  RaftClientConfigKeys.Async.setSchedulerThreads(properties, numThreads);
  try(RaftClient client = clientBuilder.build()) {
    RaftClientTestUtil.assertScheduler(client, numThreads);
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, maxOutstandingRequests, 0);
  }
}
 

public static void runTestPythagorean(
    RaftClient client, int start, int count) throws IOException {
  Preconditions.assertTrue(count > 0, () -> "count = " + count + " <= 0");
  Preconditions.assertTrue(start >= 2, () -> "start = " + start + " < 2");

  final Variable a = new Variable("a");
  final Variable b = new Variable("b");
  final Variable c = new Variable("c");
  final Expression pythagorean = SQRT.apply(ADD.apply(SQUARE.apply(a), SQUARE.apply(b)));

  final int end = start + 2*count;
  for(int n = (start & 1) == 0? start + 1: start; n < end; n += 2) {
    int n2 = n*n;
    int half_n2 = n2/2;

    assign(client, a, n);
    assign(client, b, half_n2);
    assign(client, c, pythagorean, (double)half_n2 + 1);

    assignNull(client, a);
    assignNull(client, b);
    assignNull(client, c);
  }
}
 

@Test
public void testSeveralCounter() throws IOException, InterruptedException {
  setAndStart(cluster);
  try (final RaftClient client = cluster.createClient()) {
    for (int i = 0; i < 10; i++) {
      client.send(Message.valueOf("INCREMENT"));
    }
    RaftClientReply reply1 = client.sendReadOnly(Message.valueOf("GET"));
    Assert.assertEquals("10",
        reply1.getMessage().getContent().toString(Charset.defaultCharset()));
    for (int i = 0; i < 10; i++) {
      client.send(Message.valueOf("INCREMENT"));
    }
    RaftClientReply reply2 = client.sendReadOnly(Message.valueOf("GET"));
    Assert.assertEquals("20",
        reply2.getMessage().getContent().toString(Charset.defaultCharset()));
    for (int i = 0; i < 10; i++) {
      client.send(Message.valueOf("INCREMENT"));
    }
    RaftClientReply reply3 = client.sendReadOnly(Message.valueOf("GET"));
    Assert.assertEquals("30",
        reply3.getMessage().getContent().toString(Charset.defaultCharset()));
  }
}
 

@Test
public void testHandleStateMachineException() throws Exception {
  final RaftProperties prop = getProperties();
  prop.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
      StateMachineWithException.class, StateMachine.class);
  final MiniRaftCluster cluster = newCluster(3);
  cluster.start();

  RaftPeerId leaderId = RaftTestUtil.waitForLeader(cluster).getId();

  try(final RaftClient client = cluster.createClient(leaderId)) {
    client.send(new RaftTestUtil.SimpleMessage("m"));
    fail("Exception expected");
  } catch (StateMachineException e) {
    e.printStackTrace();
    Assert.assertTrue(e.getCause().getMessage().contains("Fake Exception"));
  }
  cluster.shutdown();
}
 

void runTestGroupMismatchException(CLUSTER cluster) throws Exception {
  final RaftGroup clusterGroup = cluster.getGroup();
  Assert.assertEquals(NUM_PEERS, clusterGroup.getPeers().size());

  final RaftGroup anotherGroup = RaftGroup.valueOf(RaftGroupId.randomId(), clusterGroup.getPeers());
  Assert.assertNotEquals(clusterGroup.getGroupId(), anotherGroup.getGroupId());

  // Create client using another group
  try(RaftClient client = cluster.createClient(anotherGroup)) {
    testFailureCase("send(..) with client group being different from the server group",
        () -> client.send(Message.EMPTY),
        GroupMismatchException.class);

    testFailureCase("sendReadOnly(..) with client group being different from the server group",
        () -> client.sendReadOnly(Message.EMPTY),
        GroupMismatchException.class);

    testFailureCase("setConfiguration(..) with client group being different from the server group",
        () -> client.setConfiguration(RaftPeer.emptyArray()),
        GroupMismatchException.class);

    testFailureCase("groupRemove(..) with another group id",
        () -> client.groupRemove(anotherGroup.getGroupId(), false, clusterGroup.getPeers().iterator().next().getId()),
        GroupMismatchException.class);
  }
}
 

@Test
public void testAsyncConfiguration() throws IOException {
  LOG.info("Running testAsyncConfiguration");
  final RaftProperties properties = new RaftProperties();
  RaftClient.Builder clientBuilder = RaftClient.newBuilder()
      .setRaftGroup(RaftGroup.emptyGroup())
      .setProperties(properties);
  int maxOutstandingRequests = RaftClientConfigKeys.Async.OUTSTANDING_REQUESTS_MAX_DEFAULT;
  try(RaftClient client = clientBuilder.build()) {
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, maxOutstandingRequests, 0);
  }

  maxOutstandingRequests = 5;
  RaftClientConfigKeys.Async.setOutstandingRequestsMax(properties, maxOutstandingRequests);
  try(RaftClient client = clientBuilder.build()) {
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, maxOutstandingRequests, 0);
  }
}
 

@Override
public void run() throws Exception {
  RaftProperties raftProperties = new RaftProperties();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(getRaftGroupId())),
          getPeers());

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);


}
 

private void runTestTimeoutException(CLUSTER cluster) throws Exception {
  // send a message to make sure the cluster is working
  try(RaftClient client = cluster.createClient()) {
    final RaftClientReply reply = client.send(new SimpleMessage("m0"));
    Assert.assertTrue(reply.isSuccess());

    RaftClientConfigKeys.Rpc.setRequestTimeout(properties.get(), ONE_SECOND);
    // Block StartTransaction
    cluster.getServers().stream()
        .map(cluster::getRaftServerImpl)
        .map(SimpleStateMachine4Testing::get)
        .forEach(SimpleStateMachine4Testing::blockStartTransaction);
    final CompletableFuture<RaftClientReply> replyFuture = client.sendAsync(new SimpleMessage("m1"));
    FIVE_SECONDS.sleep();
    // Unblock StartTransaction
    cluster.getServers().stream()
        .map(cluster::getRaftServerImpl)
        .map(SimpleStateMachine4Testing::get)
        .forEach(SimpleStateMachine4Testing::unblockStartTransaction);
    // The request should succeed after start transaction is unblocked
    Assert.assertTrue(replyFuture.get(FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit()).isSuccess());
  }
}
 

void runTestHandleNotLeaderException(boolean killNewLeader, CLUSTER cluster) throws Exception {
  final RaftPeerId oldLeader = RaftTestUtil.waitForLeader(cluster).getId();
  try(final RaftClient client = cluster.createClient(oldLeader)) {
    sendMessage("m1", client);

    // enforce leader change
    final RaftPeerId newLeader = RaftTestUtil.changeLeader(cluster, oldLeader);

    if (killNewLeader) {
      // kill the new leader
      cluster.killServer(newLeader);
    }

    final RaftClientRpc rpc = client.getClientRpc();
    JavaUtils.attempt(() -> assertNotLeaderException(newLeader, "m2", oldLeader, rpc, cluster),
        10, ONE_SECOND, "assertNotLeaderException", LOG);

    sendMessage("m3", client);
  }
}
 

public static void runTestPythagorean(
    RaftClient client, int start, int count) throws IOException {
  Preconditions.assertTrue(count > 0, () -> "count = " + count + " <= 0");
  Preconditions.assertTrue(start >= 2, () -> "start = " + start + " < 2");

  final Variable a = new Variable("a");
  final Variable b = new Variable("b");
  final Variable c = new Variable("c");
  final Expression pythagorean = SQRT.apply(ADD.apply(SQUARE.apply(a), SQUARE.apply(b)));

  final int end = start + 2*count;
  for(int n = (start & 1) == 0? start + 1: start; n < end; n += 2) {
    int n2 = n*n;
    int half_n2 = n2/2;

    assign(client, a, n);
    assign(client, b, half_n2);
    assign(client, c, pythagorean, (double)half_n2 + 1);

    assignNull(client, a);
    assignNull(client, b);
    assignNull(client, c);
  }
}
 

void runTestBasicRetry(CLUSTER cluster) throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  final RaftPeerId leaderId = cluster.getLeaderAndSendFirstMessage(false).getId();
  long oldLastApplied = cluster.getLeader().getState().getLastAppliedIndex();

  try (final RaftClient client = cluster.createClient(leaderId)) {
    final RaftClientRpc rpc = client.getClientRpc();
    final long callId = 999;
    RaftClientRequest r = cluster.newRaftClientRequest(client.getId(), leaderId,
            callId, new SimpleMessage("message"));
    assertReply(rpc.sendRequest(r), client, callId);

    // retry with the same callId
    for (int i = 0; i < 5; i++) {
      assertReply(rpc.sendRequest(r), client, callId);
    }

    assertServer(cluster, client.getId(), callId, oldLastApplied);
  }
}
 

void runGaussLegendre(RaftClient client) throws IOException {
  defineVariable(client, "a0", 1);
  defineVariable(client, "b0", DIV.apply(1, SQRT.apply(2)));
  defineVariable(client, "t0", DIV.apply(1, 4));
  defineVariable(client, "p0", 1);

  double previous = 0;
  boolean converged = false;
  for(int i = 1; i < 8; i++) {
    final int i_1 = i - 1;
    final Variable a0 = new Variable("a" + i_1);
    final Variable b0 = new Variable("b" + i_1);
    final Variable t0 = new Variable("t" + i_1);
    final Variable p0 = new Variable("p" + i_1);
    final Variable a1 = defineVariable(client, "a"+i, DIV.apply(ADD.apply(a0, b0), 2));
    final Variable b1 = defineVariable(client, "b"+i, SQRT.apply(MULT.apply(a0, b0)));
    final Variable t1 = defineVariable(client, "t"+i, SUBTRACT.apply(t0, MULT.apply(p0, SQUARE.apply(SUBTRACT.apply(a0, a1)))));
    final Variable p1 = defineVariable(client, "p"+i, MULT.apply(2, p0));

    final Variable pi_i = new Variable("pi_"+i);
    final Expression e = assign(client, pi_i, DIV.apply(SQUARE.apply(a1), t0));
    final double pi = e.evaluate(null);

    if (converged) {
      Assert.assertTrue(pi == previous);
    } else if (pi == previous) {
      converged = true;
    }
    LOG.info("{} = {}, converged? {}", pi_i, pi, converged);
    previous = pi;
  }
  Assert.assertTrue(converged);
}
 

public static void main(String[] args)
    throws IOException, InterruptedException {
  //indicate the number of INCREMENT command, set 10 if no parameter passed
  int increment = args.length > 0 ? Integer.parseInt(args[0]) : 10;

  //build the counter cluster client
  RaftClient raftClient = buildClient();

  //use a executor service with 10 thread to send INCREMENT commands
  // concurrently
  ExecutorService executorService = Executors.newFixedThreadPool(10);

  //send INCREMENT commands concurrently
  System.out.printf("Sending %d increment command...\n", increment);
  for (int i = 0; i < increment; i++) {
    executorService.submit(() ->
        raftClient.send(Message.valueOf("INCREMENT")));
  }

  //shutdown the executor service and wait until they finish their work
  executorService.shutdown();
  executorService.awaitTermination(increment * 500, TimeUnit.MILLISECONDS);

  //send GET command and print the response
  RaftClientReply count = raftClient.sendReadOnly(Message.valueOf("GET"));
  String response = count.getMessage().getContent().toString(Charset.defaultCharset());
  System.out.println(response);
}
 

public RaftClient createClient(RaftPeerId leaderId, RaftGroup group,
    ClientId clientId, RetryPolicy retryPolicy) {
  RaftClient.Builder builder = RaftClient.newBuilder()
      .setClientId(clientId)
      .setRaftGroup(group)
      .setLeaderId(leaderId)
      .setProperties(properties)
      .setParameters(parameters)
      .setRetryPolicy(retryPolicy);
  return builder.build();
}
 

private void runTestRetryOnStateMachineException(CLUSTER cluster) throws Exception {
  RaftPeerId leaderId = RaftTestUtil.waitForLeader(cluster).getId();

  cluster.getLeaderAndSendFirstMessage(true);
  long oldLastApplied = cluster.getLeader().getState().getLastAppliedIndex();

  try (final RaftClient client = cluster.createClient(leaderId)) {
    final RaftClientRpc rpc = client.getClientRpc();
    final long callId = 999;
    final SimpleMessage message = new SimpleMessage("message");
    final RaftClientRequest r = cluster.newRaftClientRequest(client.getId(), leaderId, callId, message);
    RaftClientReply reply = rpc.sendRequest(r);
    Assert.assertFalse(reply.isSuccess());
    Assert.assertNotNull(reply.getStateMachineException());

    // retry with the same callId
    for (int i = 0; i < 5; i++) {
      reply = rpc.sendRequest(r);
      Assert.assertEquals(client.getId(), reply.getClientId());
      Assert.assertEquals(callId, reply.getCallId());
      Assert.assertFalse(reply.isSuccess());
      Assert.assertNotNull(reply.getStateMachineException());
    }

    for (RaftServerImpl server : cluster.iterateServerImpls()) {
      LOG.info("check server " + server.getId());

      JavaUtils.attemptRepeatedly(() -> {
        Assert.assertNotNull(
            RaftServerTestUtil.getRetryEntry(server, client.getId(), callId));
        return null;
      }, 5, BaseTest.ONE_SECOND, "GetRetryEntry", LOG);

      final RaftLog log = server.getState().getLog();
      RaftTestUtil.logEntriesContains(log, oldLastApplied + 1, log.getNextIndex(), message);
    }

    cluster.shutdown();
  }
}
 

private void runTestHandleStateMachineException(CLUSTER cluster) throws Exception {
  RaftPeerId leaderId = RaftTestUtil.waitForLeader(cluster).getId();

  try(final RaftClient client = cluster.createClient(leaderId)) {
    client.send(new RaftTestUtil.SimpleMessage("m"));
    fail("Exception expected");
  } catch (StateMachineException e) {
    e.printStackTrace();
    Assert.assertTrue(e.getCause().getMessage().contains("Fake Exception"));
  }
  cluster.shutdown();
}
 

private void runTestMultiRaftGroup(int[] idIndex, int chosen) throws Exception {

    final CheckedBiConsumer<MiniRaftCluster, RaftGroup, IOException> checker = (cluster, group) -> {
      try (final RaftClient client = cluster.createClient(group)) {
        TestArithmetic.runTestPythagorean(client, start.getAndAdd(2*count), count);
      }
    };

    GroupManagementBaseTest.runMultiGroupTest(
        cluster, idIndex, chosen, checker);
  }
 

@Test
public void testStateMachineRegistry() throws Throwable {
  final Map<RaftGroupId, StateMachine> registry = new ConcurrentHashMap<>();
  registry.put(RaftGroupId.randomId(), new SimpleStateMachine4Testing());
  registry.put(RaftGroupId.randomId(), new SMTransactionContext());

  try(MiniRaftClusterWithSimulatedRpc cluster = newCluster(0)) {
    cluster.setStateMachineRegistry(registry::get);

    final RaftPeerId id = RaftPeerId.valueOf("s0");
    cluster.putNewServer(id, null, true);
    cluster.start();

    for(RaftGroupId gid : registry.keySet()) {
      final RaftGroup newGroup = RaftGroup.valueOf(gid, cluster.getPeers());
      LOG.info("add new group: " + newGroup);
      try (final RaftClient client = cluster.createClient(newGroup)) {
        for (RaftPeer p : newGroup.getPeers()) {
          client.groupAdd(newGroup, p.getId());
        }
      }
    }

    final RaftServerProxy proxy = cluster.getServer(id);
    for(Map.Entry<RaftGroupId, StateMachine> e: registry.entrySet()) {
      final RaftServerImpl impl = RaftServerTestUtil.getRaftServerImpl(proxy, e.getKey());
      Assert.assertSame(e.getValue(), impl.getStateMachine());
    }
  }
}
 

@Test
public void testPythagorean() throws Exception {
  setAndStart(cluster);
  try (final RaftClient client = cluster.createClient()) {
    runTestPythagorean(client, 3, 10);
  }
}
 

static void writeSomething(Supplier<Message> newMessage, MiniRaftCluster cluster) throws Exception {
  try(final RaftClient client = cluster.createClient()) {
    // write some messages
    for(int i = 0; i < 10; i++) {
      Assert.assertTrue(client.send(newMessage.get()).isSuccess());
    }
  }
}
 

@Before
public void setup() throws Exception {
  ozoneContainer = Mockito.mock(OzoneContainer.class);
  stateContext = Mockito.mock(StateContext.class);
  connectionManager = Mockito.mock(SCMConnectionManager.class);
  raftClient = Mockito.mock(RaftClient.class);
  final RaftClient.Builder builder = mockRaftClientBuilder();
  Mockito.when(builder.build()).thenReturn(raftClient);
  PowerMockito.mockStatic(RaftClient.class);
  PowerMockito.when(RaftClient.newBuilder()).thenReturn(builder);
}
 

/**
 * Constuctor (with configuration). Build raft client for meta quorum
 * @param metaQuorum
 * @param config log serice configuration
 * @param properties
 */
public LogServiceClient(String metaQuorum, LogServiceConfiguration config, RaftProperties properties) {
    Set<RaftPeer> peers = getPeersFromQuorum(metaQuorum);
    RaftGroup meta = RaftGroup.valueOf(Constants.META_GROUP_ID, peers);
    client = RaftClient.newBuilder()
            .setRaftGroup(meta)
            .setClientId(ClientId.randomId())
            .setProperties(properties)
            .build();
    this.config = config;
}
 

/**
 * Constuctor (with configuration). Build raft client for meta quorum
 * @param metaQuorum
 * @param config log serice configuration
 */
public LogServiceClient(String metaQuorum, LogServiceConfiguration config) {
    Set<RaftPeer> peers = getPeersFromQuorum(metaQuorum);
    RaftProperties properties = new RaftProperties();
    RaftGroup meta = RaftGroup.valueOf(Constants.META_GROUP_ID, peers);
    client = RaftClient.newBuilder()
            .setRaftGroup(meta)
            .setClientId(ClientId.randomId())
            .setProperties(properties)
            .build();
    this.config = config;
}
 

@Override
protected void operation(RaftClient client) throws IOException {
  RaftClientReply getValue =
      client.sendReadOnly(Expression.Utils.toMessage(new Variable(name)));
  Expression response =
      Expression.Utils.bytes2Expression(getValue.getMessage().getContent().toByteArray(), 0);
  System.out.println(String.format("%s=%s", name, (DoubleValue) response).toString());
}
 

public void setConfiguration(RaftPeer... peers) throws IOException {
  try(RaftClient client = createClient()) {
    LOG.info("Start changing the configuration: {}", Arrays.asList(peers));
    final RaftClientReply reply = client.setConfiguration(peers);
    Preconditions.assertTrue(reply.isSuccess());
  }
}
 

CompletableFuture<RaftClientReply> sendWatchRequest(long logIndex, RetryPolicy policy)
    throws Exception {

  try (final RaftClient watchClient =
      cluster.createClient(RaftTestUtil.waitForLeader(cluster).getId(), policy)) {

    CompletableFuture<RaftClientReply> reply =
        watchClient.sendAsync(new RaftTestUtil.SimpleMessage("message"));
    long writeIndex = reply.get().getLogIndex();
    Assert.assertTrue(writeIndex > 0);
    watchClient.sendWatchAsync(writeIndex, ReplicationLevel.MAJORITY_COMMITTED);
    return watchClient.sendWatchAsync(logIndex, ReplicationLevel.MAJORITY);
  }

}
 

public static void testStateMachineMetrics(boolean async,
    MiniRaftCluster cluster, Logger LOG) throws Exception {
  RaftServerImpl leader = waitForLeader(cluster);
  try (final RaftClient client = cluster.createClient()) {

    Assert.assertTrue(leader.isLeader());

    Gauge appliedIndexGauge = getStatemachineGaugeWithName(leader,
        STATEMACHINE_APPLIED_INDEX_GAUGE);
    Gauge smAppliedIndexGauge = getStatemachineGaugeWithName(leader,
        STATEMACHINE_APPLY_COMPLETED_GAUGE);

    long appliedIndexBefore = (Long) appliedIndexGauge.getValue();
    long smAppliedIndexBefore = (Long) smAppliedIndexGauge.getValue();
    checkFollowerCommitLagsLeader(cluster);

    if (async) {
      CompletableFuture<RaftClientReply> replyFuture = client.sendAsync(new SimpleMessage("abc"));
      replyFuture.get();
    } else {
      client.send(new SimpleMessage("abc"));
    }

    long appliedIndexAfter = (Long) appliedIndexGauge.getValue();
    long smAppliedIndexAfter = (Long) smAppliedIndexGauge.getValue();
    checkFollowerCommitLagsLeader(cluster);

    Assert.assertTrue("StateMachine Applied Index not incremented",
        appliedIndexAfter > appliedIndexBefore);
    Assert.assertTrue("StateMachine Apply completed Index not incremented",
        smAppliedIndexAfter > smAppliedIndexBefore);
  }
}
 

@Override
protected void operation(RaftClient client) throws IOException {
  int length = Integer.parseInt(size);
  int num = Integer.parseInt(numFiles);
  AtomicLong totalBytes = new AtomicLong(0);
  String entropy = RandomStringUtils.randomAlphanumeric(10);

  byte[] fileValue = string2Bytes(RandomStringUtils.randomAscii(length));
  FileStoreClient fileStoreClient = new FileStoreClient(client);

  System.out.println("Starting load now ");
  long startTime = System.currentTimeMillis();
  List<CompletableFuture<Long>> futures = new ArrayList<>();
  for (int i = 0; i < num; i++) {
    String path = "file-" + entropy + "-" + i;
    ByteBuffer b = ByteBuffer.wrap(fileValue);
    futures.add(fileStoreClient.writeAsync(path, 0, true, b));
  }

  for (CompletableFuture<Long> future : futures) {
    Long writtenLen = future.join();
    totalBytes.addAndGet(writtenLen);
    if (writtenLen != length) {
      System.out.println("File length written is wrong: " + writtenLen + length);
    }
  }
  long endTime = System.currentTimeMillis();

  System.out.println("Total files written: " + futures.size());
  System.out.println("Each files size: " + length);
  System.out.println("Total data written: " + totalBytes + " bytes");
  System.out.println("Total time taken: " + (endTime - startTime) + " millis");

  client.close();
  System.exit(0);
}
 

void runTestStream(CLUSTER cluster) throws Exception {
  RaftTestUtil.waitForLeader(cluster);

  // stream multiple parts
  final int numParts = 9;
  final int endOfRequest = 6;
  final StringBuilder key = new StringBuilder();
  try(RaftClient client = cluster.createClient();
      MessageOutputStream out = client.getStreamApi().stream()) {
    for (int i = 1; i <= numParts; i++) {
      key.append(i);
      out.sendAsync(new SimpleMessage(i + ""), i == endOfRequest);
    }
  }

  // check if all the parts are streamed as a single message.
  final String k = key.toString();
  try(RaftClient client = cluster.createClient()) {
    final String k1 = k.substring(0, endOfRequest);
    final RaftClientReply r1= client.sendReadOnly(new SimpleMessage(k1));
    Assert.assertTrue(r1.isSuccess());

    final String k2 = k.substring(endOfRequest);
    final RaftClientReply r2 = client.sendReadOnly(new SimpleMessage(k2));
    Assert.assertTrue(r2.isSuccess());
  }
}
 

public RaftServerImpl getLeaderAndSendFirstMessage(boolean ignoreException) throws IOException {
  final RaftServerImpl leader = getLeader();
  try(RaftClient client = createClient(leader.getId())) {
    client.send(new RaftTestUtil.SimpleMessage("first msg to make leader ready"));
  } catch (IOException e) {
    if (!ignoreException) {
      throw e;
    }
  }
  return leader;
}