org.apache.ratis.protocol.RaftGroup Java Examples

@Override
public void run() throws Exception {
  RaftProperties raftProperties = new RaftProperties();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)),
      parsePeers(peers));

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);


}
 

/**
 * Removes pipeline from SCM. Sends ratis command to destroy pipeline on all
 * the datanodes.
 *
 * @param pipeline        - Pipeline to be destroyed
 * @param ozoneConf       - Ozone configuration
 * @param grpcTlsConfig
 * @throws IOException
 */
public static void destroyPipeline(Pipeline pipeline,
    ConfigurationSource ozoneConf,
    GrpcTlsConfig grpcTlsConfig) {
  final RaftGroup group = RatisHelper.newRaftGroup(pipeline);
  if (LOG.isDebugEnabled()) {
    LOG.debug("destroying pipeline:{} with {}", pipeline.getId(), group);
  }
  for (DatanodeDetails dn : pipeline.getNodes()) {
    try {
      destroyPipeline(dn, pipeline.getId(), ozoneConf, grpcTlsConfig);
    } catch (IOException e) {
      LOG.warn("Pipeline destroy failed for pipeline={} dn={} exception={}",
          pipeline.getId(), dn, e.getMessage());
    }
  }
}
 

@Override
public void addGroup(HddsProtos.PipelineID pipelineId,
    Collection<DatanodeDetails> peers) throws IOException {
  final PipelineID pipelineID = PipelineID.getFromProtobuf(pipelineId);
  final RaftGroupId groupId = RaftGroupId.valueOf(pipelineID.getId());
  final RaftGroup group = RatisHelper.newRaftGroup(groupId, peers);
  GroupManagementRequest request = GroupManagementRequest.newAdd(
      clientId, server.getId(), nextCallId(), group);

  RaftClientReply reply;
  try {
    reply = server.groupManagement(request);
  } catch (Exception e) {
    throw new IOException(e.getMessage(), e);
  }
  processReply(reply);
}
 

@Test
public void testAsyncConfiguration() throws IOException {
  LOG.info("Running testAsyncConfiguration");
  final RaftProperties properties = new RaftProperties();
  RaftClient.Builder clientBuilder = RaftClient.newBuilder()
      .setRaftGroup(RaftGroup.emptyGroup())
      .setProperties(properties);
  int numThreads = RaftClientConfigKeys.Async.SCHEDULER_THREADS_DEFAULT;
  int maxOutstandingRequests = RaftClientConfigKeys.Async.MAX_OUTSTANDING_REQUESTS_DEFAULT;
  try(RaftClient client = clientBuilder.build()) {
    RaftClientTestUtil.assertScheduler(client, numThreads);
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, maxOutstandingRequests, 0);
  }

  numThreads = 200;
  maxOutstandingRequests = 5;
  RaftClientConfigKeys.Async.setMaxOutstandingRequests(properties, maxOutstandingRequests);
  RaftClientConfigKeys.Async.setSchedulerThreads(properties, numThreads);
  try(RaftClient client = clientBuilder.build()) {
    RaftClientTestUtil.assertScheduler(client, numThreads);
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, maxOutstandingRequests, 0);
  }
}
 

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDir(properties, Collections.singletonList(storageDir));
  StateMachine stateMachine = new ArithmeticStateMachine();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(getRaftGroupId())),
          getPeers());
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

@Override
public void run() throws Exception {
  RaftProperties raftProperties = new RaftProperties();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(getRaftGroupId())),
          getPeers());

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);


}
 

/**
 * prepare the peer list when removing some peers from the conf
 */
public PeerChanges removePeers(int number, boolean removeLeader,
    Collection<RaftPeer> excluded) throws InterruptedException {
  Collection<RaftPeer> peers = new ArrayList<>(group.getPeers());
  List<RaftPeer> removedPeers = new ArrayList<>(number);
  if (removeLeader) {
    final RaftPeer leader = toRaftPeer(RaftTestUtil.waitForLeader(this));
    Preconditions.assertTrue(!excluded.contains(leader));
    peers.remove(leader);
    removedPeers.add(leader);
  }
  List<RaftServerImpl> followers = getFollowers();
  for (int i = 0, removed = 0; i < followers.size() &&
      removed < (removeLeader ? number - 1 : number); i++) {
    RaftPeer toRemove = toRaftPeer(followers.get(i));
    if (!excluded.contains(toRemove)) {
      peers.remove(toRemove);
      removedPeers.add(toRemove);
      removed++;
    }
  }
  final RaftPeer[] p = peers.toArray(RaftPeer.emptyArray());
  group = RaftGroup.valueOf(group.getGroupId(), p);
  return new PeerChanges(p, RaftPeer.emptyArray(), removedPeers.toArray(RaftPeer.emptyArray()));
}
 

@Test
public void testAsyncConfiguration() throws IOException {
  LOG.info("Running testAsyncConfiguration");
  final RaftProperties properties = new RaftProperties();
  RaftClient.Builder clientBuilder = RaftClient.newBuilder()
      .setRaftGroup(RaftGroup.emptyGroup())
      .setProperties(properties);
  int maxOutstandingRequests = RaftClientConfigKeys.Async.OUTSTANDING_REQUESTS_MAX_DEFAULT;
  try(RaftClient client = clientBuilder.build()) {
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, maxOutstandingRequests, 0);
  }

  maxOutstandingRequests = 5;
  RaftClientConfigKeys.Async.setOutstandingRequestsMax(properties, maxOutstandingRequests);
  try(RaftClient client = clientBuilder.build()) {
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, maxOutstandingRequests, 0);
  }
}
 

@Test
public void testGroupAlreadyExists() throws Exception {
  final MiniRaftCluster cluster = getCluster(1);
  cluster.start();
  final RaftPeer peer = cluster.getPeers().get(0);
  final RaftPeerId peerId = peer.getId();
  final RaftGroup group = RaftGroup.valueOf(cluster.getGroupId(), peer);
  try (final RaftClient client = cluster.createClient()) {
    Assert.assertEquals(group, cluster.getRaftServerImpl(peerId).getGroup());
    try {
      client.groupAdd(group, peer.getId());
    } catch (IOException ex) {
      // HadoopRPC throws RemoteException, which makes it hard to check if
      // the exception is instance of AlreadyExistsException
      Assert.assertTrue(ex.toString().contains(AlreadyExistsException.class.getCanonicalName()));
    }
    Assert.assertEquals(group, cluster.getRaftServerImpl(peerId).getGroup());
    cluster.shutdown();
  }
}
 

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  RaftPeer[] peers = getPeers();
  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir));
  ConfUtils.setFile(properties::setFile, FileStoreCommon.STATEMACHINE_DIR_KEY,
      storageDir);
  StateMachine stateMachine = new FileStoreStateMachine(properties);

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers);
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  RaftPeer[] peers = getPeers();
  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir));
  StateMachine stateMachine = new ArithmeticStateMachine();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers);
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

/** For the case that all {@link RaftServerImpl} objects share the same {@link StateMachine}. */
public static RaftServerProxy newRaftServer(
    RaftPeerId id, RaftGroup group, StateMachine.Registry stateMachineRegistry,
    RaftProperties properties, Parameters parameters) throws IOException {
  RaftServerProxy.LOG.debug("newRaftServer: {}, {}", id, group);
  final RaftServerProxy proxy = newRaftServer(id, stateMachineRegistry, properties, parameters);
  proxy.initGroups(group);
  return proxy;
}
 

protected int getPort(RaftPeerId id, RaftGroup g) {
  final RaftPeer p = g != null? g.getPeer(id): peers.get(id);
  final String address = p == null? null : p.getAddress();
  final InetSocketAddress inetAddress = address != null?
      NetUtils.createSocketAddr(address): NetUtils.createLocalServerAddress();
  return inetAddress.getPort();
}
 

public static RaftGroup initRaftGroup(Collection<String> ids) {
  final RaftPeer[] peers = ids.stream()
      .map(RaftPeerId::valueOf)
      .map(id -> new RaftPeer(id, NetUtils.createLocalServerAddress()))
      .toArray(RaftPeer[]::new);
  return RaftGroup.valueOf(RaftGroupId.randomId(), peers);
}
 

private CompletableFuture<Message> processGetLogRequest(
        MetaServiceProtos.MetaServiceRequestProto logServiceRequestProto) {
    MetaServiceProtos.GetLogRequestProto getLog = logServiceRequestProto.getGetLog();
    LogName logName = LogServiceProtoUtil.toLogName(getLog.getLogName());
    RaftGroup raftGroup = map.get(logName);
    if (raftGroup != null) {
        return CompletableFuture.completedFuture(Message.valueOf(
                MetaServiceProtoUtil.toGetLogReplyProto(new LogInfo(logName, raftGroup))
                        .toByteString()));
    } else {
        return CompletableFuture.completedFuture(Message.valueOf(
                MetaServiceProtoUtil.toGetLogExceptionReplyProto(
                        new LogNotFoundException(logName.getName())).build().toByteString()));
    }
}
 

public RaftServerImpl restartServer(RaftPeerId newId, RaftGroup group, boolean format) throws IOException {
  killServer(newId);
  servers.remove(newId);

  final RaftServerProxy proxy = putNewServer(newId, group, format);
  proxy.start();
  return group == null? null: proxy.getImpl(group.getGroupId());
}
 

private RaftServerProxy newRaftServer(RaftPeerId id, RaftGroup group, boolean format) {
  LOG.info("newRaftServer: {}, {}, format? {}", id, group, format);
  try {
    final File dir = getStorageDir(id);
    if (format) {
      FileUtils.deleteFully(dir);
      LOG.info("Formatted directory {}", dir);
    }
    final RaftProperties prop = new RaftProperties(properties);
    RaftServerConfigKeys.setStorageDirs(prop, Collections.singletonList(dir));
    return newRaftServer(id, getStateMachineRegistry(properties), group, prop);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

@Override
protected RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry , RaftGroup group,
    RaftProperties properties) throws IOException {
  NettyConfigKeys.Server.setPort(properties, getPort(id, group));
  return ServerImplUtils.newRaftServer(id, group, stateMachineRegistry, properties, null);
}
 

private RaftClient.Builder mockRaftClientBuilder() {
  final RaftClient.Builder builder = Mockito.mock(RaftClient.Builder.class);
  Mockito.when(builder.setClientId(Mockito.any(ClientId.class)))
      .thenReturn(builder);
  Mockito.when(builder.setRaftGroup(Mockito.any(RaftGroup.class)))
      .thenReturn(builder);
  Mockito.when(builder.setLeaderId(Mockito.any(RaftPeerId.class)))
      .thenReturn(builder);
  Mockito.when(builder.setProperties(Mockito.any(RaftProperties.class)))
      .thenReturn(builder);
  Mockito.when(builder.setRetryPolicy(Mockito.any(RetryPolicy.class)))
      .thenReturn(builder);
  return builder;
}
 

@Override
public void run() throws Exception {
  int raftSegmentPreallocatedSize = 1024 * 1024 * 1024;
  RaftProperties raftProperties = new RaftProperties();
  RaftConfigKeys.Rpc.setType(raftProperties, SupportedRpcType.GRPC);
  GrpcConfigKeys.setMessageSizeMax(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.Appender.setBufferByteLimit(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setWriteBufferSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setPreallocatedSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setSegmentSizeMax(raftProperties,
      SizeInBytes.valueOf(1 * 1024 * 1024 * 1024));

  RaftServerConfigKeys.Log.setMaxCachedSegmentNum(raftProperties, 2);

  RaftClientConfigKeys.Rpc.setRequestTimeout(raftProperties,
      TimeDuration.valueOf(50000, TimeUnit.MILLISECONDS));
  RaftClientConfigKeys.Async.setSchedulerThreads(raftProperties, 10);
  RaftClientConfigKeys.Async.setMaxOutstandingRequests(raftProperties, 1000);


  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)),
      parsePeers(peers));

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);
}
 

@Test
public void testPipelineCreation() throws IOException {

  final List<DatanodeDetails> datanodes = getDatanodes();
  final PipelineID pipelineID = PipelineID.randomId();
  final SCMCommand<CreatePipelineCommandProto> command =
      new CreatePipelineCommand(pipelineID, HddsProtos.ReplicationType.RATIS,
          HddsProtos.ReplicationFactor.THREE, datanodes);

  final XceiverServerSpi writeChanel = Mockito.mock(XceiverServerSpi.class);
  final DatanodeStateMachine dnsm = Mockito.mock(DatanodeStateMachine.class);

  Mockito.when(stateContext.getParent()).thenReturn(dnsm);
  Mockito.when(dnsm.getDatanodeDetails()).thenReturn(datanodes.get(0));
  Mockito.when(ozoneContainer.getWriteChannel()).thenReturn(writeChanel);
  Mockito.when(writeChanel.isExist(pipelineID.getProtobuf()))
      .thenReturn(false);

  final CreatePipelineCommandHandler commandHandler =
      new CreatePipelineCommandHandler(new OzoneConfiguration());
  commandHandler.handle(command, ozoneContainer, stateContext,
      connectionManager);

  Mockito.verify(writeChanel, Mockito.times(1))
      .addGroup(pipelineID.getProtobuf(), datanodes);

  Mockito.verify(raftClient, Mockito.times(2))
      .groupAdd(Mockito.any(RaftGroup.class), Mockito.any(RaftPeerId.class));
}
 

public FileStoreClient(RaftGroup group, RaftProperties properties)
    throws IOException {
  this.client = RaftClient.newBuilder()
      .setProperties(properties)
      .setRaftGroup(group)
      .build();
}
 

@Test
public void testStateMachineRegistry() throws Throwable {
  final Map<RaftGroupId, StateMachine> registry = new ConcurrentHashMap<>();
  registry.put(RaftGroupId.randomId(), new SimpleStateMachine4Testing());
  registry.put(RaftGroupId.randomId(), new SMTransactionContext());

  try(MiniRaftClusterWithSimulatedRpc cluster = newCluster(0)) {
    cluster.setStateMachineRegistry(registry::get);

    final RaftPeerId id = RaftPeerId.valueOf("s0");
    cluster.putNewServer(id, null, true);
    cluster.start();

    for(RaftGroupId gid : registry.keySet()) {
      final RaftGroup newGroup = RaftGroup.valueOf(gid, cluster.getPeers());
      LOG.info("add new group: " + newGroup);
      final RaftClient client = cluster.createClient(newGroup);
      for(RaftPeer p : newGroup.getPeers()) {
        client.groupAdd(newGroup, p.getId());
      }
    }

    final RaftServerProxy proxy = cluster.getServer(id);
    for(Map.Entry<RaftGroupId, StateMachine> e: registry.entrySet()) {
      final RaftServerImpl impl = RaftServerTestUtil.getRaftServerImpl(proxy, e.getKey());
      Assert.assertSame(e.getValue(), impl.getStateMachine());
    }
  }
}
 

@Test
public void testCommandIdempotency() throws IOException {
  final List<DatanodeDetails> datanodes = getDatanodes();
  final PipelineID pipelineID = PipelineID.randomId();
  final SCMCommand<CreatePipelineCommandProto> command =
      new CreatePipelineCommand(pipelineID, HddsProtos.ReplicationType.RATIS,
          HddsProtos.ReplicationFactor.THREE, datanodes);

  final XceiverServerSpi writeChanel = Mockito.mock(XceiverServerSpi.class);
  final DatanodeStateMachine dnsm = Mockito.mock(DatanodeStateMachine.class);

  Mockito.when(stateContext.getParent()).thenReturn(dnsm);
  Mockito.when(dnsm.getDatanodeDetails()).thenReturn(datanodes.get(0));
  Mockito.when(ozoneContainer.getWriteChannel()).thenReturn(writeChanel);
  Mockito.when(writeChanel.isExist(pipelineID.getProtobuf()))
      .thenReturn(true);

  final CreatePipelineCommandHandler commandHandler =
      new CreatePipelineCommandHandler(new OzoneConfiguration());
  commandHandler.handle(command, ozoneContainer, stateContext,
      connectionManager);

  Mockito.verify(writeChanel, Mockito.times(0))
      .addGroup(pipelineID.getProtobuf(), datanodes);

  Mockito.verify(raftClient, Mockito.times(0))
      .groupAdd(Mockito.any(RaftGroup.class), Mockito.any(RaftPeerId.class));
}
 

@Override
protected RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry , RaftGroup group,
    RaftProperties properties) throws IOException {
  serverRequestReply.addPeer(id);
  client2serverRequestReply.addPeer(id);
  return ServerImplUtils.newRaftServer(id, group, stateMachineRegistry, properties, parameters);
}
 

private void runTestGroupMismatchException(CLUSTER cluster) throws Exception {
  // send a message to make sure the cluster is working
  try(RaftClient client = cluster.createClient()) {
    final RaftClientReply reply = client.sendAsync(new SimpleMessage("first")).get();
    Assert.assertTrue(reply.isSuccess());
  }

  // create another group
  final RaftGroup clusterGroup = cluster.getGroup();
  final RaftGroup anotherGroup = RaftGroup.valueOf(RaftGroupId.randomId(), clusterGroup.getPeers());
  Assert.assertNotEquals(clusterGroup.getGroupId(), anotherGroup.getGroupId());

  // create another client using another group
  final SimpleMessage[] messages = SimpleMessage.create(5);
  try(RaftClient client = cluster.createClient(anotherGroup)) {
    // send a few messages
    final List<CompletableFuture<RaftClientReply>> futures = new ArrayList<>();
    for(SimpleMessage m : messages) {
      futures.add(client.sendAsync(m));
    }
    Assert.assertEquals(messages.length, futures.size());

    // check replies
    final Iterator<CompletableFuture<RaftClientReply>> i = futures.iterator();
    testFailureCase("First reply is GroupMismatchException",
        () -> i.next().get(),
        ExecutionException.class, GroupMismatchException.class);
    for(; i.hasNext(); ) {
      testFailureCase("Following replies are AlreadyClosedException caused by GroupMismatchException",
          () -> i.next().get(),
          ExecutionException.class, AlreadyClosedException.class, GroupMismatchException.class);
    }
  }
}
 

public RaftClient createClient(RaftPeerId leaderId, RaftGroup group, RetryPolicy retryPolicy) {
  RaftClient.Builder builder = RaftClient.newBuilder()
      .setRaftGroup(group)
      .setLeaderId(leaderId)
      .setProperties(properties)
      .setParameters(parameters)
      .setRetryPolicy(retryPolicy);
  return builder.build();
}
 

private RaftServerProxy newRaftServer(RaftPeerId id, RaftGroup group, boolean format) {
  LOG.info("newRaftServer: {}, {}, format? {}", id, group, format);
  try {
    final File dir = getStorageDir(id);
    if (format) {
      FileUtils.deleteFully(dir);
      LOG.info("Formatted directory {}", dir);
    }
    final RaftProperties prop = new RaftProperties(properties);
    RaftServerConfigKeys.setStorageDir(prop, Collections.singletonList(dir));
    return newRaftServer(id, getStateMachineRegistry(properties), group, prop);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

public RaftServerImpl restartServer(RaftPeerId serverId, RaftGroup group, boolean format) throws IOException {
  killServer(serverId);
  servers.remove(serverId);

  final RaftServerProxy proxy = putNewServer(serverId, group, format);
  proxy.start();
  return group == null? null: proxy.getImpl(group.getGroupId());
}
 

private void runTestMultiRaftGroup(int[] idIndex, int chosen) throws Exception {

    final CheckedBiConsumer<MiniRaftCluster, RaftGroup, IOException> checker = (cluster, group) -> {
      try (final RaftClient client = cluster.createClient(group)) {
        TestArithmetic.runTestPythagorean(client, start.getAndAdd(2*count), count);
      }
    };

    GroupManagementBaseTest.runMultiGroupTest(
        cluster, idIndex, chosen, checker);
  }