org.apache.ratis.client.RaftClientConfigKeys Java Examples

@Test
public void testAsyncConfiguration() throws IOException {
  LOG.info("Running testAsyncConfiguration");
  final RaftProperties properties = new RaftProperties();
  RaftClient.Builder clientBuilder = RaftClient.newBuilder()
      .setRaftGroup(RaftGroup.emptyGroup())
      .setProperties(properties);
  int numThreads = RaftClientConfigKeys.Async.SCHEDULER_THREADS_DEFAULT;
  int maxOutstandingRequests = RaftClientConfigKeys.Async.MAX_OUTSTANDING_REQUESTS_DEFAULT;
  try(RaftClient client = clientBuilder.build()) {
    RaftClientTestUtil.assertScheduler(client, numThreads);
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, maxOutstandingRequests, 0);
  }

  numThreads = 200;
  maxOutstandingRequests = 5;
  RaftClientConfigKeys.Async.setMaxOutstandingRequests(properties, maxOutstandingRequests);
  RaftClientConfigKeys.Async.setSchedulerThreads(properties, numThreads);
  try(RaftClient client = clientBuilder.build()) {
    RaftClientTestUtil.assertScheduler(client, numThreads);
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, maxOutstandingRequests, 0);
  }
}
 

@Test
public void testWatchRequestClientTimeout() throws Exception {
  final RaftProperties p = getProperties();
  RaftServerConfigKeys.Watch.setTimeout(p, TimeDuration.valueOf(100,
      TimeUnit.SECONDS));
  RaftClientConfigKeys.Rpc.setWatchRequestTimeout(p,
      TimeDuration.valueOf(15, TimeUnit.SECONDS));
  try {
    runWithNewCluster(NUM_SERVERS,
        cluster -> runSingleTest(WatchRequestTests::runTestWatchRequestClientTimeout, cluster, LOG));
  } finally {
    RaftServerConfigKeys.Watch.setTimeout(p, RaftServerConfigKeys.Watch.TIMEOUT_DEFAULT);
    RaftClientConfigKeys.Rpc.setWatchRequestTimeout(p,
        RaftClientConfigKeys.Rpc.WATCH_REQUEST_TIMEOUT_DEFAULT);
  }
}
 

private void runTestTimeoutException(CLUSTER cluster) throws Exception {
  // send a message to make sure the cluster is working
  try(RaftClient client = cluster.createClient()) {
    final RaftClientReply reply = client.send(new SimpleMessage("m0"));
    Assert.assertTrue(reply.isSuccess());

    RaftClientConfigKeys.Rpc.setRequestTimeout(properties.get(), ONE_SECOND);
    // Block StartTransaction
    cluster.getServers().stream()
        .map(cluster::getRaftServerImpl)
        .map(SimpleStateMachine4Testing::get)
        .forEach(SimpleStateMachine4Testing::blockStartTransaction);
    final CompletableFuture<RaftClientReply> replyFuture = client.sendAsync(new SimpleMessage("m1"));
    FIVE_SECONDS.sleep();
    // Unblock StartTransaction
    cluster.getServers().stream()
        .map(cluster::getRaftServerImpl)
        .map(SimpleStateMachine4Testing::get)
        .forEach(SimpleStateMachine4Testing::unblockStartTransaction);
    // The request should succeed after start transaction is unblocked
    Assert.assertTrue(replyFuture.get(FIVE_SECONDS.getDuration(), FIVE_SECONDS.getUnit()).isSuccess());
  }
}
 

@Test
public void testAsyncConfiguration() throws IOException {
  LOG.info("Running testAsyncConfiguration");
  final RaftProperties properties = new RaftProperties();
  RaftClient.Builder clientBuilder = RaftClient.newBuilder()
      .setRaftGroup(RaftGroup.emptyGroup())
      .setProperties(properties);
  int maxOutstandingRequests = RaftClientConfigKeys.Async.OUTSTANDING_REQUESTS_MAX_DEFAULT;
  try(RaftClient client = clientBuilder.build()) {
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, maxOutstandingRequests, 0);
  }

  maxOutstandingRequests = 5;
  RaftClientConfigKeys.Async.setOutstandingRequestsMax(properties, maxOutstandingRequests);
  try(RaftClient client = clientBuilder.build()) {
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, maxOutstandingRequests, 0);
  }
}
 

/**
 * Set all the properties matching with prefix
 * {@link RatisHelper#HDDS_DATANODE_RATIS_PREFIX_KEY} in
 * ozone configuration object and configure it to RaftProperties.
 * @param ozoneConf
 * @param raftProperties
 */
public static void createRaftServerProperties(ConfigurationSource ozoneConf,
     RaftProperties raftProperties) {

  Map<String, String> ratisServerConf =
      getDatanodeRatisPrefixProps(ozoneConf);
  ratisServerConf.forEach((key, val) -> {
    // Exclude ratis client configuration.
    if (!key.startsWith(RaftClientConfigKeys.PREFIX)) {
      raftProperties.set(key, val);
    }
  });
}
 

RaftClientImpl(ClientId clientId, RaftGroup group, RaftPeerId leaderId,
    RaftClientRpc clientRpc, RaftProperties properties, RetryPolicy retryPolicy) {
  this.clientId = clientId;
  this.clientRpc = clientRpc;
  this.peers = new ConcurrentLinkedQueue<>(group.getPeers());
  this.groupId = group.getGroupId();
  this.leaderId = leaderId != null? leaderId
      : !peers.isEmpty()? peers.iterator().next().getId(): null;
  Preconditions.assertTrue(retryPolicy != null, "retry policy can't be null");
  this.retryPolicy = retryPolicy;

  asyncRequestSemaphore = new Semaphore(RaftClientConfigKeys.Async.maxOutstandingRequests(properties));
  scheduler = TimeoutScheduler.newInstance(RaftClientConfigKeys.Async.schedulerThreads(properties));
  clientRpc.addServers(peers);
}
 

public GrpcClientProtocolClient(ClientId id, RaftPeer target,
                                RaftProperties properties,
                                GrpcTlsConfig tlsConf) {
  this.name = JavaUtils.memoize(() -> id + "->" + target.getId());
  this.target = target;
  final SizeInBytes flowControlWindow = GrpcConfigKeys.flowControlWindow(properties, LOG::debug);
  final SizeInBytes maxMessageSize = GrpcConfigKeys.messageSizeMax(properties, LOG::debug);
  NettyChannelBuilder channelBuilder =
      NettyChannelBuilder.forTarget(target.getAddress());

  if (tlsConf!= null) {
    SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
    if (tlsConf.getTrustStore() != null) {
      sslContextBuilder.trustManager(tlsConf.getTrustStore());
    }
    if (tlsConf.getMtlsEnabled()) {
      sslContextBuilder.keyManager(tlsConf.getCertChain(),
          tlsConf.getPrivateKey());
    }
    try {
      channelBuilder.useTransportSecurity().sslContext(sslContextBuilder.build());
    } catch (Exception ex) {
      throw new RuntimeException(ex);
    }
  } else {
    channelBuilder.negotiationType(NegotiationType.PLAINTEXT);
  }
  channel = channelBuilder.flowControlWindow(flowControlWindow.getSizeInt())
      .maxInboundMessageSize(maxMessageSize.getSizeInt())
      .build();
  blockingStub = RaftClientProtocolServiceGrpc.newBlockingStub(channel);
  asyncStub = RaftClientProtocolServiceGrpc.newStub(channel);
  adminBlockingStub = AdminProtocolServiceGrpc.newBlockingStub(channel);
  this.requestTimeoutDuration = RaftClientConfigKeys.Rpc.requestTimeout(properties);
}
 

@Override
public void run() throws Exception {
  int raftSegmentPreallocatedSize = 1024 * 1024 * 1024;
  RaftProperties raftProperties = new RaftProperties();
  RaftConfigKeys.Rpc.setType(raftProperties, SupportedRpcType.GRPC);
  GrpcConfigKeys.setMessageSizeMax(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.Appender.setBufferByteLimit(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setWriteBufferSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setPreallocatedSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setSegmentSizeMax(raftProperties,
      SizeInBytes.valueOf(1 * 1024 * 1024 * 1024));

  RaftServerConfigKeys.Log.setMaxCachedSegmentNum(raftProperties, 2);

  RaftClientConfigKeys.Rpc.setRequestTimeout(raftProperties,
      TimeDuration.valueOf(50000, TimeUnit.MILLISECONDS));
  RaftClientConfigKeys.Async.setSchedulerThreads(raftProperties, 10);
  RaftClientConfigKeys.Async.setMaxOutstandingRequests(raftProperties, 1000);


  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)),
      parsePeers(peers));

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);
}
 

@Test
public void testRequestTimeout() throws Exception {
  final TimeDuration oldExpiryTime = RaftServerConfigKeys.RetryCache.expiryTime(getProperties());
  RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), FIVE_SECONDS);
  RaftClientConfigKeys.Async.Experimental.setSendDummyRequest(getProperties(), false);
  runWithNewCluster(NUM_SERVERS, cluster -> RaftBasicTests.testRequestTimeout(true, cluster, LOG));

  //reset for the other tests
  RaftServerConfigKeys.RetryCache.setExpiryTime(getProperties(), oldExpiryTime);
  RaftClientConfigKeys.Async.Experimental.setSendDummyRequest(getProperties(), true);
}
 

@Test
public void testStreamAsync() throws Exception {
  final RaftProperties p = getProperties();
  RaftClientConfigKeys.Stream.setSubmessageSize(p, SUBMESSAGE_SIZE);
  p.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
      SimpleStateMachine4Testing.class, StateMachine.class);

  runWithNewCluster(NUM_SERVERS, this::runTestStreamAsync);
  RaftClientConfigKeys.Stream.setSubmessageSize(p);
}
 

/**
 * Set all the properties matching with regex
 * {@link RatisHelper#HDDS_DATANODE_RATIS_PREFIX_KEY} in
 * ozone configuration object and configure it to RaftProperties.
 * @param ozoneConf
 * @param raftProperties
 */
public static void createRaftClientProperties(ConfigurationSource ozoneConf,
    RaftProperties raftProperties) {

  // As for client we do not require server and grpc server/tls. exclude them.
  Map<String, String> ratisClientConf = ozoneConf.getPropsWithPrefix(
      StringUtils.appendIfNotPresent(HDDS_DATANODE_RATIS_PREFIX_KEY, '.'));
  ratisClientConf.forEach((key, val) -> {
    if (key.startsWith(RaftClientConfigKeys.PREFIX) || isGrpcClientConfig(
        key)) {
      raftProperties.set(key, val);
    }
  });
}
 

static OrderedAsync newInstance(RaftClientImpl client, RaftProperties properties) {
  final OrderedAsync ordered = new OrderedAsync(client, properties);
  // send a dummy watch request to establish the connection
  // TODO: this is a work around, it is better to fix the underlying RPC implementation
  if (RaftClientConfigKeys.Async.Experimental.sendDummyRequest(properties)) {
    ordered.send(RaftClientRequest.watchRequestType(), null, null);
  }
  return ordered;
}
 

@Test
public void testExceptionRetryAttempts()
    throws InterruptedException, IOException {
  RaftProperties prop = new RaftProperties();
  RaftClientConfigKeys.Rpc.setRequestTimeout(prop, TimeDuration.valueOf(100, TimeUnit.MILLISECONDS));
  prop.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
      SimpleStateMachine4Testing.class, StateMachine.class);
  RaftServerConfigKeys.Write.setElementLimit(prop, 1);
  MiniRaftClusterWithGrpc cluster = getFactory().newCluster(1, prop);
  RaftServerImpl leader = null;
  try {
    cluster.start();
    ExceptionDependentRetry.Builder builder =
        ExceptionDependentRetry.newBuilder();
    builder.setExceptionToPolicy(TimeoutIOException.class,
        MultipleLinearRandomRetry.parseCommaSeparated("1ms, 5"));
    builder.setDefaultPolicy(RetryPolicies.retryForeverNoSleep());

    // create a client with the exception dependent policy
    try (final RaftClient client = cluster.createClient(builder.build())) {
      client.sendAsync(new RaftTestUtil.SimpleMessage("1")).get();

      leader = cluster.getLeader();
      ((SimpleStateMachine4Testing) leader.getStateMachine()).blockWriteStateMachineData();

      client.sendAsync(new RaftTestUtil.SimpleMessage("2")).get();
    }
    Assert.fail("Test should have failed.");
  } catch (ExecutionException e) {
    RaftRetryFailureException rrfe = (RaftRetryFailureException) e.getCause();
    Assert.assertEquals(6, rrfe.getAttemptCount());
  } finally {
    ((SimpleStateMachine4Testing)leader.getStateMachine()).unblockWriteStateMachineData();
    cluster.shutdown();
  }
}
 

@BeforeClass
public static void beforeClass() {
    cluster = new LogServiceCluster(3);
    cluster.createWorkers(3);
    workers = cluster.getWorkers();
    assert(workers.size() == 3);

    RaftProperties properties = new RaftProperties();
    RaftClientConfigKeys.Rpc.setRequestTimeout(properties, TimeDuration.valueOf(15, TimeUnit.SECONDS));

    cluster.getMasters().parallelStream().forEach(master ->
        ((MetaStateMachine)master.getMetaStateMachine()).setProperties(properties));

    client = new LogServiceClient(cluster.getMetaIdentity(), properties) {
      @Override
      public LogStream createLog(LogName logName) throws IOException {
        createCount.incrementAndGet();
        return super.createLog(logName);
      }

      @Override
      public void deleteLog(LogName logName) throws IOException {
        deleteCount.incrementAndGet();
        super.deleteLog(logName);
      }

      @Override
      public List<LogInfo> listLogs() throws IOException {
        listCount.incrementAndGet();
        return super.listLogs();
      }
    };
}
 

@Override
public void run() throws Exception {
  int raftSegmentPreallocatedSize = 1024 * 1024 * 1024;
  RaftProperties raftProperties = new RaftProperties();
  RaftConfigKeys.Rpc.setType(raftProperties, SupportedRpcType.GRPC);
  GrpcConfigKeys.setMessageSizeMax(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.Appender.setBufferByteLimit(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setWriteBufferSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setPreallocatedSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setSegmentSizeMax(raftProperties,
      SizeInBytes.valueOf(1 * 1024 * 1024 * 1024));

  RaftServerConfigKeys.Log.setSegmentCacheNumMax(raftProperties, 2);

  RaftClientConfigKeys.Rpc.setRequestTimeout(raftProperties,
      TimeDuration.valueOf(50000, TimeUnit.MILLISECONDS));
  RaftClientConfigKeys.Async.setOutstandingRequestsMax(raftProperties, 1000);


  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(getRaftGroupId())),
          getPeers());

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);
}
 

@Override
void setRaftProperties(RaftProperties properties) {
  super.setRaftProperties(properties);

  // Increase the client timeout
  long rpcTimeout = getConfig().getLong(Constants.LOG_SERVICE_RPC_TIMEOUT_KEY,
    Constants.DEFAULT_RPC_TIMEOUT);
  RaftClientConfigKeys.Rpc.setRequestTimeout(properties,
    TimeDuration.valueOf(rpcTimeout, TimeUnit.MILLISECONDS));

  // Increase the segment size to avoid rolling so quickly
  long segmentSize = getConfig().getLong(Constants.RATIS_RAFT_SEGMENT_SIZE_KEY,
    Constants.DEFAULT_RATIS_RAFT_SEGMENT_SIZE);
  SizeInBytes segmentSizeBytes = SizeInBytes.valueOf(segmentSize);
  String archiveLocation = getConfig().get(Constants.LOG_SERVICE_ARCHIVAL_LOCATION_KEY);
  if (archiveLocation != null) {
    properties.set(Constants.LOG_SERVICE_ARCHIVAL_LOCATION_KEY, archiveLocation);
  }
  heartbeatInterval = getConfig().getLong(Constants.LOG_SERVICE_HEARTBEAT_INTERVAL_KEY,
    Constants.DEFAULT_HEARTBEAT_INTERVAL);
  if(heartbeatInterval <= 0) {
      LOG.warn("Heartbeat interval configuration is invalid." +
              " Setting default value "+ Constants.DEFAULT_HEARTBEAT_INTERVAL);
      heartbeatInterval = Constants.DEFAULT_HEARTBEAT_INTERVAL;
  }
  RaftServerConfigKeys.Log.setSegmentSizeMax(properties, segmentSizeBytes);
  RaftServerConfigKeys.Log.setPreallocatedSize(properties, segmentSizeBytes);

  // TODO this seems to cause errors, not sure if pushing Ratis too hard?
  // SizeInBytes writeBufferSize = SizeInBytes.valueOf("128KB");
  // RaftServerConfigKeys.Log.setWriteBufferSize(properties, writeBufferSize);
}
 

@Test
public void testNoRetryWaitOnNotLeaderException() throws Exception {
  RaftClientConfigKeys.Async.Experimental.setSendDummyRequest(getProperties(), false);
  runWithNewCluster(3, this::runTestNoRetryWaitOnNotLeaderException);
  RaftClientConfigKeys.Async.Experimental.setSendDummyRequest(getProperties(), true);
}
 

void runTestAsyncRequestSemaphore(CLUSTER cluster) throws Exception {
  waitForLeader(cluster);

  int numMessages = RaftClientConfigKeys.Async.maxOutstandingRequests(getProperties());
  CompletableFuture[] futures = new CompletableFuture[numMessages + 1];
  final SimpleMessage[] messages = SimpleMessage.create(numMessages);
  final RaftClient client = cluster.createClient();
  //Set blockTransaction flag so that transaction blocks
  cluster.getServers().stream()
      .map(cluster::getRaftServerImpl)
      .map(SimpleStateMachine4Testing::get)
      .forEach(SimpleStateMachine4Testing::blockStartTransaction);

  //Send numMessages which are blocked and do not release the client semaphore permits
  AtomicInteger blockedRequestsCount = new AtomicInteger();
  for (int i=0; i<numMessages; i++) {
    blockedRequestsCount.getAndIncrement();
    futures[i] = client.sendAsync(messages[i]);
    blockedRequestsCount.decrementAndGet();
  }
  Assert.assertEquals(0, blockedRequestsCount.get());

  futures[numMessages] = CompletableFuture.supplyAsync(() -> {
    blockedRequestsCount.incrementAndGet();
    client.sendAsync(new SimpleMessage("n1"));
    blockedRequestsCount.decrementAndGet();
    return null;
  });

  //Allow the last msg to be sent
  while (blockedRequestsCount.get() != 1) {
    Thread.sleep(1000);
  }
  Assert.assertEquals(1, blockedRequestsCount.get());
  //Since all semaphore permits are acquired the last message sent is in queue
  RaftClientTestUtil.assertAsyncRequestSemaphore(client, 0, 1);

  //Unset the blockTransaction flag so that semaphore permits can be released
  cluster.getServers().stream()
      .map(cluster::getRaftServerImpl)
      .map(SimpleStateMachine4Testing::get)
      .forEach(SimpleStateMachine4Testing::unblockStartTransaction);

  for(int i=0; i<=numMessages; i++){
    futures[i].join();
  }
  Assert.assertEquals(0, blockedRequestsCount.get());
}
 

@Before
public void setup() {
  final RaftProperties p = getProperties();
  p.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY, SimpleStateMachine4Testing.class, StateMachine.class);
  RaftClientConfigKeys.Rpc.setRequestTimeout(p, TimeDuration.valueOf(1, TimeUnit.SECONDS));
}
 

@Test
public void testRaftClientConfigKeys() {
  ConfUtils.printAll(RaftClientConfigKeys.class);
}
 

@Test
public void testGroupMismatchException() throws Exception {
  RaftClientConfigKeys.Async.Experimental.setSendDummyRequest(getProperties(), false);
  runWithNewCluster(1, this::runTestGroupMismatchException);
  RaftClientConfigKeys.Async.Experimental.setSendDummyRequest(getProperties(), true);
}
 

private OrderedAsync(RaftClientImpl client, RaftProperties properties) {
  this.client = Objects.requireNonNull(client, "client == null");
  this.requestSemaphore = new Semaphore(RaftClientConfigKeys.Async.outstandingRequestsMax(properties));
}
 

void runTestAsyncRequestSemaphore(CLUSTER cluster) throws Exception {
  waitForLeader(cluster);

  int numMessages = RaftClientConfigKeys.Async.outstandingRequestsMax(getProperties());
  CompletableFuture[] futures = new CompletableFuture[numMessages + 1];
  final SimpleMessage[] messages = SimpleMessage.create(numMessages);
  try (final RaftClient client = cluster.createClient()) {
    //Set blockTransaction flag so that transaction blocks
    cluster.getServers().stream()
            .map(cluster::getRaftServerImpl)
            .map(SimpleStateMachine4Testing::get)
            .forEach(SimpleStateMachine4Testing::blockStartTransaction);

    //Send numMessages which are blocked and do not release the client semaphore permits
    AtomicInteger blockedRequestsCount = new AtomicInteger();
    for (int i = 0; i < numMessages; i++) {
      blockedRequestsCount.getAndIncrement();
      futures[i] = client.sendAsync(messages[i]);
      blockedRequestsCount.decrementAndGet();
    }
    Assert.assertEquals(0, blockedRequestsCount.get());

    futures[numMessages] = CompletableFuture.supplyAsync(() -> {
      blockedRequestsCount.incrementAndGet();
      client.sendAsync(new SimpleMessage("n1"));
      blockedRequestsCount.decrementAndGet();
      return null;
    });

    //Allow the last msg to be sent
    while (blockedRequestsCount.get() != 1) {
      Thread.sleep(1000);
    }
    Assert.assertEquals(1, blockedRequestsCount.get());
    //Since all semaphore permits are acquired the last message sent is in queue
    RaftClientTestUtil.assertAsyncRequestSemaphore(client, 0, 1);

    //Unset the blockTransaction flag so that semaphore permits can be released
    cluster.getServers().stream()
            .map(cluster::getRaftServerImpl)
            .map(SimpleStateMachine4Testing::get)
            .forEach(SimpleStateMachine4Testing::unblockStartTransaction);

    for (int i = 0; i <= numMessages; i++) {
      futures[i].join();
    }
    Assert.assertEquals(0, blockedRequestsCount.get());
  }
}
 

void runTestRequestAsyncWithRetryFailure(boolean initialMessages) throws Exception {
  RaftClientConfigKeys.Async.Experimental.setSendDummyRequest(getProperties(), false);
  runWithNewCluster(1, initialMessages, cluster -> runTestRequestAsyncWithRetryFailure(initialMessages, cluster));
  RaftClientConfigKeys.Async.Experimental.setSendDummyRequest(getProperties(), true);
}
 

GrpcClientProtocolClient(ClientId id, RaftPeer target, RaftProperties properties, GrpcTlsConfig tlsConf) {
  this.name = JavaUtils.memoize(() -> id + "->" + target.getId());
  this.target = target;
  final SizeInBytes flowControlWindow = GrpcConfigKeys.flowControlWindow(properties, LOG::debug);
  final SizeInBytes maxMessageSize = GrpcConfigKeys.messageSizeMax(properties, LOG::debug);
  NettyChannelBuilder channelBuilder =
      NettyChannelBuilder.forTarget(target.getAddress());

  if (tlsConf!= null) {
    SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
    if (tlsConf.isFileBasedConfig()) {
      sslContextBuilder.trustManager(tlsConf.getTrustStoreFile());
    } else {
      sslContextBuilder.trustManager(tlsConf.getTrustStore());
    }
    if (tlsConf.getMtlsEnabled()) {
      if (tlsConf.isFileBasedConfig()) {
        sslContextBuilder.keyManager(tlsConf.getCertChainFile(),
            tlsConf.getPrivateKeyFile());
      } else {
        sslContextBuilder.keyManager(tlsConf.getPrivateKey(),
            tlsConf.getCertChain());
      }
    }
    try {
      channelBuilder.useTransportSecurity().sslContext(
          sslContextBuilder.build());
    } catch (Exception ex) {
      throw new RuntimeException(ex);
    }
  } else {
    channelBuilder.negotiationType(NegotiationType.PLAINTEXT);
  }
  channel = channelBuilder.flowControlWindow(flowControlWindow.getSizeInt())
      .maxInboundMessageSize(maxMessageSize.getSizeInt())
      .build();
  blockingStub = RaftClientProtocolServiceGrpc.newBlockingStub(channel);
  asyncStub = RaftClientProtocolServiceGrpc.newStub(channel);
  adminBlockingStub = AdminProtocolServiceGrpc.newBlockingStub(channel);
  this.requestTimeoutDuration = RaftClientConfigKeys.Rpc.requestTimeout(properties);
  this.watchRequestTimeoutDuration =
      RaftClientConfigKeys.Rpc.watchRequestTimeout(properties);
}
 

public NettyRpcProxy(RaftPeer peer, RaftProperties properties, EventLoopGroup group) throws InterruptedException {
  this.peer = peer;
  this.connection = new Connection(group);
  this.requestTimeoutDuration = RaftClientConfigKeys.Rpc.requestTimeout(properties);
}
 

private StreamImpl(RaftClientImpl client, RaftProperties properties) {
  this.client = Objects.requireNonNull(client, "client == null");
  this.submessageSize = RaftClientConfigKeys.Stream.submessageSize(properties);
}