org.apache.ratis.grpc.GrpcConfigKeys Java Examples

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDir(properties, Collections.singletonList(storageDir));
  StateMachine stateMachine = new ArithmeticStateMachine();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(getRaftGroupId())),
          getPeers());
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

/**
 * Sets common Ratis server properties for both the log and metadata state machines.
 */
void setRaftProperties(RaftProperties properties) {
  // Set the ports for the server
  GrpcConfigKeys.Server.setPort(properties, opts.getPort());
  NettyConfigKeys.Server.setPort(properties, opts.getPort());

  // Ozone sets the leader election timeout (min) to 1second.
  long leaderElectionTimeoutMinVal = getLeaderElectionTimeoutMin();
  TimeDuration leaderElectionTimeoutMin = TimeDuration.valueOf(leaderElectionTimeoutMinVal,
    TimeUnit.MILLISECONDS);
  RaftServerConfigKeys.Rpc.setTimeoutMin(properties, leaderElectionTimeoutMin);
  long leaderElectionTimeoutMaxVal = getLeaderElectionTimeoutMax();

  TimeDuration leaderElectionMaxTimeout = TimeDuration.valueOf(
    leaderElectionTimeoutMaxVal,
      TimeUnit.MILLISECONDS);
  RaftServerConfigKeys.Rpc.setTimeoutMax(properties, leaderElectionMaxTimeout);
}
 

GrpcClientStreamer(RaftProperties prop, RaftGroup group,
    RaftPeerId leaderId, ClientId clientId, GrpcTlsConfig tlsConfig) {
  this.clientId = clientId;
  maxPendingNum = GrpcConfigKeys.OutputStream.outstandingAppendsMax(prop);
  maxMessageSize = GrpcConfigKeys.messageSizeMax(prop, LOG::debug);
  dataQueue = new ConcurrentLinkedDeque<>();
  ackQueue = new ConcurrentLinkedDeque<>();
  exceptionAndRetry = new ExceptionAndRetry(prop);

  this.groupId = group.getGroupId();
  this.peers = group.getPeers().stream().collect(
      Collectors.toMap(RaftPeer::getId, Function.identity()));
  proxyMap = new PeerProxyMap<>(clientId.toString(),
      raftPeer -> new GrpcClientProtocolProxy(clientId, raftPeer,
          ResponseHandler::new, prop, tlsConfig));
  proxyMap.addPeers(group.getPeers());
  refreshLeaderProxy(leaderId, null);

  senderThread = new Sender();
  senderThread.setName(this.toString() + "-sender");
  senderThread.start();
}
 

GrpcClientStreamer(RaftProperties prop, RaftGroup group,
    RaftPeerId leaderId, ClientId clientId, GrpcTlsConfig tlsConfig) {
  this.clientId = clientId;
  maxPendingNum = GrpcConfigKeys.OutputStream.outstandingAppendsMax(prop);
  maxMessageSize = GrpcConfigKeys.messageSizeMax(prop, LOG::debug);
  dataQueue = new ConcurrentLinkedDeque<>();
  ackQueue = new ConcurrentLinkedDeque<>();
  exceptionAndRetry = new ExceptionAndRetry(prop);

  this.groupId = group.getGroupId();
  this.peers = group.getPeers().stream().collect(
      Collectors.toMap(RaftPeer::getId, Function.identity()));
  proxyMap = new PeerProxyMap<>(clientId.toString(),
      raftPeer -> new GrpcClientProtocolProxy(clientId, raftPeer,
          ResponseHandler::new, prop, tlsConfig));
  proxyMap.addPeers(group.getPeers());
  refreshLeaderProxy(leaderId, null);

  senderThread = new Sender();
  senderThread.setName(this.toString() + "-sender");
  senderThread.start();
}
 

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  RaftPeer[] peers = getPeers();
  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir));
  StateMachine stateMachine = new ArithmeticStateMachine();

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers);
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

@Override
public void run() throws Exception {
  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  RaftPeer[] peers = getPeers();
  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(storageDir));
  ConfUtils.setFile(properties::setFile, FileStoreCommon.STATEMACHINE_DIR_KEY,
      storageDir);
  StateMachine stateMachine = new FileStoreStateMachine(properties);

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)), peers);
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();
  raftServer.start();

  for(; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED;) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

public GrpcLogAppender(RaftServerImpl server, LeaderState leaderState,
                       FollowerInfo f) {
  super(server, leaderState, f);

  this.rpcService = (GrpcService) server.getServerRpc();

  maxPendingRequestsNum = GrpcConfigKeys.Server.leaderOutstandingAppendsMax(
      server.getProxy().getProperties());
  requestTimeoutDuration = RaftServerConfigKeys.Rpc.requestTimeout(server.getProxy().getProperties());
  installSnapshotEnabled = RaftServerConfigKeys.Log.Appender.installSnapshotEnabled(
      server.getProxy().getProperties());
  grpcServerMetrics = new GrpcServerMetrics(server.getMemberId().toString());
  grpcServerMetrics.addPendingRequestsCount(getFollowerId().toString(),
      () -> pendingRequests.logRequestsSize());
}
 

public GrpcLogAppender(RaftServerImpl server, LeaderState leaderState,
                       FollowerInfo f) {
  super(server, leaderState, f);

  this.rpcService = (GrpcService) server.getServerRpc();

  maxPendingRequestsNum = GrpcConfigKeys.Server.leaderOutstandingAppendsMax(
      server.getProxy().getProperties());
  requestTimeoutDuration = RaftServerConfigKeys.Rpc.requestTimeout(server.getProxy().getProperties());
  pendingRequests = new ConcurrentHashMap<>();
}
 

private GrpcService(RaftServer raftServer, Supplier<RaftPeerId> idSupplier, int port,
    SizeInBytes grpcMessageSizeMax, SizeInBytes appenderBufferSize,
    SizeInBytes flowControlWindow,TimeDuration requestTimeoutDuration, GrpcTlsConfig tlsConfig) {
  super(idSupplier, id -> new PeerProxyMap<>(id.toString(),
      p -> new GrpcServerProtocolClient(p, flowControlWindow.getSizeInt(),
          requestTimeoutDuration, tlsConfig)));
  if (appenderBufferSize.getSize() > grpcMessageSizeMax.getSize()) {
    throw new IllegalArgumentException("Illegal configuration: "
        + RaftServerConfigKeys.Log.Appender.BUFFER_BYTE_LIMIT_KEY + " = " + appenderBufferSize
        + " > " + GrpcConfigKeys.MESSAGE_SIZE_MAX_KEY + " = " + grpcMessageSizeMax);
  }

  NettyServerBuilder nettyServerBuilder = NettyServerBuilder.forPort(port)
      .maxInboundMessageSize(grpcMessageSizeMax.getSizeInt())
      .flowControlWindow(flowControlWindow.getSizeInt())
      .addService(new GrpcServerProtocolService(idSupplier, raftServer))
      .addService(new GrpcClientProtocolService(idSupplier, raftServer))
      .addService(new GrpcAdminProtocolService(raftServer));

  if (tlsConfig != null) {
    SslContextBuilder sslContextBuilder =
        SslContextBuilder.forServer(tlsConfig.getCertChain(),
            tlsConfig.getPrivateKey());
    if (tlsConfig.getMtlsEnabled()) {
      sslContextBuilder.clientAuth(ClientAuth.REQUIRE);
      sslContextBuilder.trustManager(tlsConfig.getCertChain());
    }
    sslContextBuilder = GrpcSslContexts.configure(sslContextBuilder, OPENSSL);
    try {
      nettyServerBuilder.sslContext(sslContextBuilder.build());
    } catch (Exception ex) {
      throw new IllegalArgumentException("Failed to build SslContext, tlsConfig=" + tlsConfig, ex);
    }
  }
  server = nettyServerBuilder.build();
  addressSupplier = JavaUtils.memoize(() -> new InetSocketAddress(port != 0? port: server.getPort()));
}
 

private GrpcService(RaftServer server, GrpcTlsConfig tlsConfig) {
  this(server, server::getId,
      GrpcConfigKeys.Server.port(server.getProperties()),
      GrpcConfigKeys.messageSizeMax(server.getProperties(), LOG::info),
      RaftServerConfigKeys.Log.Appender.bufferByteLimit(server.getProperties()),
      GrpcConfigKeys.flowControlWindow(server.getProperties(), LOG::info),
      RaftServerConfigKeys.Rpc.requestTimeout(server.getProperties()),
      tlsConfig);
}
 

public GrpcOutputStream(RaftProperties prop, ClientId clientId,
    RaftGroup group, RaftPeerId leaderId, GrpcTlsConfig tlsConfig) {
  final int bufferSize = GrpcConfigKeys.OutputStream.bufferSize(prop).getSizeInt();
  buf = new byte[bufferSize];
  count = 0;
  this.clientId = clientId;
  streamer = new GrpcClientStreamer(prop, group, leaderId, clientId, tlsConfig);
}
 

public GrpcClientRpc(ClientId clientId, RaftProperties properties, GrpcTlsConfig tlsConfig) {
  super(new PeerProxyMap<>(clientId.toString(),
      p -> new GrpcClientProtocolClient(clientId, p, properties, tlsConfig)));
  this.clientId = clientId;
  this.maxMessageSize = GrpcConfigKeys.messageSizeMax(properties, LOG::debug).getSizeInt();
  this.tlsConfig = tlsConfig;
}
 

public GrpcClientProtocolClient(ClientId id, RaftPeer target,
                                RaftProperties properties,
                                GrpcTlsConfig tlsConf) {
  this.name = JavaUtils.memoize(() -> id + "->" + target.getId());
  this.target = target;
  final SizeInBytes flowControlWindow = GrpcConfigKeys.flowControlWindow(properties, LOG::debug);
  final SizeInBytes maxMessageSize = GrpcConfigKeys.messageSizeMax(properties, LOG::debug);
  NettyChannelBuilder channelBuilder =
      NettyChannelBuilder.forTarget(target.getAddress());

  if (tlsConf!= null) {
    SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
    if (tlsConf.getTrustStore() != null) {
      sslContextBuilder.trustManager(tlsConf.getTrustStore());
    }
    if (tlsConf.getMtlsEnabled()) {
      sslContextBuilder.keyManager(tlsConf.getCertChain(),
          tlsConf.getPrivateKey());
    }
    try {
      channelBuilder.useTransportSecurity().sslContext(sslContextBuilder.build());
    } catch (Exception ex) {
      throw new RuntimeException(ex);
    }
  } else {
    channelBuilder.negotiationType(NegotiationType.PLAINTEXT);
  }
  channel = channelBuilder.flowControlWindow(flowControlWindow.getSizeInt())
      .maxInboundMessageSize(maxMessageSize.getSizeInt())
      .build();
  blockingStub = RaftClientProtocolServiceGrpc.newBlockingStub(channel);
  asyncStub = RaftClientProtocolServiceGrpc.newStub(channel);
  adminBlockingStub = AdminProtocolServiceGrpc.newBlockingStub(channel);
  this.requestTimeoutDuration = RaftClientConfigKeys.Rpc.requestTimeout(properties);
}
 

public void start() throws IOException  {
    final ServerOpts opts = getServerOpts();
    if (opts.getHost() == null) {
        opts.setHost(LogServiceUtils.getHostName());
    }
    this.lifeCycle = new LifeCycle(this.id);
    RaftProperties properties = new RaftProperties();
    if(opts.getWorkingDir() != null) {
        RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(new File(opts.getWorkingDir())));
    }
    GrpcConfigKeys.Server.setPort(properties, opts.getPort());
    NettyConfigKeys.Server.setPort(properties, opts.getPort());
    Set<RaftPeer> peers = getPeersFromQuorum(opts.getMetaQuorum());
    RaftGroupId raftMetaGroupId = RaftGroupId.valueOf(opts.getMetaGroupId());
    RaftGroup metaGroup = RaftGroup.valueOf(raftMetaGroupId, peers);
    metaStateMachine = new MetaStateMachine(raftMetaGroupId, RaftGroupId.valueOf(opts.getLogServerGroupId()));
    server = RaftServer.newBuilder()
            .setGroup(metaGroup)
            .setServerId(RaftPeerId.valueOf(id))
            .setStateMachineRegistry(raftGroupId -> {
                if(raftGroupId.equals(META_GROUP_ID)) {
                    return metaStateMachine;
                }
                return null;
            })
            .setProperties(properties).build();
    lifeCycle.startAndTransition(() -> {
        server.start();
    }, IOException.class);
}
 

@Override
public void run() throws Exception {
  int raftSegmentPreallocatedSize = 1024 * 1024 * 1024;
  RaftProperties raftProperties = new RaftProperties();
  RaftConfigKeys.Rpc.setType(raftProperties, SupportedRpcType.GRPC);
  GrpcConfigKeys.setMessageSizeMax(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.Appender.setBufferByteLimit(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setWriteBufferSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setPreallocatedSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setSegmentSizeMax(raftProperties,
      SizeInBytes.valueOf(1 * 1024 * 1024 * 1024));

  RaftServerConfigKeys.Log.setMaxCachedSegmentNum(raftProperties, 2);

  RaftClientConfigKeys.Rpc.setRequestTimeout(raftProperties,
      TimeDuration.valueOf(50000, TimeUnit.MILLISECONDS));
  RaftClientConfigKeys.Async.setSchedulerThreads(raftProperties, 10);
  RaftClientConfigKeys.Async.setMaxOutstandingRequests(raftProperties, 1000);


  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(raftGroupId)),
      parsePeers(peers));

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);
}
 

private GrpcService(RaftServer server, GrpcTlsConfig tlsConfig) {
  this(server, server::getId,
      GrpcConfigKeys.Server.port(server.getProperties()),
      GrpcConfigKeys.messageSizeMax(server.getProperties(), LOG::info),
      RaftServerConfigKeys.Log.Appender.bufferByteLimit(server.getProperties()),
      GrpcConfigKeys.flowControlWindow(server.getProperties(), LOG::info),
      RaftServerConfigKeys.Rpc.requestTimeout(server.getProperties()),
      tlsConfig);
}
 

public GrpcOutputStream(RaftProperties prop, ClientId clientId,
    RaftGroup group, RaftPeerId leaderId, GrpcTlsConfig tlsConfig) {
  final int bufferSize = GrpcConfigKeys.OutputStream.bufferSize(prop).getSizeInt();
  buf = new byte[bufferSize];
  count = 0;
  this.clientId = clientId;
  streamer = new GrpcClientStreamer(prop, group, leaderId, clientId, tlsConfig);
}
 

public GrpcClientRpc(ClientId clientId, RaftProperties properties, GrpcTlsConfig tlsConfig) {
  super(new PeerProxyMap<>(clientId.toString(),
      p -> new GrpcClientProtocolClient(clientId, p, properties, tlsConfig)));
  this.clientId = clientId;
  this.maxMessageSize = GrpcConfigKeys.messageSizeMax(properties, LOG::debug).getSizeInt();
  this.tlsConfig = tlsConfig;
}
 

public static void main(String[] args) throws IOException, InterruptedException {
  if (args.length < 1) {
    System.err.println("Usage: java -cp *.jar org.apache.ratis.examples.counter.server.CounterServer {serverIndex}");
    System.err.println("{serverIndex} could be 1, 2 or 3");
    System.exit(1);
  }

  //find current peer object based on application parameter
  RaftPeer currentPeer =
      CounterCommon.PEERS.get(Integer.parseInt(args[0]) - 1);

  //create a property object
  RaftProperties properties = new RaftProperties();

  //set the storage directory (different for each peer) in RaftProperty object
  File raftStorageDir = new File("./" + currentPeer.getId().toString());
  RaftServerConfigKeys.setStorageDir(properties,
      Collections.singletonList(raftStorageDir));

  //set the port which server listen to in RaftProperty object
  final int port = NetUtils.createSocketAddr(currentPeer.getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);

  //create the counter state machine which hold the counter value
  CounterStateMachine counterStateMachine = new CounterStateMachine();

  //create and start the Raft server
  RaftServer server = RaftServer.newBuilder()
      .setGroup(CounterCommon.RAFT_GROUP)
      .setProperties(properties)
      .setServerId(currentPeer.getId())
      .setStateMachine(counterStateMachine)
      .build();
  server.start();

  //exit when any input entered
  Scanner scanner = new Scanner(System.in);
  scanner.nextLine();
  server.close();
}
 

@Override
public void run() throws Exception {
  int raftSegmentPreallocatedSize = 1024 * 1024 * 1024;
  RaftProperties raftProperties = new RaftProperties();
  RaftConfigKeys.Rpc.setType(raftProperties, SupportedRpcType.GRPC);
  GrpcConfigKeys.setMessageSizeMax(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.Appender.setBufferByteLimit(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setWriteBufferSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setPreallocatedSize(raftProperties,
      SizeInBytes.valueOf(raftSegmentPreallocatedSize));
  RaftServerConfigKeys.Log.setSegmentSizeMax(raftProperties,
      SizeInBytes.valueOf(1 * 1024 * 1024 * 1024));

  RaftServerConfigKeys.Log.setSegmentCacheNumMax(raftProperties, 2);

  RaftClientConfigKeys.Rpc.setRequestTimeout(raftProperties,
      TimeDuration.valueOf(50000, TimeUnit.MILLISECONDS));
  RaftClientConfigKeys.Async.setOutstandingRequestsMax(raftProperties, 1000);


  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(getRaftGroupId())),
          getPeers());

  RaftClient.Builder builder =
      RaftClient.newBuilder().setProperties(raftProperties);
  builder.setRaftGroup(raftGroup);
  builder.setClientRpc(new GrpcFactory(new Parameters()).newRaftClientRpc(ClientId.randomId(), raftProperties));
  RaftClient client = builder.build();

  operation(client);
}
 

@Override
public void run() throws Exception {
  JVMMetrics.initJvmMetrics(TimeDuration.valueOf(10, TimeUnit.SECONDS));

  RaftPeerId peerId = RaftPeerId.valueOf(id);
  RaftProperties properties = new RaftProperties();

  final int port = NetUtils.createSocketAddr(getPeer(peerId).getAddress()).getPort();
  GrpcConfigKeys.Server.setPort(properties, port);
  properties.setInt(GrpcConfigKeys.OutputStream.RETRY_TIMES_KEY, Integer.MAX_VALUE);
  RaftServerConfigKeys.setStorageDir(properties, Collections.singletonList(storageDir));
  ConfUtils.setFile(properties::setFile, FileStoreCommon.STATEMACHINE_DIR_KEY,
      storageDir);
  StateMachine stateMachine = new FileStoreStateMachine(properties);

  final RaftGroup raftGroup = RaftGroup.valueOf(RaftGroupId.valueOf(ByteString.copyFromUtf8(getRaftGroupId())),
          getPeers());
  RaftServer raftServer = RaftServer.newBuilder()
      .setServerId(RaftPeerId.valueOf(id))
      .setStateMachine(stateMachine).setProperties(properties)
      .setGroup(raftGroup)
      .build();

  raftServer.start();

  for (; raftServer.getLifeCycleState() != LifeCycle.State.CLOSED; ) {
    TimeUnit.SECONDS.sleep(1);
  }
}
 

private static boolean isGrpcClientConfig(String key) {
  return key.startsWith(GrpcConfigKeys.PREFIX) && !key
      .startsWith(GrpcConfigKeys.TLS.PREFIX) && !key
      .startsWith(GrpcConfigKeys.Server.PREFIX);
}
 

@SuppressWarnings("checkstyle:ParameterNumber") // private constructor
private GrpcService(RaftServer raftServer, Supplier<RaftPeerId> idSupplier, int port,
    SizeInBytes grpcMessageSizeMax, SizeInBytes appenderBufferSize,
    SizeInBytes flowControlWindow,TimeDuration requestTimeoutDuration, GrpcTlsConfig tlsConfig) {
  super(idSupplier, id -> new PeerProxyMap<>(id.toString(),
      p -> new GrpcServerProtocolClient(p, flowControlWindow.getSizeInt(),
          requestTimeoutDuration, tlsConfig)));
  if (appenderBufferSize.getSize() > grpcMessageSizeMax.getSize()) {
    throw new IllegalArgumentException("Illegal configuration: "
        + RaftServerConfigKeys.Log.Appender.BUFFER_BYTE_LIMIT_KEY + " = " + appenderBufferSize
        + " > " + GrpcConfigKeys.MESSAGE_SIZE_MAX_KEY + " = " + grpcMessageSizeMax);
  }

  this.clientProtocolService = new GrpcClientProtocolService(idSupplier, raftServer);

  NettyServerBuilder nettyServerBuilder = NettyServerBuilder.forPort(port)
      .withChildOption(ChannelOption.SO_REUSEADDR, true)
      .maxInboundMessageSize(grpcMessageSizeMax.getSizeInt())
      .flowControlWindow(flowControlWindow.getSizeInt())
      .addService(new GrpcServerProtocolService(idSupplier, raftServer))
      .addService(clientProtocolService)
      .addService(new GrpcAdminProtocolService(raftServer));

  if (tlsConfig != null) {
    SslContextBuilder sslContextBuilder =
        tlsConfig.isFileBasedConfig()?
            SslContextBuilder.forServer(tlsConfig.getCertChainFile(),
                tlsConfig.getPrivateKeyFile()):
            SslContextBuilder.forServer(tlsConfig.getPrivateKey(),
                tlsConfig.getCertChain());
    if (tlsConfig.getMtlsEnabled()) {
      sslContextBuilder.clientAuth(ClientAuth.REQUIRE);
      if (tlsConfig.isFileBasedConfig()) {
        sslContextBuilder.trustManager(tlsConfig.getTrustStoreFile());
      } else {
          sslContextBuilder.trustManager(tlsConfig.getTrustStore());
      }
    }
    sslContextBuilder = GrpcSslContexts.configure(sslContextBuilder, OPENSSL);
    try {
      nettyServerBuilder.sslContext(sslContextBuilder.build());
    } catch (Exception ex) {
      throw new IllegalArgumentException("Failed to build SslContext, tlsConfig=" + tlsConfig, ex);
    }
  }
  server = nettyServerBuilder.build();
  addressSupplier = JavaUtils.memoize(() -> new InetSocketAddress(port != 0? port: server.getPort()));
}
 

public void start() throws IOException {
    final ServerOpts opts = getServerOpts();
    Set<RaftPeer> peers = LogServiceUtils.getPeersFromQuorum(opts.getMetaQuorum());
    RaftProperties properties = new RaftProperties();
    properties.set("raft.client.rpc.request.timeout", "100000");
    GrpcConfigKeys.Server.setPort(properties, opts.getPort());
    NettyConfigKeys.Server.setPort(properties, opts.getPort());
    InetSocketAddress addr = new InetSocketAddress(opts.getHost(), opts.getPort());
    if(opts.getWorkingDir() != null) {
        RaftServerConfigKeys.setStorageDirs(properties, Collections.singletonList(new File(opts.getWorkingDir())));
    }
    String id = opts.getHost() +"_" +  opts.getPort();
    RaftPeer peer = new RaftPeer(RaftPeerId.valueOf(id), addr);
    final RaftGroupId logServerGroupId = RaftGroupId.valueOf(opts.getLogServerGroupId());
    RaftGroup all = RaftGroup.valueOf(logServerGroupId, peer);
    RaftGroup meta = RaftGroup.valueOf(RaftGroupId.valueOf(opts.getMetaGroupId()), peers);
    raftServer = RaftServer.newBuilder()
            .setStateMachineRegistry(new StateMachine.Registry() {
                private final StateMachine managementMachine = new ManagementStateMachine();
                private final StateMachine logMachine  = new LogStateMachine();
                @Override
                public StateMachine apply(RaftGroupId raftGroupId) {
                    // TODO this looks wrong. Why isn't this metaGroupId?
                    if(raftGroupId.equals(logServerGroupId)) {
                        return managementMachine;
                    }
                    return logMachine;
                }
            })
            .setProperties(properties)
            .setServerId(RaftPeerId.valueOf(id))
            .setGroup(all)
            .build();
    raftServer.start();

    metaClient = RaftClient.newBuilder()
            .setRaftGroup(meta)
            .setClientId(ClientId.randomId())
            .setProperties(properties)
            .build();
    metaClient.send(() -> MetaServiceProtoUtil.toPingRequestProto(peer).toByteString());
}
 

ExceptionAndRetry(RaftProperties prop) {
  maxRetryTimes = GrpcConfigKeys.OutputStream.retryTimes(prop);
  retryInterval = GrpcConfigKeys.OutputStream.retryInterval(prop);
}
 

ExceptionAndRetry(RaftProperties prop) {
  maxRetryTimes = GrpcConfigKeys.OutputStream.retryTimes(prop);
  retryInterval = GrpcConfigKeys.OutputStream.retryInterval(prop);
}
 

GrpcClientProtocolClient(ClientId id, RaftPeer target, RaftProperties properties, GrpcTlsConfig tlsConf) {
  this.name = JavaUtils.memoize(() -> id + "->" + target.getId());
  this.target = target;
  final SizeInBytes flowControlWindow = GrpcConfigKeys.flowControlWindow(properties, LOG::debug);
  final SizeInBytes maxMessageSize = GrpcConfigKeys.messageSizeMax(properties, LOG::debug);
  NettyChannelBuilder channelBuilder =
      NettyChannelBuilder.forTarget(target.getAddress());

  if (tlsConf!= null) {
    SslContextBuilder sslContextBuilder = GrpcSslContexts.forClient();
    if (tlsConf.isFileBasedConfig()) {
      sslContextBuilder.trustManager(tlsConf.getTrustStoreFile());
    } else {
      sslContextBuilder.trustManager(tlsConf.getTrustStore());
    }
    if (tlsConf.getMtlsEnabled()) {
      if (tlsConf.isFileBasedConfig()) {
        sslContextBuilder.keyManager(tlsConf.getCertChainFile(),
            tlsConf.getPrivateKeyFile());
      } else {
        sslContextBuilder.keyManager(tlsConf.getPrivateKey(),
            tlsConf.getCertChain());
      }
    }
    try {
      channelBuilder.useTransportSecurity().sslContext(
          sslContextBuilder.build());
    } catch (Exception ex) {
      throw new RuntimeException(ex);
    }
  } else {
    channelBuilder.negotiationType(NegotiationType.PLAINTEXT);
  }
  channel = channelBuilder.flowControlWindow(flowControlWindow.getSizeInt())
      .maxInboundMessageSize(maxMessageSize.getSizeInt())
      .build();
  blockingStub = RaftClientProtocolServiceGrpc.newBlockingStub(channel);
  asyncStub = RaftClientProtocolServiceGrpc.newStub(channel);
  adminBlockingStub = AdminProtocolServiceGrpc.newBlockingStub(channel);
  this.requestTimeoutDuration = RaftClientConfigKeys.Rpc.requestTimeout(properties);
  this.watchRequestTimeoutDuration =
      RaftClientConfigKeys.Rpc.watchRequestTimeout(properties);
}
 

@Test
public void testGrpcConfigKeys() {
  ConfUtils.printAll(GrpcConfigKeys.class);
}