org.apache.ratis.util.JavaUtils Java Examples

@Override
public EntryWithData getEntryWithData(long index) throws RaftLogIOException {
  final LogEntryProto entry = get(index);
  if (!ServerProtoUtils.shouldReadStateMachineData(entry)) {
    return new EntryWithData(entry, null);
  }

  try {
    return new EntryWithData(entry, server.map(s -> s.getStateMachine().readStateMachineData(entry)).orElse(null));
  } catch (Throwable e) {
    final String err = getSelfId() + ": Failed readStateMachineData for " +
        ServerProtoUtils.toLogEntryString(entry);
    LOG.error(err, e);
    throw new RaftLogIOException(err, JavaUtils.unwrapCompletionException(e));
  }
}
 

@Override
public CompletableFuture<RaftClientReply> groupManagementAsync(GroupManagementRequest request) {
  final RaftGroupId groupId = request.getRaftGroupId();
  if (groupId == null) {
    return JavaUtils.completeExceptionally(new GroupMismatchException(
        getId() + ": Request group id == null"));
  }
  final GroupManagementRequest.Add add = request.getAdd();
  if (add != null) {
    return groupAddAsync(request, add.getGroup());
  }
  final GroupManagementRequest.Remove remove = request.getRemove();
  if (remove != null) {
    return groupRemoveAsync(request, remove.getGroupId(), remove.isDeleteDirectory());
  }
  return JavaUtils.completeExceptionally(new UnsupportedOperationException(
      getId() + ": Request not supported " + request));
}
 

static void runTestRaftLogMetrics(MiniRaftCluster cluster) throws Exception {
  int numMsg = 2;
  final RaftTestUtil.SimpleMessage[] messages = RaftTestUtil.SimpleMessage.create(numMsg);

  try (final RaftClient client = cluster.createClient()) {
    for (RaftTestUtil.SimpleMessage message : messages) {
      client.send(message);
    }
  }

  // For leader, flush must happen before client can get replies.
  assertFlushCount(cluster.getLeader());
  assertRaftLogWritePathMetrics(cluster.getLeader());

  // For followers, flush can be lagged behind.  Attempt multiple times.
  for(RaftServerImpl f : cluster.getFollowers()) {
    JavaUtils.attempt(() -> assertFlushCount(f), 10, HUNDRED_MILLIS, f.getId() + "-assertFlushCount", null);
    // We have already waited enough for follower metrics to populate.
    assertRaftLogWritePathMetrics(f);
  }

  // Wait for commits to happen on leader
  JavaUtils.attempt(() -> assertCommitCount(cluster.getLeader(), numMsg), 10, HUNDRED_MILLIS, cluster.getLeader().getId() + "-assertCommitCount", null);
}
 

static void assertFlushCount(RaftServerImpl server) throws Exception {
  final String flushTimeMetric = RaftStorageTestUtils.getLogFlushTimeMetric(server.getMemberId().toString());
  RatisMetricRegistry ratisMetricRegistry = new RaftLogMetrics(server.getMemberId().toString()).getRegistry();
  Timer tm = (Timer) ratisMetricRegistry.get(RAFT_LOG_FLUSH_TIME);
  Assert.assertNotNull(tm);

  final MetricsStateMachine stateMachine = MetricsStateMachine.get(server);
  final int expectedFlush = stateMachine.getFlushCount();

  JavaUtils.attemptRepeatedly(() -> {
    Assert.assertEquals(expectedFlush, tm.getCount());
    return null;
  }, 50, HUNDRED_MILLIS, "expectedFlush == tm.getCount()", null);

  Assert.assertTrue(tm.getMeanRate() > 0);

  // Test jmx
  ObjectName oname = new ObjectName(RATIS_APPLICATION_NAME_METRICS, "name", flushTimeMetric);
  Assert.assertEquals(expectedFlush,
      ((Long) ManagementFactory.getPlatformMBeanServer().getAttribute(oname, "Count"))
          .intValue());
}
 

private long load(SingleFileSnapshotInfo snapshot, boolean reload) throws IOException {
  if (snapshot == null) {
    LOG.warn("The snapshot info is null.");
    return RaftServerConstants.INVALID_LOG_INDEX;
  }
  final File snapshotFile = snapshot.getFile().getPath().toFile();
  if (!snapshotFile.exists()) {
    LOG.warn("The snapshot file {} does not exist for snapshot {}", snapshotFile, snapshot);
    return RaftServerConstants.INVALID_LOG_INDEX;
  }

  final TermIndex last = SimpleStateMachineStorage.getTermIndexFromSnapshotFile(snapshotFile);
  try(final AutoCloseableLock writeLock = writeLock();
      final ObjectInputStream in = new ObjectInputStream(
          new BufferedInputStream(new FileInputStream(snapshotFile)))) {
    if (reload) {
      reset();
    }
    setLastAppliedTermIndex(last);
    variables.putAll(JavaUtils.cast(in.readObject()));
  } catch (ClassNotFoundException e) {
    throw new IllegalStateException(e);
  }
  return last.getIndex();
}
 

/**
 * Query the n-th log entry.
 * @param request an index represented in a UTF-8 String, or an empty message.
 * @return a completed future of the n-th log entry,
 *         where n is the last applied index if the request is empty,
 *         otherwise, n is the index represented in the request.
 */
@Override
public CompletableFuture<Message> query(Message request) {
  final String string = request.getContent().toStringUtf8();
  Exception exception;
  try {
    LOG.info("query " + string);
    final LogEntryProto entry = dataMap.get(string);
    if (entry != null) {
      return CompletableFuture.completedFuture(Message.valueOf(entry.toByteString()));
    }
    exception = new IndexOutOfBoundsException(getId() + ": LogEntry not found for query " + string);
  } catch (Exception e) {
    LOG.warn("Failed request " + request, e);
    exception = e;
  }
  return JavaUtils.completeExceptionally(new StateMachineException(
      "Failed request " + request, exception));
}
 

CompletableFuture<RaftClientReply> onNext(RaftClientRequest request) {
  final long callId = request.getCallId();
  final CompletableFuture<RaftClientReply> f = replies.putNew(callId);
  if (f == null) {
    return JavaUtils.completeExceptionally(new AlreadyClosedException(getName() + " is closed."));
  }
  try {
    if (!requestStreamer.onNext(ClientProtoUtils.toRaftClientRequestProto(request))) {
      return JavaUtils.completeExceptionally(new AlreadyClosedException(getName() + ": the stream is closed."));
    }
  } catch(Throwable t) {
    handleReplyFuture(request.getCallId(), future -> future.completeExceptionally(t));
    return f;
  }

  if (RaftClientRequestProto.TypeCase.WATCH.equals(request.getType().getTypeCase())) {
    scheduler.onTimeout(watchRequestTimeoutDuration, () ->
            timeoutCheck(callId, watchRequestTimeoutDuration), LOG,
        () -> "Timeout check failed for client request #" + callId);
  } else {
    scheduler.onTimeout(requestTimeoutDuration,
        () -> timeoutCheck(callId, requestTimeoutDuration), LOG,
        () -> "Timeout check failed for client request #" + callId);
  }
  return f;
}
 

CompletableFuture<RaftClientReply> send(RaftClientRequest.Type type, Message message, RaftPeerId server) {
  if (!type.is(TypeCase.WATCH) && !type.is(TypeCase.STREAM)) {
    Objects.requireNonNull(message, "message == null");
  }
  try {
    requestSemaphore.acquire();
  } catch (InterruptedException e) {
    return JavaUtils.completeExceptionally(IOUtils.toInterruptedIOException(
        "Interrupted when sending " + type + ", message=" + message, e));
  }

  final long callId = RaftClientImpl.nextCallId();
  final LongFunction<PendingOrderedRequest> constructor = seqNum -> new PendingOrderedRequest(callId, seqNum,
      slidingWindowEntry -> client.newRaftClientRequest(server, callId, message, type, slidingWindowEntry));
  return getSlidingWindow(server).submitNewRequest(constructor, this::sendRequestWithRetry
  ).getReplyFuture(
  ).thenApply(reply -> RaftClientImpl.handleRaftException(reply, CompletionException::new)
  ).whenComplete((r, e) -> requestSemaphore.release());
}
 

RaftClientImpl(ClientId clientId, RaftGroup group, RaftPeerId leaderId,
    RaftClientRpc clientRpc, RaftProperties properties, RetryPolicy retryPolicy) {
  this.clientId = clientId;
  this.clientRpc = clientRpc;
  this.peers = new ConcurrentLinkedQueue<>(group.getPeers());
  this.groupId = group.getGroupId();
  this.leaderId = leaderId != null? leaderId
      : !peers.isEmpty()? peers.iterator().next().getId(): null;
  Preconditions.assertTrue(retryPolicy != null, "retry policy can't be null");
  this.retryPolicy = retryPolicy;

  scheduler = TimeoutScheduler.getInstance();
  clientRpc.addServers(peers);

  this.orderedAsync = JavaUtils.memoize(() -> OrderedAsync.newInstance(this, properties));
  this.streamApi = JavaUtils.memoize(() -> StreamImpl.newInstance(this, properties));
}
 

@Override
public EntryWithData getEntryWithData(long index) throws RaftLogIOException {
  final LogEntryProto entry = get(index);
  if (entry == null) {
    throw new RaftLogIOException("Log entry not found: index = " + index);
  }
  if (!ServerProtoUtils.shouldReadStateMachineData(entry)) {
    return new EntryWithData(entry, null);
  }

  try {
    CompletableFuture<ByteString> future = null;
    if (stateMachine != null) {
      future = stateMachine.data().read(entry).exceptionally(ex -> {
        stateMachine.notifyLogFailed(ex, entry);
        return null;
      });
    }
    return new EntryWithData(entry, future);
  } catch (Throwable e) {
    final String err = getName() + ": Failed readStateMachineData for " +
        ServerProtoUtils.toLogEntryString(entry);
    LOG.error(err, e);
    throw new RaftLogIOException(err, JavaUtils.unwrapCompletionException(e));
  }
}
 

synchronized CompletableFuture<RaftServerImpl> addNew(RaftGroup group) {
  if (isClosed) {
    return JavaUtils.completeExceptionally(new AlreadyClosedException(
        getId() + ": Failed to add " + group + " since the server is already closed"));
  }
  if (containsGroup(group.getGroupId())) {
    return JavaUtils.completeExceptionally(new AlreadyExistsException(
        getId() + ": Failed to add " + group + " since the group already exists in the map."));
  }
  final RaftGroupId groupId = group.getGroupId();
  final CompletableFuture<RaftServerImpl> newImpl = newRaftServerImpl(group);
  final CompletableFuture<RaftServerImpl> previous = map.put(groupId, newImpl);
  Preconditions.assertNull(previous, "previous");
  LOG.info("{}: addNew {} returns {}", getId(), group, toString(groupId, newImpl));
  return newImpl;
}
 

void runTestHandleNotLeaderException(boolean killNewLeader, CLUSTER cluster) throws Exception {
  final RaftPeerId oldLeader = RaftTestUtil.waitForLeader(cluster).getId();
  try(final RaftClient client = cluster.createClient(oldLeader)) {
    sendMessage("m1", client);

    // enforce leader change
    final RaftPeerId newLeader = RaftTestUtil.changeLeader(cluster, oldLeader);

    if (killNewLeader) {
      // kill the new leader
      cluster.killServer(newLeader);
    }

    final RaftClientRpc rpc = client.getClientRpc();
    JavaUtils.attempt(() -> assertNotLeaderException(newLeader, "m2", oldLeader, rpc, cluster),
        10, ONE_SECOND, "assertNotLeaderException", LOG);

    sendMessage("m3", client);
  }
}
 

public static RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry, RaftProperties properties, Parameters parameters)
    throws IOException {
  final TimeDuration sleepTime = TimeDuration.valueOf(500, TimeUnit.MILLISECONDS);
  final RaftServerProxy proxy;
  try {
    // attempt multiple times to avoid temporary bind exception
    proxy = JavaUtils.attemptRepeatedly(
        () -> new RaftServerProxy(id, stateMachineRegistry, properties, parameters),
        5, sleepTime, "new RaftServerProxy", RaftServerProxy.LOG);
  } catch (InterruptedException e) {
    throw IOUtils.toInterruptedIOException(
        "Interrupted when creating RaftServer " + id, e);
  }
  return proxy;
}
 

default void runWithNewCluster(int numServers, boolean startCluster, CheckedConsumer<CLUSTER, Exception> testCase)
    throws Exception {
  final StackTraceElement caller = JavaUtils.getCallerStackTraceElement();
  LOG.info("Running " + caller.getMethodName());
  final CLUSTER cluster = newCluster(numServers);
  try {
    if (startCluster) {
      cluster.start();
    }
    testCase.accept(cluster);
  } catch(Throwable t) {
    LOG.info(cluster.printServers());
    LOG.error("Failed " + caller, t);
    throw t;
  } finally {
    cluster.shutdown();
  }
}
 

public static void assertLeaderContent(MiniRaftCluster cluster) throws Exception {
  final RaftServerImpl leader = RaftTestUtil.waitForLeader(cluster);
  final RaftLog leaderLog = leader.getState().getLog();
  final long lastIndex = leaderLog.getLastEntryTermIndex().getIndex();
  final LogEntryProto e = leaderLog.get(lastIndex);
  Assert.assertTrue(e.hasMetadataEntry());

  JavaUtils.attemptRepeatedly(() -> {
    Assert.assertEquals(leaderLog.getLastCommittedIndex() - 1, e.getMetadataEntry().getCommitIndex());
    return null;
  }, 50, BaseTest.HUNDRED_MILLIS, "CheckMetadataEntry", LOG);

  SimpleStateMachine4Testing simpleStateMachine = SimpleStateMachine4Testing.get(leader);
  Assert.assertTrue("Is not notified as a leader", simpleStateMachine.isNotifiedAsLeader());
  final LogEntryProto[] entries = simpleStateMachine.getContent();
  long message = 0;
  for (int i = 0; i < entries.length; i++) {
    LOG.info("{}) {} {}", i, message, entries[i]);
    if (entries[i].hasStateMachineLogEntry()) {
      final SimpleMessage m = new SimpleMessage("m" + message++);
      Assert.assertArrayEquals(m.getContent().toByteArray(),
          entries[i].getStateMachineLogEntry().getLogData().toByteArray());
    }
  }
}
 

/**
 * Query the n-th log entry.
 * @param request an index represented in a UTF-8 String, or an empty message.
 * @return a completed future of the n-th log entry,
 *         where n is the last applied index if the request is empty,
 *         otherwise, n is the index represented in the request.
 */
@Override
public CompletableFuture<Message> query(Message request) {
  final String string = request.getContent().toStringUtf8();
  Exception exception;
  try {
    LOG.info("query " + string);
    final LogEntryProto entry = dataMap.get(string);
    if (entry != null) {
      return CompletableFuture.completedFuture(Message.valueOf(entry.toByteString()));
    }
    exception = new IndexOutOfBoundsException(getId() + ": LogEntry not found for query " + string);
  } catch (Exception e) {
    LOG.warn("Failed request " + request, e);
    exception = e;
  }
  return JavaUtils.completeExceptionally(new StateMachineException(
      "Failed request " + request, exception));
}
 

@Override
public void testFailure(Failure failure) {
  final Throwable timeoutException = getTimeoutException(failure);
  if (timeoutException != null) {
    out.format("%n%s ", JavaUtils.date());
    timeoutException.printStackTrace(out);
    JavaUtils.dumpAllThreads(out::println);
  }
}
 

private void sendRequestWithRetry(PendingOrderedRequest pending) {
  final CompletableFuture<RaftClientReply> f = pending.getReplyFuture();
  if (f.isDone()) {
    return;
  }

  final RaftClientRequest request = pending.newRequestImpl();
  if (request == null) { // already done
    LOG.debug("{} newRequestImpl returns null", pending);
    return;
  }

  final RetryPolicy retryPolicy = client.getRetryPolicy();
  sendRequest(pending).thenAccept(reply -> {
    if (f.isDone()) {
      return;
    }
    if (reply == null) {
      scheduleWithTimeout(pending, request, retryPolicy, null);
    } else {
      f.complete(reply);
    }
  }).exceptionally(e -> {
    if (e instanceof CompletionException) {
      e = JavaUtils.unwrapCompletionException(e);
      scheduleWithTimeout(pending, request, retryPolicy, e);
      return null;
    }
    f.completeExceptionally(e);
    return null;
  });
}
 

private static RaftServerProxy newRaftServer(
    RaftPeerId id, StateMachine.Registry stateMachineRegistry, RaftProperties properties, Parameters parameters)
    throws IOException {
  final RaftServerProxy proxy;
  try {
    // attempt multiple times to avoid temporary bind exception
    proxy = JavaUtils.attempt(
        () -> new RaftServerProxy(id, stateMachineRegistry, properties, parameters),
        5, 500L, "new RaftServerProxy", RaftServerProxy.LOG);
  } catch (InterruptedException e) {
    throw IOUtils.toInterruptedIOException(
        "Interrupted when creating RaftServer " + id, e);
  }
  return proxy;
}
 

@Override
public CompletableFuture<RaftClientReply> sendRequestAsync(
    RaftClientRequest request) {
  final RaftPeerId serverId = request.getServerId();
  try {
    final GrpcClientProtocolClient proxy = getProxies().getProxy(serverId);
    // Reuse the same grpc stream for all async calls.
    return proxy.getOrderedStreamObservers().onNext(request);
  } catch (Throwable e) {
    return JavaUtils.completeExceptionally(e);
  }
}
 

/**
 * Keep generating writing traffic and make sure snapshots are taken.
 * We then restart the whole raft peer and check if it can correctly load
 * snapshots + raft log.
 */
@Test
public void testRestartPeer() throws Exception {
  RaftTestUtil.waitForLeader(cluster);
  final RaftPeerId leaderId = cluster.getLeader().getId();
  int i = 0;
  try(final RaftClient client = cluster.createClient(leaderId)) {
    for (; i < SNAPSHOT_TRIGGER_THRESHOLD * 2 - 1; i++) {
      RaftClientReply reply = client.send(new SimpleMessage("m" + i));
      Assert.assertTrue(reply.isSuccess());
    }
  }

  final long nextIndex = cluster.getLeader().getState().getLog().getNextIndex();
  LOG.info("nextIndex = {}", nextIndex);
  // wait for the snapshot to be done
  final List<File> snapshotFiles = getSnapshotFiles(cluster, nextIndex - SNAPSHOT_TRIGGER_THRESHOLD, nextIndex);
  JavaUtils.attemptRepeatedly(() -> {
    Assert.assertTrue(snapshotFiles.stream().anyMatch(RaftSnapshotBaseTest::exists));
    return null;
  }, 10, ONE_SECOND, "snapshotFile.exist", LOG);

  // restart the peer and check if it can correctly load snapshot
  cluster.restart(false);
  try {
    // 200 messages + two leader elections --> last committed = 201
    assertLeaderContent(cluster);
  } finally {
    cluster.shutdown();
  }
}
 

CompletableFuture<RaftServerImpl> get(RaftGroupId groupId) {
  final CompletableFuture<RaftServerImpl> i = map.get(groupId);
  if (i == null) {
    return JavaUtils.completeExceptionally(new GroupMismatchException(
        getId() + ": " + groupId + " not found."));
  }
  return i;
}
 

@SafeVarargs
public static Throwable testFailureCaseAsync(
    String description, Supplier<CompletableFuture<?>> testCode,
    Class<? extends Throwable> expectedThrowableClass, Logger log,
    Class<? extends Throwable>... expectedCauseClasses) {
  try {
    testCode.get().join();
  } catch (Throwable t) {
    t = JavaUtils.unwrapCompletionException(t);
    assertThrowable(description, t, expectedThrowableClass, log, expectedCauseClasses);
    return t;
  }
  throw new AssertionError("The test \"" + description + "\" does not throw anything.");
}
 

@Test
public void testLateServerStart() throws Exception {
  final int numServer = 3;
  LOG.info("Running testLateServerStart");
  final MiniRaftCluster cluster = newCluster(numServer);
  cluster.initServers();

  // start all except one servers
  final Iterator<RaftServerProxy> i = cluster.getServers().iterator();
  for(int j = 1; j < numServer; j++) {
    i.next().start();
  }

  final RaftServerImpl leader = waitForLeader(cluster);
  final TimeDuration sleepTime = TimeDuration.valueOf(3, TimeUnit.SECONDS);
  LOG.info("sleep " + sleepTime);
  sleepTime.sleep();

  // start the last server
  final RaftServerProxy lastServer = i.next();
  lastServer.start();
  final RaftPeerId lastServerLeaderId = JavaUtils.attemptRepeatedly(
      () -> Optional.ofNullable(lastServer.getImpls().iterator().next().getState().getLeaderId())
          .orElseThrow(() -> new IllegalStateException("No leader yet")),
      10, ONE_SECOND, "getLeaderId", LOG);
  LOG.info(cluster.printServers());
  Assert.assertEquals(leader.getId(), lastServerLeaderId);
}
 

static CompletableFuture<RaftClientReply> failWithException(
    Throwable t, CacheEntry entry) {
  if (entry != null) {
    entry.failWithException(t);
    return entry.getReplyFuture();
  } else {
    return JavaUtils.completeExceptionally(t);
  }
}
 

private void runTestRetryOnStateMachineException(CLUSTER cluster) throws Exception {
  RaftPeerId leaderId = RaftTestUtil.waitForLeader(cluster).getId();

  cluster.getLeaderAndSendFirstMessage(true);
  long oldLastApplied = cluster.getLeader().getState().getLastAppliedIndex();

  try (final RaftClient client = cluster.createClient(leaderId)) {
    final RaftClientRpc rpc = client.getClientRpc();
    final long callId = 999;
    final SimpleMessage message = new SimpleMessage("message");
    final RaftClientRequest r = cluster.newRaftClientRequest(client.getId(), leaderId, callId, message);
    RaftClientReply reply = rpc.sendRequest(r);
    Assert.assertFalse(reply.isSuccess());
    Assert.assertNotNull(reply.getStateMachineException());

    // retry with the same callId
    for (int i = 0; i < 5; i++) {
      reply = rpc.sendRequest(r);
      Assert.assertEquals(client.getId(), reply.getClientId());
      Assert.assertEquals(callId, reply.getCallId());
      Assert.assertFalse(reply.isSuccess());
      Assert.assertNotNull(reply.getStateMachineException());
    }

    for (RaftServerImpl server : cluster.iterateServerImpls()) {
      LOG.info("check server " + server.getId());

      JavaUtils.attemptRepeatedly(() -> {
        Assert.assertNotNull(
            RaftServerTestUtil.getRetryEntry(server, client.getId(), callId));
        return null;
      }, 5, BaseTest.ONE_SECOND, "GetRetryEntry", LOG);

      final RaftLog log = server.getState().getLog();
      RaftTestUtil.logEntriesContains(log, oldLastApplied + 1, log.getNextIndex(), message);
    }

    cluster.shutdown();
  }
}
 

void responseError(Throwable t, Supplier<String> message) {
  if (isClosed.compareAndSet(false, true)) {
    t = JavaUtils.unwrapCompletionException(t);
    if (LOG.isDebugEnabled()) {
      LOG.debug(name + ": Failed " + message.get(), t);
    }
    responseObserver.onError(GrpcUtil.wrapException(t));
    slidingWindow.close();
  }
}
 

static StatusRuntimeException wrapException(Throwable t, long callId) {
  t = JavaUtils.unwrapCompletionException(t);
  Metadata trailers = new StatusRuntimeExceptionMetadataBuilder(t)
      .addCallId(callId)
      .build();
  return wrapException(t, trailers);
}
 

@Override
public CompletableFuture<RaftClientReply> sendRequestAsyncUnordered(RaftClientRequest request) {
  final RaftPeerId serverId = request.getServerId();
  try {
    final GrpcClientProtocolClient proxy = getProxies().getProxy(serverId);
    // Reuse the same grpc stream for all async calls.
    return proxy.getUnorderedAsyncStreamObservers().onNext(request);
  } catch (Throwable t) {
    LOG.error(clientId + ": XXX Failed " + request, t);
    return JavaUtils.completeExceptionally(t);
  }
}
 

public void start() throws IOException {
  LOG.info(".............................................................. ");
  LOG.info("... ");
  LOG.info("...     Starting " + getClass().getSimpleName());
  LOG.info("... ");
  LOG.info(".............................................................. ");

  initServers();
  startServers(servers.values());

  this.timer.updateAndGet(t -> t != null? t
      : JavaUtils.runRepeatedly(() -> LOG.info("TIMED-PRINT: " + printServers()), 10, 10, TimeUnit.SECONDS));
}