org.apache.ratis.protocol.RaftRetryFailureException Java Examples

public static Map<Class<? extends Throwable>,
    RetryPolicy> getRetryPolicyByException(int maxRetryCount,
    long retryInterval) {
  Map<Class<? extends Throwable>, RetryPolicy> policyMap = new HashMap<>();
  for (Class<? extends Exception> ex : EXCEPTION_LIST) {
    if (ex == TimeoutException.class
        || ex == RaftRetryFailureException.class) {
      // retry without sleep
      policyMap.put(ex, createRetryPolicy(maxRetryCount, 0));
    } else {
      // retry with fixed sleep between retries
      policyMap.put(ex, createRetryPolicy(maxRetryCount, retryInterval));
    }
  }
  // Default retry policy
  policyMap
      .put(Exception.class, createRetryPolicy(maxRetryCount, retryInterval));
  return policyMap;
}
 

static void runTestWatchRequestClientTimeout(TestParameters p) throws Exception {
  final Logger LOG = p.log;

  CompletableFuture<RaftClientReply> watchReply;
  // watch 1000 which will never be committed
  // so client can not receive reply, and connection closed, throw TimeoutException.
  // We should not retry, because if retry, RaftClientImpl::handleIOException will random select a leader,
  // then sometimes throw NotLeaderException.
  watchReply = p.sendWatchRequest(1000, RetryPolicies.noRetry());

  try {
    watchReply.get();
    fail("runTestWatchRequestClientTimeout failed");
  } catch (Exception ex) {
    LOG.error("error occurred", ex);
    Assert.assertTrue(ex.getCause().getClass() == AlreadyClosedException.class ||
        ex.getCause().getClass() == RaftRetryFailureException.class);
    if (ex.getCause() != null) {
      if (ex.getCause().getCause() != null) {
        Assert.assertEquals(TimeoutIOException.class,
            ex.getCause().getCause().getClass());
      }
    }
  }
}
 

@Test
public void testExceptionRetryAttempts()
    throws InterruptedException, IOException {
  RaftProperties prop = new RaftProperties();
  RaftClientConfigKeys.Rpc.setRequestTimeout(prop, TimeDuration.valueOf(100, TimeUnit.MILLISECONDS));
  prop.setClass(MiniRaftCluster.STATEMACHINE_CLASS_KEY,
      SimpleStateMachine4Testing.class, StateMachine.class);
  RaftServerConfigKeys.Write.setElementLimit(prop, 1);
  MiniRaftClusterWithGrpc cluster = getFactory().newCluster(1, prop);
  RaftServerImpl leader = null;
  try {
    cluster.start();
    ExceptionDependentRetry.Builder builder =
        ExceptionDependentRetry.newBuilder();
    builder.setExceptionToPolicy(TimeoutIOException.class,
        MultipleLinearRandomRetry.parseCommaSeparated("1ms, 5"));
    builder.setDefaultPolicy(RetryPolicies.retryForeverNoSleep());

    // create a client with the exception dependent policy
    try (final RaftClient client = cluster.createClient(builder.build())) {
      client.sendAsync(new RaftTestUtil.SimpleMessage("1")).get();

      leader = cluster.getLeader();
      ((SimpleStateMachine4Testing) leader.getStateMachine()).blockWriteStateMachineData();

      client.sendAsync(new RaftTestUtil.SimpleMessage("2")).get();
    }
    Assert.fail("Test should have failed.");
  } catch (ExecutionException e) {
    RaftRetryFailureException rrfe = (RaftRetryFailureException) e.getCause();
    Assert.assertEquals(6, rrfe.getAttemptCount());
  } finally {
    ((SimpleStateMachine4Testing)leader.getStateMachine()).unblockWriteStateMachineData();
    cluster.shutdown();
  }
}
 

@Test
public void testDatanodeFailureWithSingleNodeRatis() throws Exception {
  String keyName = getKeyName();
  OzoneOutputStream key =
      createKey(keyName, ReplicationType.RATIS, 0, ReplicationFactor.ONE);
  int dataLength = maxFlushSize + chunkSize;
  // write data more than 1 chunk
  byte[] data1 =
      ContainerTestHelper.getFixedLengthString(keyString, dataLength)
          .getBytes(UTF_8);
  key.write(data1);
  // since its hitting the full bufferCondition, it will call watchForCommit
  // and completes at least putBlock for first flushSize worth of data
  Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
  KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();

  Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
  OutputStream stream =
      keyOutputStream.getStreamEntries().get(0).getOutputStream();
  Assert.assertTrue(stream instanceof BlockOutputStream);
  BlockOutputStream blockOutputStream = (BlockOutputStream) stream;

  // we have just written data more than flush Size(2 chunks), at this time
  // buffer pool will have 3 buffers allocated worth of chunk size

  Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
  // writtenDataLength as well flushedDataLength will be updated here
  Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());

  Assert.assertEquals(maxFlushSize,
      blockOutputStream.getTotalDataFlushedLength());

  // since data equals to maxBufferSize is written, this will be a blocking
  // call and hence will wait for atleast flushSize worth of data to get
  // ack'd by all servers right here
  Assert.assertTrue(blockOutputStream.getTotalAckDataLength() >= flushSize);

  // watchForCommit will clean up atleast flushSize worth of data buffer
  // where each entry corresponds to flushSize worth of data
  Assert.assertTrue(
      blockOutputStream.getCommitIndex2flushedDataMap().size() <= 2);

  // Now do a flush. This will flush the data and update the flush length and
  // the map.
  key.flush();

  // Since the data in the buffer is already flushed, flush here will have
  // no impact on the counters and data structures

  Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
  Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());

  Assert.assertEquals(dataLength,
      blockOutputStream.getTotalDataFlushedLength());
  //  flush will make sure one more entry gets updated in the map
  Assert.assertTrue(
      blockOutputStream.getCommitIndex2flushedDataMap().size() == 0);

  XceiverClientRatis raftClient =
      (XceiverClientRatis) blockOutputStream.getXceiverClient();
  Assert.assertEquals(1, raftClient.getCommitInfoMap().size());
  Pipeline pipeline = raftClient.getPipeline();
  cluster.shutdownHddsDatanode(pipeline.getNodes().get(0));

  // again write data with more than max buffer limit. This will call
  // watchForCommit again. No write will happen in the current block and
  // data will be rewritten to the next block.

  key.write(data1);

  key.flush();

  Assert.assertTrue(HddsClientUtils.checkForException(blockOutputStream
      .getIoException()) instanceof RaftRetryFailureException);
  Assert.assertEquals(1, raftClient.getCommitInfoMap().size());
  // Make sure the retryCount is reset after the exception is handled
  Assert.assertTrue(keyOutputStream.getRetryCount() == 0);
  Assert.assertEquals(2, keyOutputStream.getStreamEntries().size());
  // now close the stream, It will update the ack length after watchForCommit
  key.close();
  Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
  // make sure the bufferPool is empty
  Assert
      .assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
  Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
  Assert.assertEquals(0, keyOutputStream.getStreamEntries().size());
  Assert.assertEquals(0, keyOutputStream.getLocationInfoList().size());
  // Written the same data twice
  String dataString = new String(data1, UTF_8);
  cluster.restartHddsDatanode(pipeline.getNodes().get(0), true);
  validateData(keyName, dataString.concat(dataString).getBytes());
}
 

@Test
public void testDatanodeFailureWithPreAllocation() throws Exception {
  String keyName = getKeyName();
  OzoneOutputStream key =
      createKey(keyName, ReplicationType.RATIS, 3 * blockSize,
          ReplicationFactor.ONE);
  int dataLength = maxFlushSize + chunkSize;
  // write data more than 1 chunk
  byte[] data1 =
      ContainerTestHelper.getFixedLengthString(keyString, dataLength)
          .getBytes(UTF_8);
  key.write(data1);
  // since its hitting the full bufferCondition, it will call watchForCommit
  // and completes at least putBlock for first flushSize worth of data
  Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
  KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();

  Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 3);
  OutputStream stream =
      keyOutputStream.getStreamEntries().get(0).getOutputStream();
  Assert.assertTrue(stream instanceof BlockOutputStream);
  BlockOutputStream blockOutputStream = (BlockOutputStream) stream;

  // we have just written data more than flush Size(2 chunks), at this time
  // buffer pool will have 3 buffers allocated worth of chunk size

  Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
  // writtenDataLength as well flushedDataLength will be updated here
  Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());

  Assert.assertEquals(maxFlushSize,
      blockOutputStream.getTotalDataFlushedLength());

  // since data equals to maxBufferSize is written, this will be a blocking
  // call and hence will wait for atleast flushSize worth of data to get
  // ack'd by all servers right here
  Assert.assertTrue(blockOutputStream.getTotalAckDataLength() >= flushSize);

  // watchForCommit will clean up atleast flushSize worth of data buffer
  // where each entry corresponds to flushSize worth of data
  Assert.assertTrue(
      blockOutputStream.getCommitIndex2flushedDataMap().size() <= 2);

  // Now do a flush. This will flush the data and update the flush length and
  // the map.
  key.flush();

  // Since the data in the buffer is already flushed, flush here will have
  // no impact on the counters and data structures

  Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
  Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());

  Assert.assertEquals(dataLength,
      blockOutputStream.getTotalDataFlushedLength());
  //  flush will make sure one more entry gets updated in the map
  Assert.assertTrue(
      blockOutputStream.getCommitIndex2flushedDataMap().size() == 0);

  XceiverClientRatis raftClient =
      (XceiverClientRatis) blockOutputStream.getXceiverClient();
  Assert.assertEquals(1, raftClient.getCommitInfoMap().size());
  Pipeline pipeline = raftClient.getPipeline();
  cluster.shutdownHddsDatanode(pipeline.getNodes().get(0));

  // again write data with more than max buffer limit. This will call
  // watchForCommit again. No write will happen and

  key.write(data1);

  key.flush();

  Assert.assertTrue(HddsClientUtils.checkForException(blockOutputStream
      .getIoException()) instanceof RaftRetryFailureException);

  // Make sure the retryCount is reset after the exception is handled
  Assert.assertTrue(keyOutputStream.getRetryCount() == 0);
  Assert.assertEquals(1, raftClient.getCommitInfoMap().size());

  // now close the stream, It will update the ack length after watchForCommit
  key.close();
  Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
  // make sure the bufferPool is empty
  Assert
      .assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
  Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
  Assert.assertEquals(0, keyOutputStream.getLocationInfoList().size());
  // Written the same data twice
  String dataString = new String(data1, UTF_8);
  cluster.restartHddsDatanode(pipeline.getNodes().get(0), true);
  validateData(keyName, dataString.concat(dataString).getBytes());
}
 

@Test
public void testReleaseBuffersOnException() throws Exception {
  int capacity = 2;
  BufferPool bufferPool = new BufferPool(chunkSize, capacity);
  XceiverClientManager clientManager = new XceiverClientManager(conf);
  ContainerWithPipeline container = storageContainerLocationClient
      .allocateContainer(HddsProtos.ReplicationType.RATIS,
          HddsProtos.ReplicationFactor.THREE, OzoneConsts.OZONE);
  Pipeline pipeline = container.getPipeline();
  long containerId = container.getContainerInfo().getContainerID();
  XceiverClientSpi xceiverClient = clientManager.acquireClient(pipeline);
  Assert.assertEquals(1, xceiverClient.getRefcount());
  Assert.assertTrue(xceiverClient instanceof XceiverClientRatis);
  XceiverClientRatis ratisClient = (XceiverClientRatis) xceiverClient;
  CommitWatcher watcher = new CommitWatcher(bufferPool, ratisClient);
  BlockID blockID = ContainerTestHelper.getTestBlockID(containerId);
  List<XceiverClientReply> replies = new ArrayList<>();
  long length = 0;
  List<CompletableFuture<ContainerProtos.ContainerCommandResponseProto>>
      futures = new ArrayList<>();
  for (int i = 0; i < capacity; i++) {
    ContainerProtos.ContainerCommandRequestProto writeChunkRequest =
        ContainerTestHelper
            .getWriteChunkRequest(pipeline, blockID, chunkSize, null);
    // add the data to the buffer pool
    final ChunkBuffer byteBuffer = bufferPool.allocateBufferIfNeeded(0);
    byteBuffer.put(writeChunkRequest.getWriteChunk().getData());
    ratisClient.sendCommandAsync(writeChunkRequest);
    ContainerProtos.ContainerCommandRequestProto putBlockRequest =
        ContainerTestHelper
            .getPutBlockRequest(pipeline, writeChunkRequest.getWriteChunk());
    XceiverClientReply reply = ratisClient.sendCommandAsync(putBlockRequest);
    final List<ChunkBuffer> bufferList = singletonList(byteBuffer);
    length += byteBuffer.position();
    CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future =
        reply.getResponse().thenApply(v -> {
          watcher.updateCommitInfoMap(reply.getLogIndex(), bufferList);
          return v;
        });
    futures.add(future);
    watcher.getFutureMap().put(length, future);
    replies.add(reply);
  }

  Assert.assertTrue(replies.size() == 2);
  // wait on the 1st putBlock to complete
  CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future1 =
      futures.get(0);
  CompletableFuture<ContainerProtos.ContainerCommandResponseProto> future2 =
      futures.get(1);
  future1.get();
  Assert.assertNotNull(watcher.getFutureMap().get(new Long(chunkSize)));
  Assert.assertTrue(
      watcher.getFutureMap().get(new Long(chunkSize)).equals(future1));
  // wait on 2nd putBlock to complete
  future2.get();
  Assert.assertNotNull(watcher.getFutureMap().get(new Long(2 * chunkSize)));
  Assert.assertTrue(
      watcher.getFutureMap().get(new Long(2 * chunkSize)).equals(future2));
  Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().size() == 2);
  watcher.watchOnFirstIndex();
  Assert.assertFalse(watcher.getCommitIndex2flushedDataMap()
      .containsKey(replies.get(0).getLogIndex()));
  Assert.assertFalse(watcher.getFutureMap().containsKey(chunkSize));
  Assert.assertTrue(watcher.getTotalAckDataLength() >= chunkSize);
  cluster.shutdownHddsDatanode(pipeline.getNodes().get(0));
  cluster.shutdownHddsDatanode(pipeline.getNodes().get(1));
  try {
    // just watch for a higher index so as to ensure, it does an actual
    // call to Ratis. Otherwise, it may just return in case the commitInfoMap
    // is updated to the latest index in putBlock response.
    watcher.watchForCommit(replies.get(1).getLogIndex() + 100);
    Assert.fail("Expected exception not thrown");
  } catch(IOException ioe) {
    // with retry count set to lower limit and a lower watch request
    // timeout, watch request will eventually
    // fail with RaftRetryFailure exception from ratis client or the client
    // can itself get AlreadyClosedException from the Ratis Server
    Throwable t = HddsClientUtils.checkForException(ioe);
    Assert.assertTrue(t instanceof RaftRetryFailureException ||
            t instanceof AlreadyClosedException ||
            t instanceof NotReplicatedException);
  }
  if (ratisClient.getReplicatedMinCommitIndex() < replies.get(1)
      .getLogIndex()) {
    Assert.assertTrue(watcher.getTotalAckDataLength() == chunkSize);
    Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().size() == 1);
    Assert.assertTrue(watcher.getFutureMap().size() == 1);
  } else {
    Assert.assertTrue(watcher.getTotalAckDataLength() == 2 * chunkSize);
    Assert.assertTrue(watcher.getFutureMap().isEmpty());
    Assert.assertTrue(watcher.getCommitIndex2flushedDataMap().isEmpty());
  }
}
 

@Test
public void testDatanodeFailureWithSingleNodeRatis() throws Exception {
  String keyName = getKeyName();
  OzoneOutputStream key =
      createKey(keyName, ReplicationType.RATIS, 0, ReplicationFactor.ONE);
  int dataLength = maxFlushSize + 50;
  // write data more than 1 chunk
  byte[] data1 =
      ContainerTestHelper.getFixedLengthString(keyString, dataLength)
          .getBytes(UTF_8);
  key.write(data1);
  // since its hitting the full bufferCondition, it will call watchForCommit
  // and completes at least putBlock for first flushSize worth of data
  Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
  KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();

  Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 1);
  OutputStream stream =
      keyOutputStream.getStreamEntries().get(0).getOutputStream();
  Assert.assertTrue(stream instanceof BlockOutputStream);
  BlockOutputStream blockOutputStream = (BlockOutputStream) stream;

  // we have just written data more than flush Size(2 chunks), at this time
  // buffer pool will have 3 buffers allocated worth of chunk size

  Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
  // writtenDataLength as well flushedDataLength will be updated here
  Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());

  Assert.assertEquals(maxFlushSize,
      blockOutputStream.getTotalDataFlushedLength());

  // since data equals to maxBufferSize is written, this will be a blocking
  // call and hence will wait for atleast flushSize worth of data to get
  // ack'd by all servers right here
  Assert.assertTrue(blockOutputStream.getTotalAckDataLength() >= flushSize);

  // watchForCommit will clean up atleast flushSize worth of data buffer
  // where each entry corresponds to flushSize worth of data
  Assert.assertTrue(
      blockOutputStream.getCommitIndex2flushedDataMap().size() <= 2);

  // Now do a flush. This will flush the data and update the flush length and
  // the map.
  key.flush();

  // Since the data in the buffer is already flushed, flush here will have
  // no impact on the counters and data structures

  Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
  Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());

  Assert.assertEquals(dataLength,
      blockOutputStream.getTotalDataFlushedLength());
  //  flush will make sure one more entry gets updated in the map
  Assert.assertTrue(
      blockOutputStream.getCommitIndex2flushedDataMap().size() == 0);

  XceiverClientRatis raftClient =
      (XceiverClientRatis) blockOutputStream.getXceiverClient();
  Assert.assertEquals(1, raftClient.getCommitInfoMap().size());
  Pipeline pipeline = raftClient.getPipeline();
  cluster.shutdownHddsDatanode(pipeline.getNodes().get(0));

  // again write data with more than max buffer limit. This will call
  // watchForCommit again. No write will happen in the current block and
  // data will be rewritten to the next block.

  key.write(data1);

  key.flush();

  Assert.assertTrue(HddsClientUtils.checkForException(blockOutputStream
      .getIoException()) instanceof RaftRetryFailureException);
  Assert.assertEquals(1, raftClient.getCommitInfoMap().size());
  // Make sure the retryCount is reset after the exception is handled
  Assert.assertTrue(keyOutputStream.getRetryCount() == 0);
  Assert.assertEquals(2, keyOutputStream.getStreamEntries().size());
  // now close the stream, It will update the ack length after watchForCommit
  key.close();
  Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
  // make sure the bufferPool is empty
  Assert
      .assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
  Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
  Assert.assertEquals(0, keyOutputStream.getStreamEntries().size());
  Assert.assertEquals(0, keyOutputStream.getLocationInfoList().size());
  // Written the same data twice
  String dataString = new String(data1, UTF_8);
  cluster.restartHddsDatanode(pipeline.getNodes().get(0), true);
  validateData(keyName, dataString.concat(dataString).getBytes());
}
 

@Test
public void testDatanodeFailureWithPreAllocation() throws Exception {
  String keyName = getKeyName();
  OzoneOutputStream key =
      createKey(keyName, ReplicationType.RATIS, 3 * blockSize,
          ReplicationFactor.ONE);
  int dataLength = maxFlushSize + 50;
  // write data more than 1 chunk
  byte[] data1 =
      ContainerTestHelper.getFixedLengthString(keyString, dataLength)
          .getBytes(UTF_8);
  key.write(data1);
  // since its hitting the full bufferCondition, it will call watchForCommit
  // and completes at least putBlock for first flushSize worth of data
  Assert.assertTrue(key.getOutputStream() instanceof KeyOutputStream);
  KeyOutputStream keyOutputStream = (KeyOutputStream) key.getOutputStream();

  Assert.assertTrue(keyOutputStream.getStreamEntries().size() == 3);
  OutputStream stream =
      keyOutputStream.getStreamEntries().get(0).getOutputStream();
  Assert.assertTrue(stream instanceof BlockOutputStream);
  BlockOutputStream blockOutputStream = (BlockOutputStream) stream;

  // we have just written data more than flush Size(2 chunks), at this time
  // buffer pool will have 3 buffers allocated worth of chunk size

  Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
  // writtenDataLength as well flushedDataLength will be updated here
  Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());

  Assert.assertEquals(maxFlushSize,
      blockOutputStream.getTotalDataFlushedLength());

  // since data equals to maxBufferSize is written, this will be a blocking
  // call and hence will wait for atleast flushSize worth of data to get
  // ack'd by all servers right here
  Assert.assertTrue(blockOutputStream.getTotalAckDataLength() >= flushSize);

  // watchForCommit will clean up atleast flushSize worth of data buffer
  // where each entry corresponds to flushSize worth of data
  Assert.assertTrue(
      blockOutputStream.getCommitIndex2flushedDataMap().size() <= 2);

  // Now do a flush. This will flush the data and update the flush length and
  // the map.
  key.flush();

  // Since the data in the buffer is already flushed, flush here will have
  // no impact on the counters and data structures

  Assert.assertEquals(4, blockOutputStream.getBufferPool().getSize());
  Assert.assertEquals(dataLength, blockOutputStream.getWrittenDataLength());

  Assert.assertEquals(dataLength,
      blockOutputStream.getTotalDataFlushedLength());
  //  flush will make sure one more entry gets updated in the map
  Assert.assertTrue(
      blockOutputStream.getCommitIndex2flushedDataMap().size() == 0);

  XceiverClientRatis raftClient =
      (XceiverClientRatis) blockOutputStream.getXceiverClient();
  Assert.assertEquals(1, raftClient.getCommitInfoMap().size());
  Pipeline pipeline = raftClient.getPipeline();
  cluster.shutdownHddsDatanode(pipeline.getNodes().get(0));

  // again write data with more than max buffer limit. This will call
  // watchForCommit again. No write will happen and

  key.write(data1);

  key.flush();

  Assert.assertTrue(HddsClientUtils.checkForException(blockOutputStream
      .getIoException()) instanceof RaftRetryFailureException);

  // Make sure the retryCount is reset after the exception is handled
  Assert.assertTrue(keyOutputStream.getRetryCount() == 0);
  Assert.assertEquals(1, raftClient.getCommitInfoMap().size());

  // now close the stream, It will update the ack length after watchForCommit
  key.close();
  Assert.assertEquals(dataLength, blockOutputStream.getTotalAckDataLength());
  // make sure the bufferPool is empty
  Assert
      .assertEquals(0, blockOutputStream.getBufferPool().computeBufferData());
  Assert.assertNull(blockOutputStream.getCommitIndex2flushedDataMap());
  Assert.assertEquals(0, keyOutputStream.getLocationInfoList().size());
  // Written the same data twice
  String dataString = new String(data1, UTF_8);
  cluster.restartHddsDatanode(pipeline.getNodes().get(0), true);
  validateData(keyName, dataString.concat(dataString).getBytes());
}
 

static void assertRaftRetryFailureException(RaftRetryFailureException rfe, RetryPolicy retryPolicy, String name) {
  Assert.assertNotNull(name + " does not have RaftRetryFailureException", rfe);
  Assert.assertTrue(name + ": unexpected error message, rfe=" + rfe + ", retryPolicy=" + retryPolicy,
      rfe.getMessage().contains(retryPolicy.toString()));
}
 

static void assertRaftRetryFailureException(RaftRetryFailureException rfe, RetryPolicy retryPolicy, String name) {
  Assert.assertNotNull(name + " does not have RaftRetryFailureException", rfe);
  Assert.assertTrue(name + ": unexpected error message, rfe=" + rfe + ", retryPolicy=" + retryPolicy,
      rfe.getMessage().contains(retryPolicy.toString()));
}
 

/**
 * Checks if the provided exception signifies retry failure in ratis client.
 * In case of retry failure, ratis client throws RaftRetryFailureException
 * and all succeeding operations are failed with AlreadyClosedException.
 */
private boolean checkForRetryFailure(Throwable t) {
  return t instanceof RaftRetryFailureException
      || t instanceof AlreadyClosedException;
}