org.apache.ratis.retry.RetryPolicy Java Examples

/**
 * Sends ratis command to destroy pipeline on the given datanode.
 *
 * @param dn         - Datanode on which pipeline needs to be destroyed
 * @param pipelineID - ID of pipeline to be destroyed
 * @param ozoneConf  - Ozone configuration
 * @param grpcTlsConfig - grpc tls configuration
 * @throws IOException
 */
static void destroyPipeline(DatanodeDetails dn, PipelineID pipelineID,
    ConfigurationSource ozoneConf, GrpcTlsConfig grpcTlsConfig)
    throws IOException {
  final String rpcType = ozoneConf
      .get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
          ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
  final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf);
  final RaftPeer p = RatisHelper.toRaftPeer(dn);
  try(RaftClient client = RatisHelper
      .newRaftClient(SupportedRpcType.valueOfIgnoreCase(rpcType), p,
          retryPolicy, grpcTlsConfig, ozoneConf)) {
    client.groupRemove(RaftGroupId.valueOf(pipelineID.getId()),
        true, p.getId());
  }
}
 

RaftClientImpl(ClientId clientId, RaftGroup group, RaftPeerId leaderId,
    RaftClientRpc clientRpc, RaftProperties properties, RetryPolicy retryPolicy) {
  this.clientId = clientId;
  this.clientRpc = clientRpc;
  this.peers = new ConcurrentLinkedQueue<>(group.getPeers());
  this.groupId = group.getGroupId();
  this.leaderId = leaderId != null? leaderId
      : !peers.isEmpty()? peers.iterator().next().getId(): null;
  Preconditions.assertTrue(retryPolicy != null, "retry policy can't be null");
  this.retryPolicy = retryPolicy;

  scheduler = TimeoutScheduler.getInstance();
  clientRpc.addServers(peers);

  this.orderedAsync = JavaUtils.memoize(() -> OrderedAsync.newInstance(this, properties));
  this.streamApi = JavaUtils.memoize(() -> StreamImpl.newInstance(this, properties));
}
 

/**
 * Table mapping exception type to retry policy used for the exception in
 * write and watch request.
 * ---------------------------------------------------------------------------
 * |        Exception            | RetryPolicy for     | RetryPolicy for     |
 * |                             | Write request       | Watch request       |
 * |-------------------------------------------------------------------------|
 * | NotReplicatedException      | NO_RETRY            | NO_RETRY            |
 * |-------------------------------------------------------------------------|
 * | GroupMismatchException      | NO_RETRY            | NO_RETRY            |
 * |-------------------------------------------------------------------------|
 * | StateMachineException       | NO_RETRY            | NO_RETRY            |
 * |-------------------------------------------------------------------------|
 * | TimeoutIOException          | EXPONENTIAL_BACKOFF | NO_RETRY            |
 * |-------------------------------------------------------------------------|
 * | ResourceUnavailableException| EXPONENTIAL_BACKOFF | EXPONENTIAL_BACKOFF |
 * |-------------------------------------------------------------------------|
 * | Others                      | MULTILINEAR_RANDOM  | MULTILINEAR_RANDOM  |
 * |                             | _RETRY             | _RETRY               |
 * ---------------------------------------------------------------------------
 */
public static RetryPolicy createRetryPolicy(ConfigurationSource conf) {
  RatisClientConfig ratisClientConfig = OzoneConfiguration.of(conf)
      .getObject(RatisClientConfig.class);
  ExponentialBackoffRetry exponentialBackoffRetry =
      createExponentialBackoffPolicy(ratisClientConfig);
  MultipleLinearRandomRetry multipleLinearRandomRetry =
      MultipleLinearRandomRetry
          .parseCommaSeparated(ratisClientConfig.getMultilinearPolicy());

  long writeTimeout = ratisClientConfig.getWriteRequestTimeoutInMs();
  long watchTimeout = ratisClientConfig.getWatchRequestTimeoutInMs();

  return RequestTypeDependentRetryPolicy.newBuilder()
      .setRetryPolicy(RaftProtos.RaftClientRequestProto.TypeCase.WRITE,
          createExceptionDependentPolicy(exponentialBackoffRetry,
              multipleLinearRandomRetry, exponentialBackoffRetry))
      .setRetryPolicy(RaftProtos.RaftClientRequestProto.TypeCase.WATCH,
          createExceptionDependentPolicy(exponentialBackoffRetry,
              multipleLinearRandomRetry, RetryPolicies.noRetry()))
      .setTimeout(RaftProtos.RaftClientRequestProto.TypeCase.WRITE,
          TimeDuration.valueOf(writeTimeout, TimeUnit.MILLISECONDS))
      .setTimeout(RaftProtos.RaftClientRequestProto.TypeCase.WATCH,
          TimeDuration.valueOf(watchTimeout, TimeUnit.MILLISECONDS))
      .build();
}
 

public RaftClient createClient(RaftPeerId leaderId, RaftGroup group, RetryPolicy retryPolicy) {
  RaftClient.Builder builder = RaftClient.newBuilder()
      .setRaftGroup(group)
      .setLeaderId(leaderId)
      .setProperties(properties)
      .setParameters(parameters)
      .setRetryPolicy(retryPolicy);
  return builder.build();
}
 

CompletableFuture<RaftClientReply> sendWatchRequest(long logIndex, RetryPolicy policy)
    throws Exception {

  try (final RaftClient watchClient =
      cluster.createClient(RaftTestUtil.waitForLeader(cluster).getId(), policy)) {

    CompletableFuture<RaftClientReply> reply =
        watchClient.sendAsync(new RaftTestUtil.SimpleMessage("message"));
    long writeIndex = reply.get().getLogIndex();
    Assert.assertTrue(writeIndex > 0);
    watchClient.sendWatchAsync(writeIndex, ReplicationLevel.MAJORITY_COMMITTED);
    return watchClient.sendWatchAsync(logIndex, ReplicationLevel.MAJORITY);
  }

}
 

@Test
public void testRetryMultipleTimesWithFixedSleep() {
  RetryPolicy retryPolicy = RetryPolicies
      .retryUpToMaximumCountWithFixedSleep(2,
          TimeDuration.valueOf(1000L, TimeUnit.MILLISECONDS));
   boolean shouldRetry = retryPolicy.shouldRetry(1);
  Assert.assertTrue(shouldRetry);
  Assert.assertTrue(1000 == retryPolicy.getSleepTime().getDuration());
  Assert.assertFalse(retryPolicy.shouldRetry(3));
}
 

private RequestTypeDependentRetryPolicy(
    EnumMap<RaftProtos.RaftClientRequestProto.TypeCase, RetryPolicy> map,
    EnumMap<RaftProtos.RaftClientRequestProto.TypeCase, TimeDuration> timeoutMap) {
  this.retryPolicyMap = Collections.unmodifiableMap(map);
  this.timeoutMap = timeoutMap;
  this.myString = () -> {
    final StringBuilder b = new StringBuilder(getClass().getSimpleName()).append("{");
    map.forEach((key, value) -> b.append(key).append("->").append(value).append(", "));
    b.setLength(b.length() - 2);
    return b.append("}").toString();
  };
}
 

RaftClientImpl(ClientId clientId, RaftGroup group, RaftPeerId leaderId,
    RaftClientRpc clientRpc, RaftProperties properties, RetryPolicy retryPolicy) {
  this.clientId = clientId;
  this.clientRpc = clientRpc;
  this.peers = new ConcurrentLinkedQueue<>(group.getPeers());
  this.groupId = group.getGroupId();
  this.leaderId = leaderId != null? leaderId
      : !peers.isEmpty()? peers.iterator().next().getId(): null;
  Preconditions.assertTrue(retryPolicy != null, "retry policy can't be null");
  this.retryPolicy = retryPolicy;

  asyncRequestSemaphore = new Semaphore(RaftClientConfigKeys.Async.maxOutstandingRequests(properties));
  scheduler = TimeoutScheduler.newInstance(RaftClientConfigKeys.Async.schedulerThreads(properties));
  clientRpc.addServers(peers);
}
 

public RaftClient createClient(RaftPeerId leaderId, RaftGroup group,
    ClientId clientId, RetryPolicy retryPolicy) {
  RaftClient.Builder builder = RaftClient.newBuilder()
      .setClientId(clientId)
      .setRaftGroup(group)
      .setLeaderId(leaderId)
      .setProperties(properties)
      .setParameters(parameters)
      .setRetryPolicy(retryPolicy);
  return builder.build();
}
 

private void scheduleWithTimeout(PendingOrderedRequest pending,
    RaftClientRequest request, RetryPolicy retryPolicy, Throwable e) {
  final int attempt = pending.getAttemptCount();
  final ClientRetryEvent event = new ClientRetryEvent(request, e, pending);
  final TimeDuration sleepTime = client.getEffectiveSleepTime(e,
      retryPolicy.handleAttemptFailure(event).getSleepTime());
  LOG.debug("schedule* attempt #{} with sleep {} and policy {} for {}", attempt, sleepTime, retryPolicy, request);
  scheduleWithTimeout(pending, sleepTime, getSlidingWindow(request));
}
 

private void sendRequestWithRetry(PendingOrderedRequest pending) {
  final CompletableFuture<RaftClientReply> f = pending.getReplyFuture();
  if (f.isDone()) {
    return;
  }

  final RaftClientRequest request = pending.newRequestImpl();
  if (request == null) { // already done
    LOG.debug("{} newRequestImpl returns null", pending);
    return;
  }

  final RetryPolicy retryPolicy = client.getRetryPolicy();
  sendRequest(pending).thenAccept(reply -> {
    if (f.isDone()) {
      return;
    }
    if (reply == null) {
      scheduleWithTimeout(pending, request, retryPolicy, null);
    } else {
      f.complete(reply);
    }
  }).exceptionally(e -> {
    if (e instanceof CompletionException) {
      e = JavaUtils.unwrapCompletionException(e);
      scheduleWithTimeout(pending, request, retryPolicy, e);
      return null;
    }
    f.completeExceptionally(e);
    return null;
  });
}
 

private RaftClient.Builder mockRaftClientBuilder() {
  final RaftClient.Builder builder = Mockito.mock(RaftClient.Builder.class);
  Mockito.when(builder.setClientId(Mockito.any(ClientId.class)))
      .thenReturn(builder);
  Mockito.when(builder.setRaftGroup(Mockito.any(RaftGroup.class)))
      .thenReturn(builder);
  Mockito.when(builder.setLeaderId(Mockito.any(RaftPeerId.class)))
      .thenReturn(builder);
  Mockito.when(builder.setProperties(Mockito.any(RaftProperties.class)))
      .thenReturn(builder);
  Mockito.when(builder.setRetryPolicy(Mockito.any(RetryPolicy.class)))
      .thenReturn(builder);
  return builder;
}
 

private static ExceptionDependentRetry createExceptionDependentPolicy(
    ExponentialBackoffRetry exponentialBackoffRetry,
    MultipleLinearRandomRetry multipleLinearRandomRetry,
    RetryPolicy timeoutPolicy) {
  ExceptionDependentRetry.Builder builder =
      ExceptionDependentRetry.newBuilder();
  for (Class c : NO_RETRY_EXCEPTIONS) {
    builder.setExceptionToPolicy(c, RetryPolicies.noRetry());
  }
  return builder.setExceptionToPolicy(ResourceUnavailableException.class,
          exponentialBackoffRetry)
      .setExceptionToPolicy(TimeoutIOException.class, timeoutPolicy)
      .setDefaultPolicy(multipleLinearRandomRetry)
      .build();
}
 

@SuppressWarnings("checkstyle:ParameterNumber")
private static RaftClient newRaftClient(RpcType rpcType, RaftPeerId leader,
    RaftGroup group, RetryPolicy retryPolicy,
    GrpcTlsConfig tlsConfig, ConfigurationSource ozoneConfiguration) {
  if (LOG.isTraceEnabled()) {
    LOG.trace("newRaftClient: {}, leader={}, group={}",
        rpcType, leader, group);
  }
  final RaftProperties properties = new RaftProperties();

  RaftConfigKeys.Rpc.setType(properties, rpcType);

  // Set the ratis client headers which are matching with regex.
  createRaftClientProperties(ozoneConfiguration, properties);

  RaftClient.Builder builder =  RaftClient.newBuilder()
      .setRaftGroup(group)
      .setLeaderId(leader)
      .setProperties(properties)
      .setRetryPolicy(retryPolicy);

  // TODO: GRPC TLS only for now, netty/hadoop RPC TLS support later.
  if (tlsConfig != null && rpcType == SupportedRpcType.GRPC) {
    builder.setParameters(GrpcFactory.newRaftParameters(tlsConfig));
  }
  return builder.build();
}
 

public static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader,
    RetryPolicy retryPolicy,
    ConfigurationSource ozoneConfiguration) {
  return newRaftClient(rpcType, leader.getId(),
      newRaftGroup(Collections.singletonList(leader)), retryPolicy, null,
      ozoneConfiguration);
}
 

public static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader,
    RetryPolicy retryPolicy, GrpcTlsConfig tlsConfig,
    ConfigurationSource configuration) {
  return newRaftClient(rpcType, leader.getId(),
      newRaftGroup(Collections.singletonList(leader)), retryPolicy,
      tlsConfig, configuration);
}
 

public static RaftClient newRaftClient(RpcType rpcType, Pipeline pipeline,
    RetryPolicy retryPolicy, GrpcTlsConfig tlsConfig,
    ConfigurationSource ozoneConfiguration) throws IOException {
  return newRaftClient(rpcType,
      toRaftPeerId(pipeline.getLeaderNode()),
      newRaftGroup(RaftGroupId.valueOf(pipeline.getId().getId()),
          pipeline.getNodes()), retryPolicy, tlsConfig, ozoneConfiguration);
}
 

/**
 * Constructs a client.
 */
private XceiverClientRatis(Pipeline pipeline, RpcType rpcType,
    RetryPolicy retryPolicy, GrpcTlsConfig tlsConfig,
    ConfigurationSource configuration) {
  super();
  this.pipeline = pipeline;
  this.rpcType = rpcType;
  this.retryPolicy = retryPolicy;
  commitInfoMap = new ConcurrentHashMap<>();
  this.tlsConfig = tlsConfig;
  metrics = XceiverClientManager.getXceiverClientMetrics();
  this.ozoneConfiguration = configuration;
}
 

public static XceiverClientRatis newXceiverClientRatis(
    org.apache.hadoop.hdds.scm.pipeline.Pipeline pipeline,
    ConfigurationSource ozoneConf, X509Certificate caCert) {
  final String rpcType = ozoneConf
      .get(ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
          ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
  final RetryPolicy retryPolicy = RatisHelper.createRetryPolicy(ozoneConf);
  final GrpcTlsConfig tlsConfig = RatisHelper.createTlsClientConfig(new
      SecurityConfig(ozoneConf), caCert);
  return new XceiverClientRatis(pipeline,
      SupportedRpcType.valueOfIgnoreCase(rpcType),
      retryPolicy, tlsConfig, ozoneConf);
}
 

public RaftClient createClient(RaftPeerId leaderId, RetryPolicy retryPolicy) {
  return createClient(leaderId, group, null, retryPolicy);
}
 

private RetryPolicy getDefaultRetryPolicy() {
  return RetryPolicies.retryForeverWithSleep(RETRY_INTERVAL_DEFAULT);
}
 

/** Set {@link RetryPolicy}. */
public Builder setRetryPolicy(RetryPolicy retryPolicy) {
  this.retryPolicy = retryPolicy;
  return this;
}
 

public static RaftClient newRaftClient(ClientId clientId, RaftGroup group,
    RaftPeerId leaderId, RaftClientRpc clientRpc, RaftProperties properties,
    RetryPolicy retryPolicy) {
  return new RaftClientImpl(clientId, group, leaderId, clientRpc, properties,
      retryPolicy);
}
 

static void assertRaftRetryFailureException(RaftRetryFailureException rfe, RetryPolicy retryPolicy, String name) {
  Assert.assertNotNull(name + " does not have RaftRetryFailureException", rfe);
  Assert.assertTrue(name + ": unexpected error message, rfe=" + rfe + ", retryPolicy=" + retryPolicy,
      rfe.getMessage().contains(retryPolicy.toString()));
}
 

static void assertRaftRetryFailureException(RaftRetryFailureException rfe, RetryPolicy retryPolicy, String name) {
  Assert.assertNotNull(name + " does not have RaftRetryFailureException", rfe);
  Assert.assertTrue(name + ": unexpected error message, rfe=" + rfe + ", retryPolicy=" + retryPolicy,
      rfe.getMessage().contains(retryPolicy.toString()));
}
 

public RaftClient createClient(RaftPeerId leaderId, RetryPolicy retryPolicy) {
  return createClient(leaderId, group, retryPolicy);
}
 

public RaftClient createClient(RetryPolicy retryPolicy) {
  return createClient(null, group, retryPolicy);
}
 

private RetryPolicy getDefaultRetryPolicy() {
  return RetryPolicies.retryForeverWithSleep(RETRY_INTERVAL_DEFAULT);
}
 

/** Set the given policy for the given type. */
public Builder setRetryPolicy(RaftProtos.RaftClientRequestProto.TypeCase type, RetryPolicy policy) {
  final RetryPolicy previous = retryPolicyMap.put(type, policy);
  Preconditions.assertNull(previous, () -> "The retryPolicy for type " + type + " is already set to " + previous);
  return this;
}
 

/** Set {@link RetryPolicy}. */
public Builder setRetryPolicy(RetryPolicy retryPolicy) {
  this.retryPolicy = retryPolicy;
  return this;
}