Java Code Examples for org.apache.hadoop.io.retry.RetryPolicies#retryByException()

The following examples show how to use org.apache.hadoop.io.retry.RetryPolicies#retryByException() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: AvatarZKShell.java From RDFS with Apache License 2.0

6 votes

private static AvatarProtocol createAvatarnode(AvatarProtocol rpcAvatarnode)
    throws IOException {
  RetryPolicy createPolicy = RetryPolicies
      .retryUpToMaximumCountWithFixedSleep(5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class, RetryPolicies
      .retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL,
          remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (AvatarProtocol) RetryProxy.create(AvatarProtocol.class,
      rpcAvatarnode, methodNameToPolicyMap);
}

Example 2

Source File: DFSClient.java From hadoop-gpu with Apache License 2.0

6 votes

private static ClientProtocol createNamenode(ClientProtocol rpcNamenode)
  throws IOException {
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      5, LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
  
  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy);

  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class, 
      RetryPolicies.retryByRemoteException(
          RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  
  methodNameToPolicyMap.put("create", methodPolicy);

  return (ClientProtocol) RetryProxy.create(ClientProtocol.class,
      rpcNamenode, methodNameToPolicyMap);
}

Example 3

Source File: S3FileSystem.java From hadoop-gpu with Apache License 2.0

6 votes

private static FileSystemStore createDefaultStore(Configuration conf) {
  FileSystemStore store = new Jets3tFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                             conf.getInt("fs.s3.maxRetries", 4),
                                                                             conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                            RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  methodNameToPolicyMap.put("storeBlock", methodPolicy);
  methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
  
  return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                             store, methodNameToPolicyMap);
}

Example 4

Source File: NativeS3FileSystem.java From hadoop-gpu with Apache License 2.0

6 votes

private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}

Example 5

Source File: HighTideShell.java From RDFS with Apache License 2.0

6 votes

private static HighTideProtocol createHighTidenode(HighTideProtocol rpcHighTidenode)
  throws IOException {
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();

  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class,
      RetryPolicies.retryByRemoteException(
          RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (HighTideProtocol) RetryProxy.create(HighTideProtocol.class,
      rpcHighTidenode, methodNameToPolicyMap);
}

Example 6

Source File: DFSClient.java From RDFS with Apache License 2.0

6 votes

private static ClientProtocol createNamenode(ClientProtocol rpcNamenode,
    Configuration conf)
  throws IOException {
  long sleepTime = conf.getLong("dfs.client.rpc.retry.sleep",
      LEASE_SOFTLIMIT_PERIOD);
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      5, sleepTime, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy);

  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class,
      RetryPolicies.retryByRemoteException(
          RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (ClientProtocol) RetryProxy.create(ClientProtocol.class,
      rpcNamenode, methodNameToPolicyMap);
}

Example 7

Source File: S3FileSystem.java From RDFS with Apache License 2.0

6 votes

private static FileSystemStore createDefaultStore(Configuration conf) {
  FileSystemStore store = new Jets3tFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                             conf.getInt("fs.s3.maxRetries", 4),
                                                                             conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                            RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  methodNameToPolicyMap.put("storeBlock", methodPolicy);
  methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
  
  return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                             store, methodNameToPolicyMap);
}

Example 8

Source File: NativeS3FileSystem.java From RDFS with Apache License 2.0

6 votes

private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}

Example 9

Source File: AvatarShell.java From RDFS with Apache License 2.0

6 votes

private static AvatarProtocol createAvatarnode(AvatarProtocol rpcAvatarnode)
    throws IOException {
  RetryPolicy createPolicy = RetryPolicies
      .retryUpToMaximumCountWithFixedSleep(5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>, RetryPolicy> remoteExceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap = new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class, RetryPolicies
      .retryByRemoteException(RetryPolicies.TRY_ONCE_THEN_FAIL,
          remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap = new HashMap<String, RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (AvatarProtocol) RetryProxy.create(AvatarProtocol.class,
      rpcAvatarnode, methodNameToPolicyMap);
}

Example 10

Source File: SnapshotShell.java From RDFS with Apache License 2.0

6 votes

private static SnapshotProtocol createSnapshotNode(SnapshotProtocol rpcSnapshotNode) 
throws IOException {
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                 5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
       new HashMap<Class<? extends Exception>, RetryPolicy>();
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();

  exceptionToPolicyMap.put(RemoteException.class,
RetryPolicies.retryByRemoteException(
    RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
    RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);

  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (SnapshotProtocol) RetryProxy.create(SnapshotProtocol.class, 
                               rpcSnapshotNode, methodNameToPolicyMap);
}

Example 11

Source File: RaidShell.java From RDFS with Apache License 2.0

6 votes

private static RaidProtocol createRaidnode(RaidProtocol rpcRaidnode)
  throws IOException {
  RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      5, 5000, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();

  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(RemoteException.class,
      RetryPolicies.retryByRemoteException(
          RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();

  methodNameToPolicyMap.put("create", methodPolicy);

  return (RaidProtocol) RetryProxy.create(RaidProtocol.class,
      rpcRaidnode, methodNameToPolicyMap);
}

Example 12

Source File: S3FileSystem.java From big-c with Apache License 2.0

6 votes

private static FileSystemStore createDefaultStore(Configuration conf) {
  FileSystemStore store = new Jets3tFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                             conf.getInt("fs.s3.maxRetries", 4),
                                                                             conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                            RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  methodNameToPolicyMap.put("storeBlock", methodPolicy);
  methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
  
  return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                             store, methodNameToPolicyMap);
}

Example 13

Source File: S3FileSystem.java From hadoop with Apache License 2.0

6 votes

private static FileSystemStore createDefaultStore(Configuration conf) {
  FileSystemStore store = new Jets3tFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                             conf.getInt("fs.s3.maxRetries", 4),
                                                                             conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                            RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
  methodNameToPolicyMap.put("storeBlock", methodPolicy);
  methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
  
  return (FileSystemStore) RetryProxy.create(FileSystemStore.class,
                                             store, methodNameToPolicyMap);
}

Example 14

Source File: NativeS3FileSystem.java From hadoop with Apache License 2.0

6 votes

private static NativeFileSystemStore createDefaultStore(Configuration conf) {
  NativeFileSystemStore store = new Jets3tNativeFileSystemStore();
  
  RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
      conf.getInt("fs.s3.maxRetries", 4),
      conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(IOException.class, basePolicy);
  exceptionToPolicyMap.put(S3Exception.class, basePolicy);
  
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
  Map<String, RetryPolicy> methodNameToPolicyMap =
    new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("storeFile", methodPolicy);
  methodNameToPolicyMap.put("rename", methodPolicy);
  
  return (NativeFileSystemStore)
    RetryProxy.create(NativeFileSystemStore.class, store,
        methodNameToPolicyMap);
}

Example 15

Source File: ServerProxy.java From hadoop with Apache License 2.0

5 votes

protected static RetryPolicy createRetryPolicy(Configuration conf,
    String maxWaitTimeStr, long defMaxWaitTime,
    String connectRetryIntervalStr, long defRetryInterval) {
  long maxWaitTime = conf.getLong(maxWaitTimeStr, defMaxWaitTime);
  long retryIntervalMS =
      conf.getLong(connectRetryIntervalStr, defRetryInterval);
  if (maxWaitTime == -1) {
    // wait forever.
    return RetryPolicies.RETRY_FOREVER;
  }

  Preconditions.checkArgument(maxWaitTime > 0, "Invalid Configuration. "
      + maxWaitTimeStr + " should be a positive value.");
  Preconditions.checkArgument(retryIntervalMS > 0, "Invalid Configuration. "
      + connectRetryIntervalStr + "should be a positive value.");

  RetryPolicy retryPolicy =
      RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
        retryIntervalMS, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);
  exceptionToPolicyMap.put(NMNotYetReadyException.class, retryPolicy);

  return RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
    exceptionToPolicyMap);
}

Example 16

Source File: Balancer.java From hadoop-gpu with Apache License 2.0

5 votes

private static NamenodeProtocol createNamenode(Configuration conf)
  throws IOException {
  InetSocketAddress nameNodeAddr = NameNode.getAddress(conf);
  RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(
      5, 200, TimeUnit.MILLISECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      timeoutPolicy, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap =
      new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("getBlocks", methodPolicy);

  UserGroupInformation ugi;
  try {
    ugi = UnixUserGroupInformation.login(conf);
  } catch (javax.security.auth.login.LoginException e) {
    throw new IOException(StringUtils.stringifyException(e));
  }

  return (NamenodeProtocol) RetryProxy.create(
      NamenodeProtocol.class,
      RPC.getProxy(NamenodeProtocol.class,
          NamenodeProtocol.versionID,
          nameNodeAddr,
          ugi,
          conf,
          NetUtils.getDefaultSocketFactory(conf)),
      methodNameToPolicyMap);
}

Example 17

Source File: Balancer.java From RDFS with Apache License 2.0

5 votes

private static NamenodeProtocol createNamenode(InetSocketAddress nameNodeAddr, Configuration conf)
  throws IOException {
  RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(
      5, 200, TimeUnit.MILLISECONDS);
  Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
    new HashMap<Class<? extends Exception>, RetryPolicy>();
  RetryPolicy methodPolicy = RetryPolicies.retryByException(
      timeoutPolicy, exceptionToPolicyMap);
  Map<String,RetryPolicy> methodNameToPolicyMap =
      new HashMap<String, RetryPolicy>();
  methodNameToPolicyMap.put("getBlocks", methodPolicy);

  UserGroupInformation ugi;
  try {
    ugi = UnixUserGroupInformation.login(conf);
  } catch (javax.security.auth.login.LoginException e) {
    throw new IOException(StringUtils.stringifyException(e));
  }

  return (NamenodeProtocol) RetryProxy.create(
      NamenodeProtocol.class,
      RPC.getProxy(NamenodeProtocol.class,
          NamenodeProtocol.versionID,
          nameNodeAddr,
          ugi,
          conf,
          NetUtils.getDefaultSocketFactory(conf)),
      methodNameToPolicyMap);
}

Example 18

Source File: ServerProxy.java From big-c with Apache License 2.0

5 votes

protected static RetryPolicy createRetryPolicy(Configuration conf,
    String maxWaitTimeStr, long defMaxWaitTime,
    String connectRetryIntervalStr, long defRetryInterval) {
  long maxWaitTime = conf.getLong(maxWaitTimeStr, defMaxWaitTime);
  long retryIntervalMS =
      conf.getLong(connectRetryIntervalStr, defRetryInterval);
  if (maxWaitTime == -1) {
    // wait forever.
    return RetryPolicies.RETRY_FOREVER;
  }

  Preconditions.checkArgument(maxWaitTime > 0, "Invalid Configuration. "
      + maxWaitTimeStr + " should be a positive value.");
  Preconditions.checkArgument(retryIntervalMS > 0, "Invalid Configuration. "
      + connectRetryIntervalStr + "should be a positive value.");

  RetryPolicy retryPolicy =
      RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
        retryIntervalMS, TimeUnit.MILLISECONDS);

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);
  exceptionToPolicyMap.put(NMNotYetReadyException.class, retryPolicy);

  return RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,
    exceptionToPolicyMap);
}

Example 19

Source File: RMProxy.java From big-c with Apache License 2.0

4 votes

/**
 * Fetch retry policy from Configuration
 */
@Private
@VisibleForTesting
public static RetryPolicy createRetryPolicy(Configuration conf) {
  long rmConnectWaitMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
          YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
  long rmConnectionRetryIntervalMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
          YarnConfiguration
              .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS);

  boolean waitForEver = (rmConnectWaitMS == -1);
  if (!waitForEver) {
    if (rmConnectWaitMS < 0) {
      throw new YarnRuntimeException("Invalid Configuration. "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " can be -1, but can not be other negative numbers");
    }

    // try connect once
    if (rmConnectWaitMS < rmConnectionRetryIntervalMS) {
      LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " is smaller than "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
          + ". Only try connect once.");
      rmConnectWaitMS = 0;
    }
  }

  // Handle HA case first
  if (HAUtil.isHAEnabled(conf)) {
    final long failoverSleepBaseMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS,
        rmConnectionRetryIntervalMS);

    final long failoverSleepMaxMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_MAX_MS,
        rmConnectionRetryIntervalMS);

    int maxFailoverAttempts = conf.getInt(
        YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, -1);

    if (maxFailoverAttempts == -1) {
      if (waitForEver) {
        maxFailoverAttempts = Integer.MAX_VALUE;
      } else {
        maxFailoverAttempts = (int) (rmConnectWaitMS / failoverSleepBaseMs);
      }
    }

    return RetryPolicies.failoverOnNetworkException(
        RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts,
        failoverSleepBaseMs, failoverSleepMaxMs);
  }

  if (rmConnectionRetryIntervalMS < 0) {
    throw new YarnRuntimeException("Invalid Configuration. " +
        YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
        " should not be negative.");
  }

  RetryPolicy retryPolicy = null;
  if (waitForEver) {
    retryPolicy = RetryPolicies.RETRY_FOREVER;
  } else {
    retryPolicy =
        RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
            rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
  }

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();

  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);

  return RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
}

Example 20

Source File: RMProxy.java From hadoop with Apache License 2.0

4 votes

/**
 * Fetch retry policy from Configuration
 */
@Private
@VisibleForTesting
public static RetryPolicy createRetryPolicy(Configuration conf) {
  long rmConnectWaitMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
          YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
  long rmConnectionRetryIntervalMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
          YarnConfiguration
              .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS);

  boolean waitForEver = (rmConnectWaitMS == -1);
  if (!waitForEver) {
    if (rmConnectWaitMS < 0) {
      throw new YarnRuntimeException("Invalid Configuration. "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " can be -1, but can not be other negative numbers");
    }

    // try connect once
    if (rmConnectWaitMS < rmConnectionRetryIntervalMS) {
      LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " is smaller than "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
          + ". Only try connect once.");
      rmConnectWaitMS = 0;
    }
  }

  // Handle HA case first
  if (HAUtil.isHAEnabled(conf)) {
    final long failoverSleepBaseMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS,
        rmConnectionRetryIntervalMS);

    final long failoverSleepMaxMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_MAX_MS,
        rmConnectionRetryIntervalMS);

    int maxFailoverAttempts = conf.getInt(
        YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, -1);

    if (maxFailoverAttempts == -1) {
      if (waitForEver) {
        maxFailoverAttempts = Integer.MAX_VALUE;
      } else {
        maxFailoverAttempts = (int) (rmConnectWaitMS / failoverSleepBaseMs);
      }
    }

    return RetryPolicies.failoverOnNetworkException(
        RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts,
        failoverSleepBaseMs, failoverSleepMaxMs);
  }

  if (rmConnectionRetryIntervalMS < 0) {
    throw new YarnRuntimeException("Invalid Configuration. " +
        YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
        " should not be negative.");
  }

  RetryPolicy retryPolicy = null;
  if (waitForEver) {
    retryPolicy = RetryPolicies.RETRY_FOREVER;
  } else {
    retryPolicy =
        RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
            rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
  }

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();

  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);

  return RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
}