org.apache.hadoop.net.ConnectTimeoutException Java Examples

@Override
public boolean canInfinitelyRetry(Throwable t){
    t=Throwables.getRootCause(t);
    t=processPipelineException(t);
    if(t instanceof NotServingPartitionException
            || t instanceof NoServerForRegionException
            || t instanceof WrongPartitionException
            || t instanceof PipelineTooBusy
            || t instanceof RegionBusyException
            || t instanceof NoRouteToHostException
            || t instanceof org.apache.hadoop.hbase.ipc.FailedServerException
            || t instanceof FailedServerException
            || t instanceof ServerNotRunningYetException
            || t instanceof ConnectTimeoutException
            || t instanceof IndexNotSetUpException) return true;
    return false;
}
 

@Override
public Socket createSocket() throws IOException {
  Socket spy = Mockito.spy(defaultFactory.createSocket());
  // Simplify our spying job by not having to also spy on the channel
  Mockito.doReturn(null).when(spy).getChannel();
  // Throw a ConnectTimeoutException when connecting to our target "bad"
  // host.
  Mockito.doThrow(new ConnectTimeoutException("injected"))
    .when(spy).connect(
        Mockito.argThat(new MatchesPort()),
        Mockito.anyInt());
  return spy;
}
 

@Override
public RetryAction shouldRetry(Exception e, int retries,
    int failovers, boolean isIdempotentOrAtMostOnce) throws Exception {
  if (failovers >= maxFailovers) {
    return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
        "failovers (" + failovers + ") exceeded maximum allowed ("
        + maxFailovers + ")");
  }
  if (retries - failovers > maxRetries) {
    return new RetryAction(RetryAction.RetryDecision.FAIL, 0, "retries ("
        + retries + ") exceeded maximum allowed (" + maxRetries + ")");
  }
  
  if (e instanceof ConnectException ||
      e instanceof NoRouteToHostException ||
      e instanceof UnknownHostException ||
      e instanceof StandbyException ||
      e instanceof ConnectTimeoutException ||
      isWrappedStandbyException(e)) {
    return new RetryAction(RetryAction.RetryDecision.FAILOVER_AND_RETRY,
        getFailoverOrRetrySleepTime(failovers));
  } else if (e instanceof RetriableException
      || getWrappedRetriableException(e) != null) {
    // RetriableException or RetriableException wrapped 
    return new RetryAction(RetryAction.RetryDecision.RETRY,
          getFailoverOrRetrySleepTime(retries));
  } else if (e instanceof SocketException
      || (e instanceof IOException && !(e instanceof RemoteException))) {
    if (isIdempotentOrAtMostOnce) {
      return RetryAction.FAILOVER_AND_RETRY;
    } else {
      return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
          "the invoked method is not idempotent, and unable to determine "
              + "whether it was invoked");
    }
  } else {
      return fallbackPolicy.shouldRetry(e, retries, failovers,
          isIdempotentOrAtMostOnce);
  }
}
 

private void assertRetriesOnSocketTimeouts(Configuration conf,
    int maxTimeoutRetries) throws IOException {
  SocketFactory mockFactory = Mockito.mock(SocketFactory.class);
  doThrow(new ConnectTimeoutException("fake")).when(mockFactory).createSocket();
  Client client = new Client(IntWritable.class, conf, mockFactory);
  InetSocketAddress address = new InetSocketAddress("127.0.0.1", 9090);
  try {
    client.call(new IntWritable(RANDOM.nextInt()), address, null, null, 0,
        conf);
    fail("Not throwing the SocketTimeoutException");
  } catch (SocketTimeoutException e) {
    Mockito.verify(mockFactory, Mockito.times(maxTimeoutRetries))
        .createSocket();
  }
}
 

@Override
public Socket createSocket() throws IOException {
  Socket spy = Mockito.spy(defaultFactory.createSocket());
  // Simplify our spying job by not having to also spy on the channel
  Mockito.doReturn(null).when(spy).getChannel();
  // Throw a ConnectTimeoutException when connecting to our target "bad"
  // host.
  Mockito.doThrow(new ConnectTimeoutException("injected"))
    .when(spy).connect(
        Mockito.argThat(new MatchesPort()),
        Mockito.anyInt());
  return spy;
}
 

@Override
public RetryAction shouldRetry(Exception e, int retries,
    int failovers, boolean isIdempotentOrAtMostOnce) throws Exception {
  if (failovers >= maxFailovers) {
    return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
        "failovers (" + failovers + ") exceeded maximum allowed ("
        + maxFailovers + ")");
  }
  if (retries - failovers > maxRetries) {
    return new RetryAction(RetryAction.RetryDecision.FAIL, 0, "retries ("
        + retries + ") exceeded maximum allowed (" + maxRetries + ")");
  }
  
  if (e instanceof ConnectException ||
      e instanceof NoRouteToHostException ||
      e instanceof UnknownHostException ||
      e instanceof StandbyException ||
      e instanceof ConnectTimeoutException ||
      isWrappedStandbyException(e)) {
    return new RetryAction(RetryAction.RetryDecision.FAILOVER_AND_RETRY,
        getFailoverOrRetrySleepTime(failovers));
  } else if (e instanceof RetriableException
      || getWrappedRetriableException(e) != null) {
    // RetriableException or RetriableException wrapped 
    return new RetryAction(RetryAction.RetryDecision.RETRY,
          getFailoverOrRetrySleepTime(retries));
  } else if (e instanceof SocketException
      || (e instanceof IOException && !(e instanceof RemoteException))) {
    if (isIdempotentOrAtMostOnce) {
      return RetryAction.FAILOVER_AND_RETRY;
    } else {
      return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
          "the invoked method is not idempotent, and unable to determine "
              + "whether it was invoked");
    }
  } else {
      return fallbackPolicy.shouldRetry(e, retries, failovers,
          isIdempotentOrAtMostOnce);
  }
}
 

private void assertRetriesOnSocketTimeouts(Configuration conf,
    int maxTimeoutRetries) throws IOException {
  SocketFactory mockFactory = Mockito.mock(SocketFactory.class);
  doThrow(new ConnectTimeoutException("fake")).when(mockFactory).createSocket();
  Client client = new Client(IntWritable.class, conf, mockFactory);
  InetSocketAddress address = new InetSocketAddress("127.0.0.1", 9090);
  try {
    client.call(new IntWritable(RANDOM.nextInt()), address, null, null, 0,
        conf);
    fail("Not throwing the SocketTimeoutException");
  } catch (SocketTimeoutException e) {
    Mockito.verify(mockFactory, Mockito.times(maxTimeoutRetries))
        .createSocket();
  }
}
 

/**
 * Fetch retry policy from Configuration
 */
@Private
@VisibleForTesting
public static RetryPolicy createRetryPolicy(Configuration conf) {
  long rmConnectWaitMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
          YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
  long rmConnectionRetryIntervalMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
          YarnConfiguration
              .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS);

  boolean waitForEver = (rmConnectWaitMS == -1);
  if (!waitForEver) {
    if (rmConnectWaitMS < 0) {
      throw new YarnRuntimeException("Invalid Configuration. "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " can be -1, but can not be other negative numbers");
    }

    // try connect once
    if (rmConnectWaitMS < rmConnectionRetryIntervalMS) {
      LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " is smaller than "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
          + ". Only try connect once.");
      rmConnectWaitMS = 0;
    }
  }

  // Handle HA case first
  if (HAUtil.isHAEnabled(conf)) {
    final long failoverSleepBaseMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS,
        rmConnectionRetryIntervalMS);

    final long failoverSleepMaxMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_MAX_MS,
        rmConnectionRetryIntervalMS);

    int maxFailoverAttempts = conf.getInt(
        YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, -1);

    if (maxFailoverAttempts == -1) {
      if (waitForEver) {
        maxFailoverAttempts = Integer.MAX_VALUE;
      } else {
        maxFailoverAttempts = (int) (rmConnectWaitMS / failoverSleepBaseMs);
      }
    }

    return RetryPolicies.failoverOnNetworkException(
        RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts,
        failoverSleepBaseMs, failoverSleepMaxMs);
  }

  if (rmConnectionRetryIntervalMS < 0) {
    throw new YarnRuntimeException("Invalid Configuration. " +
        YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
        " should not be negative.");
  }

  RetryPolicy retryPolicy = null;
  if (waitForEver) {
    retryPolicy = RetryPolicies.RETRY_FOREVER;
  } else {
    retryPolicy =
        RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
            rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
  }

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();

  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);

  return RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
}
 

private synchronized void setupConnection() throws IOException {
  short ioFailures = 0;
  short timeoutFailures = 0;
  while (true) {
    try {
      this.socket = socketFactory.createSocket();
      this.socket.setTcpNoDelay(tcpNoDelay);
      this.socket.setKeepAlive(true);
      
      /*
       * Bind the socket to the host specified in the principal name of the
       * client, to ensure Server matching address of the client connection
       * to host name in principal passed.
       */
      UserGroupInformation ticket = remoteId.getTicket();
      if (ticket != null && ticket.hasKerberosCredentials()) {
        KerberosInfo krbInfo = 
          remoteId.getProtocol().getAnnotation(KerberosInfo.class);
        if (krbInfo != null && krbInfo.clientPrincipal() != null) {
          String host = 
            SecurityUtil.getHostFromPrincipal(remoteId.getTicket().getUserName());
          
          // If host name is a valid local address then bind socket to it
          InetAddress localAddr = NetUtils.getLocalInetAddress(host);
          if (localAddr != null) {
            this.socket.bind(new InetSocketAddress(localAddr, 0));
          }
        }
      }
      
      NetUtils.connect(this.socket, server, connectionTimeout);
      if (rpcTimeout > 0) {
        pingInterval = rpcTimeout;  // rpcTimeout overwrites pingInterval
      }
      this.socket.setSoTimeout(pingInterval);
      return;
    } catch (ConnectTimeoutException toe) {
      /* Check for an address change and update the local reference.
       * Reset the failure counter if the address was changed
       */
      if (updateAddress()) {
        timeoutFailures = ioFailures = 0;
      }
      handleConnectionTimeout(timeoutFailures++,
          maxRetriesOnSocketTimeouts, toe);
    } catch (IOException ie) {
      if (updateAddress()) {
        timeoutFailures = ioFailures = 0;
      }
      handleConnectionFailure(ioFailures++, ie);
    }
  }
}
 

/**
 * Fetch retry policy from Configuration
 */
@Private
@VisibleForTesting
public static RetryPolicy createRetryPolicy(Configuration conf) {
  long rmConnectWaitMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
          YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
  long rmConnectionRetryIntervalMS =
      conf.getLong(
          YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
          YarnConfiguration
              .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS);

  boolean waitForEver = (rmConnectWaitMS == -1);
  if (!waitForEver) {
    if (rmConnectWaitMS < 0) {
      throw new YarnRuntimeException("Invalid Configuration. "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " can be -1, but can not be other negative numbers");
    }

    // try connect once
    if (rmConnectWaitMS < rmConnectionRetryIntervalMS) {
      LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
          + " is smaller than "
          + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
          + ". Only try connect once.");
      rmConnectWaitMS = 0;
    }
  }

  // Handle HA case first
  if (HAUtil.isHAEnabled(conf)) {
    final long failoverSleepBaseMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS,
        rmConnectionRetryIntervalMS);

    final long failoverSleepMaxMs = conf.getLong(
        YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_MAX_MS,
        rmConnectionRetryIntervalMS);

    int maxFailoverAttempts = conf.getInt(
        YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, -1);

    if (maxFailoverAttempts == -1) {
      if (waitForEver) {
        maxFailoverAttempts = Integer.MAX_VALUE;
      } else {
        maxFailoverAttempts = (int) (rmConnectWaitMS / failoverSleepBaseMs);
      }
    }

    return RetryPolicies.failoverOnNetworkException(
        RetryPolicies.TRY_ONCE_THEN_FAIL, maxFailoverAttempts,
        failoverSleepBaseMs, failoverSleepMaxMs);
  }

  if (rmConnectionRetryIntervalMS < 0) {
    throw new YarnRuntimeException("Invalid Configuration. " +
        YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
        " should not be negative.");
  }

  RetryPolicy retryPolicy = null;
  if (waitForEver) {
    retryPolicy = RetryPolicies.RETRY_FOREVER;
  } else {
    retryPolicy =
        RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
            rmConnectionRetryIntervalMS, TimeUnit.MILLISECONDS);
  }

  Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();

  exceptionToPolicyMap.put(EOFException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectException.class, retryPolicy);
  exceptionToPolicyMap.put(NoRouteToHostException.class, retryPolicy);
  exceptionToPolicyMap.put(UnknownHostException.class, retryPolicy);
  exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
  exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
  exceptionToPolicyMap.put(SocketException.class, retryPolicy);

  return RetryPolicies.retryByException(
      RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
}
 

private synchronized void setupConnection() throws IOException {
  short ioFailures = 0;
  short timeoutFailures = 0;
  while (true) {
    try {
      this.socket = socketFactory.createSocket();
      this.socket.setTcpNoDelay(tcpNoDelay);
      this.socket.setKeepAlive(true);
      
      /*
       * Bind the socket to the host specified in the principal name of the
       * client, to ensure Server matching address of the client connection
       * to host name in principal passed.
       */
      UserGroupInformation ticket = remoteId.getTicket();
      if (ticket != null && ticket.hasKerberosCredentials()) {
        KerberosInfo krbInfo = 
          remoteId.getProtocol().getAnnotation(KerberosInfo.class);
        if (krbInfo != null && krbInfo.clientPrincipal() != null) {
          String host = 
            SecurityUtil.getHostFromPrincipal(remoteId.getTicket().getUserName());
          
          // If host name is a valid local address then bind socket to it
          InetAddress localAddr = NetUtils.getLocalInetAddress(host);
          if (localAddr != null) {
            this.socket.bind(new InetSocketAddress(localAddr, 0));
          }
        }
      }
      
      NetUtils.connect(this.socket, server, connectionTimeout);
      if (rpcTimeout > 0) {
        pingInterval = rpcTimeout;  // rpcTimeout overwrites pingInterval
      }
      this.socket.setSoTimeout(pingInterval);
      return;
    } catch (ConnectTimeoutException toe) {
      /* Check for an address change and update the local reference.
       * Reset the failure counter if the address was changed
       */
      if (updateAddress()) {
        timeoutFailures = ioFailures = 0;
      }
      handleConnectionTimeout(timeoutFailures++,
          maxRetriesOnSocketTimeouts, toe);
    } catch (IOException ie) {
      if (updateAddress()) {
        timeoutFailures = ioFailures = 0;
      }
      handleConnectionFailure(ioFailures++, ie);
    }
  }
}
 

@Test
public void testRetryNoRouteToHostException() throws Throwable {
    WriteConfiguration config = new DefaultWriteConfiguration(new Monitor(0,0,10,10L,0),pef);
    WriteResult writeResult = new WriteResult(Code.FAILED, "NoRouteToHostException:No route to host");
    BulkWriteResult bulkWriteResult = new BulkWriteResult(writeResult);
    WriteResponse response = config.processGlobalResult(bulkWriteResult);
    Assert.assertEquals(WriteResponse.RETRY, response);

    writeResult = new WriteResult(Code.FAILED, "FailedServerException:This server is in the failed servers list");
    bulkWriteResult = new BulkWriteResult(writeResult);
    response = config.processGlobalResult(bulkWriteResult);
    Assert.assertEquals(WriteResponse.RETRY, response);

    writeResult = new WriteResult(Code.FAILED, "ServerNotRunningYetException");
    bulkWriteResult = new BulkWriteResult(writeResult);
    response = config.processGlobalResult(bulkWriteResult);
    Assert.assertEquals(WriteResponse.RETRY, response);

    writeResult = new WriteResult(Code.FAILED, "ConnectTimeoutException");
    bulkWriteResult = new BulkWriteResult(writeResult);
    response = config.processGlobalResult(bulkWriteResult);
    Assert.assertEquals(WriteResponse.RETRY, response);

    writeResult = new WriteResult(Code.PARTIAL);
    IntObjectHashMap<WriteResult> failedRows = new IntObjectHashMap<>();
    failedRows.put(1, new WriteResult(Code.FAILED, "NoRouteToHostException:No route to host"));
    bulkWriteResult = new BulkWriteResult(writeResult, new IntHashSet(), failedRows);
    response = config.partialFailure(bulkWriteResult, null);
    Assert.assertEquals(WriteResponse.RETRY, response);


    writeResult = new WriteResult(Code.PARTIAL);
    failedRows = new IntObjectHashMap<>();
    failedRows.put(1, new WriteResult(Code.FAILED, "FailedServerException:This server is in the failed servers list"));
    bulkWriteResult = new BulkWriteResult(writeResult, new IntHashSet(), failedRows);
    response = config.partialFailure(bulkWriteResult, null);
    Assert.assertEquals(WriteResponse.RETRY, response);

    writeResult = new WriteResult(Code.PARTIAL);
    failedRows = new IntObjectHashMap<>();
    failedRows.put(1, new WriteResult(Code.FAILED, "ServerNotRunningYetException"));
    bulkWriteResult = new BulkWriteResult(writeResult, new IntHashSet(), failedRows);
    response = config.partialFailure(bulkWriteResult, null);
    Assert.assertEquals(WriteResponse.RETRY, response);

    writeResult = new WriteResult(Code.PARTIAL);
    failedRows = new IntObjectHashMap<>();
    failedRows.put(1, new WriteResult(Code.FAILED, "ConnectTimeoutException"));
    bulkWriteResult = new BulkWriteResult(writeResult, new IntHashSet(), failedRows);
    response = config.partialFailure(bulkWriteResult, null);
    Assert.assertEquals(WriteResponse.RETRY, response);


    NoRouteToHostException nrthe = new NoRouteToHostException();
    response = config.globalError(nrthe);
    Assert.assertEquals(WriteResponse.RETRY, response);

    FailedServerException failedServerException = new FailedServerException("Failed server");
    response = config.globalError(failedServerException);
    Assert.assertEquals(WriteResponse.RETRY, response);

    ServerNotRunningYetException serverNotRunningYetException = new ServerNotRunningYetException("Server not running");
    response = config.globalError(serverNotRunningYetException);
    Assert.assertEquals(WriteResponse.RETRY, response);

    ConnectTimeoutException connectTimeoutException = new ConnectTimeoutException("connect timeout");
    response = config.globalError(connectTimeoutException);
    Assert.assertEquals(WriteResponse.RETRY, response);
}