org.apache.hadoop.ipc.StandbyException Java Examples

The following examples show how to use org.apache.hadoop.ipc.StandbyException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestFailoverProxy.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailoverOnNetworkExceptionIdempotentOperation()
    throws UnreliableException, IOException, StandbyException {
  UnreliableInterface unreliable = (UnreliableInterface)RetryProxy.create(
      UnreliableInterface.class,
      newFlipFlopProxyProvider(
          TypeOfExceptionToFailWith.IO_EXCEPTION,
          TypeOfExceptionToFailWith.UNRELIABLE_EXCEPTION),
      RetryPolicies.failoverOnNetworkException(1));
  
  assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningString());
  try {
    unreliable.succeedsOnceThenFailsReturningString();
    fail("should not have succeeded twice");
  } catch (IOException e) {
    // Make sure we *don't* fail over since the first implementation threw an
    // IOException and this method is not idempotent
    assertEquals("impl1", e.getMessage());
  }
  
  assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningStringIdempotent());
  // Make sure we fail over since the first implementation threw an
  // IOException and this method is idempotent.
  assertEquals("impl2", unreliable.succeedsOnceThenFailsReturningStringIdempotent());
}
 
Example #2
Source File: DelegationTokenSecretManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public byte[] retrievePassword(
    DelegationTokenIdentifier identifier) throws InvalidToken {
  try {
    // this check introduces inconsistency in the authentication to a
    // HA standby NN.  non-token auths are allowed into the namespace which
    // decides whether to throw a StandbyException.  tokens are a bit
    // different in that a standby may be behind and thus not yet know
    // of all tokens issued by the active NN.  the following check does
    // not allow ANY token auth, however it should allow known tokens in
    namesystem.checkOperation(OperationCategory.READ);
  } catch (StandbyException se) {
    // FIXME: this is a hack to get around changing method signatures by
    // tunneling a non-InvalidToken exception as the cause which the
    // RPC server will unwrap before returning to the client
    InvalidToken wrappedStandby = new InvalidToken("StandbyException");
    wrappedStandby.initCause(se);
    throw wrappedStandby;
  }
  return super.retrievePassword(identifier);
}
 
Example #3
Source File: DelegationTokenSecretManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public byte[] retriableRetrievePassword(DelegationTokenIdentifier identifier)
    throws InvalidToken, StandbyException, RetriableException, IOException {
  namesystem.checkOperation(OperationCategory.READ);
  try {
    return super.retrievePassword(identifier);
  } catch (InvalidToken it) {
    if (namesystem.inTransitionToActive()) {
      // if the namesystem is currently in the middle of transition to 
      // active state, let client retry since the corresponding editlog may 
      // have not been applied yet
      throw new RetriableException(it);
    } else {
      throw it;
    }
  }
}
 
Example #4
Source File: AdminService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public RefreshNodesResponse refreshNodes(RefreshNodesRequest request)
    throws YarnException, StandbyException {
  String argName = "refreshNodes";
  final String msg = "refresh nodes.";
  UserGroupInformation user = checkAcls("refreshNodes");

  checkRMStatus(user.getShortUserName(), argName, msg);

  try {
    Configuration conf =
        getConfiguration(new Configuration(false),
            YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
    rmContext.getNodesListManager().refreshNodes(conf);
    RMAuditLogger.logSuccess(user.getShortUserName(), argName,
        "AdminService");
    return recordFactory.newRecordInstance(RefreshNodesResponse.class);
  } catch (IOException ioe) {
    throw logAndWrapException(ioe, user.getShortUserName(), argName, msg);
  }
}
 
Example #5
Source File: AdminService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
    throws YarnException, StandbyException {
  String argName = "refreshQueues";
  final String msg = "refresh queues.";
  UserGroupInformation user = checkAcls(argName);

  checkRMStatus(user.getShortUserName(), argName, msg);

  RefreshQueuesResponse response =
      recordFactory.newRecordInstance(RefreshQueuesResponse.class);
  try {
    rmContext.getScheduler().reinitialize(getConfig(), this.rmContext);
    // refresh the reservation system
    ReservationSystem rSystem = rmContext.getReservationSystem();
    if (rSystem != null) {
      rSystem.reinitialize(getConfig(), rmContext);
    }
    RMAuditLogger.logSuccess(user.getShortUserName(), argName,
        "AdminService");
    return response;
  } catch (IOException ioe) {
    throw logAndWrapException(ioe, user.getShortUserName(), argName, msg);
  }
}
 
Example #6
Source File: HAUtil.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Used to ensure that at least one of the given HA NNs is currently in the
 * active state..
 * 
 * @param namenodes list of RPC proxies for each NN to check.
 * @return true if at least one NN is active, false if all are in the standby state.
 * @throws IOException in the event of error.
 */
public static boolean isAtLeastOneActive(List<ClientProtocol> namenodes)
    throws IOException {
  for (ClientProtocol namenode : namenodes) {
    try {
      namenode.getFileInfo("/");
      return true;
    } catch (RemoteException re) {
      IOException cause = re.unwrapRemoteException();
      if (cause instanceof StandbyException) {
        // This is expected to happen for a standby NN.
      } else {
        throw re;
      }
    }
  }
  return false;
}
 
Example #7
Source File: HAUtil.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Used to ensure that at least one of the given HA NNs is currently in the
 * active state..
 * 
 * @param namenodes list of RPC proxies for each NN to check.
 * @return true if at least one NN is active, false if all are in the standby state.
 * @throws IOException in the event of error.
 */
public static boolean isAtLeastOneActive(List<ClientProtocol> namenodes)
    throws IOException {
  for (ClientProtocol namenode : namenodes) {
    try {
      namenode.getFileInfo("/");
      return true;
    } catch (RemoteException re) {
      IOException cause = re.unwrapRemoteException();
      if (cause instanceof StandbyException) {
        // This is expected to happen for a standby NN.
      } else {
        throw re;
      }
    }
  }
  return false;
}
 
Example #8
Source File: TestFailoverProxy.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccedsOnceThenFailOver() throws UnreliableException,
    IOException, StandbyException {
  UnreliableInterface unreliable = (UnreliableInterface)RetryProxy.create(
      UnreliableInterface.class, newFlipFlopProxyProvider(),
      new FailOverOnceOnAnyExceptionPolicy());
  
  assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningString());
  assertEquals("impl2", unreliable.succeedsOnceThenFailsReturningString());
  try {
    unreliable.succeedsOnceThenFailsReturningString();
    fail("should not have succeeded more than twice");
  } catch (UnreliableException e) {
    // expected
  }
}
 
Example #9
Source File: DelegationTokenSecretManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public byte[] retriableRetrievePassword(DelegationTokenIdentifier identifier)
    throws InvalidToken, StandbyException, RetriableException, IOException {
  namesystem.checkOperation(OperationCategory.READ);
  try {
    return super.retrievePassword(identifier);
  } catch (InvalidToken it) {
    if (namesystem.inTransitionToActive()) {
      // if the namesystem is currently in the middle of transition to 
      // active state, let client retry since the corresponding editlog may 
      // have not been applied yet
      throw new RetriableException(it);
    } else {
      throw it;
    }
  }
}
 
Example #10
Source File: DelegationTokenSecretManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public byte[] retrievePassword(
    DelegationTokenIdentifier identifier) throws InvalidToken {
  try {
    // this check introduces inconsistency in the authentication to a
    // HA standby NN.  non-token auths are allowed into the namespace which
    // decides whether to throw a StandbyException.  tokens are a bit
    // different in that a standby may be behind and thus not yet know
    // of all tokens issued by the active NN.  the following check does
    // not allow ANY token auth, however it should allow known tokens in
    namesystem.checkOperation(OperationCategory.READ);
  } catch (StandbyException se) {
    // FIXME: this is a hack to get around changing method signatures by
    // tunneling a non-InvalidToken exception as the cause which the
    // RPC server will unwrap before returning to the client
    InvalidToken wrappedStandby = new InvalidToken("StandbyException");
    wrappedStandby.initCause(se);
    throw wrappedStandby;
  }
  return super.retrievePassword(identifier);
}
 
Example #11
Source File: AdminService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public RefreshNodesResponse refreshNodes(RefreshNodesRequest request)
    throws YarnException, StandbyException {
  String argName = "refreshNodes";
  final String msg = "refresh nodes.";
  UserGroupInformation user = checkAcls("refreshNodes");

  checkRMStatus(user.getShortUserName(), argName, msg);

  try {
    Configuration conf =
        getConfiguration(new Configuration(false),
            YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
    rmContext.getNodesListManager().refreshNodes(conf);
    RMAuditLogger.logSuccess(user.getShortUserName(), argName,
        "AdminService");
    return recordFactory.newRecordInstance(RefreshNodesResponse.class);
  } catch (IOException ioe) {
    throw logAndWrapException(ioe, user.getShortUserName(), argName, msg);
  }
}
 
Example #12
Source File: ThrottleInvocationHandler.java    From nnproxy with Apache License 2.0 6 votes vote down vote up
@Override
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
    AtomicLong counter = opCounter.apply(method);
    Preconditions.checkState(counter != null);
    long current = counter.getAndIncrement();
    try {
        if (current > threshold) {
            NNProxy.proxyMetrics.throttledOps.incr();
            throw new StandbyException("Too many requests (" + current + "/" + threshold + "), try later");
        }
        Object ret = method.invoke(underlying, args);
        NNProxy.proxyMetrics.successOps.incr();
        return ret;
    } catch (InvocationTargetException e) {
        NNProxy.proxyMetrics.failedOps.incr();
        throw e.getCause();
    } finally {
        counter.decrementAndGet();
    }
}
 
Example #13
Source File: TestFailoverProxy.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailoverOnNetworkExceptionIdempotentOperation()
    throws UnreliableException, IOException, StandbyException {
  UnreliableInterface unreliable = (UnreliableInterface)RetryProxy.create(
      UnreliableInterface.class,
      newFlipFlopProxyProvider(
          TypeOfExceptionToFailWith.IO_EXCEPTION,
          TypeOfExceptionToFailWith.UNRELIABLE_EXCEPTION),
      RetryPolicies.failoverOnNetworkException(1));
  
  assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningString());
  try {
    unreliable.succeedsOnceThenFailsReturningString();
    fail("should not have succeeded twice");
  } catch (IOException e) {
    // Make sure we *don't* fail over since the first implementation threw an
    // IOException and this method is not idempotent
    assertEquals("impl1", e.getMessage());
  }
  
  assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningStringIdempotent());
  // Make sure we fail over since the first implementation threw an
  // IOException and this method is idempotent.
  assertEquals("impl2", unreliable.succeedsOnceThenFailsReturningStringIdempotent());
}
 
Example #14
Source File: TestFailoverProxy.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testNeverFailOver() throws UnreliableException,
    IOException, StandbyException {
  UnreliableInterface unreliable = (UnreliableInterface)RetryProxy.create(
      UnreliableInterface.class,
      newFlipFlopProxyProvider(),
      RetryPolicies.TRY_ONCE_THEN_FAIL);

  unreliable.succeedsOnceThenFailsReturningString();
  try {
    unreliable.succeedsOnceThenFailsReturningString();
    fail("should not have succeeded twice");
  } catch (UnreliableException e) {
    assertEquals("impl1", e.getMessage());
  }
}
 
Example #15
Source File: TestFailoverProxy.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testNeverFailOver() throws UnreliableException,
    IOException, StandbyException {
  UnreliableInterface unreliable = (UnreliableInterface)RetryProxy.create(
      UnreliableInterface.class,
      newFlipFlopProxyProvider(),
      RetryPolicies.TRY_ONCE_THEN_FAIL);

  unreliable.succeedsOnceThenFailsReturningString();
  try {
    unreliable.succeedsOnceThenFailsReturningString();
    fail("should not have succeeded twice");
  } catch (UnreliableException e) {
    assertEquals("impl1", e.getMessage());
  }
}
 
Example #16
Source File: TestFailoverProxy.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccedsOnceThenFailOver() throws UnreliableException,
    IOException, StandbyException {
  UnreliableInterface unreliable = (UnreliableInterface)RetryProxy.create(
      UnreliableInterface.class, newFlipFlopProxyProvider(),
      new FailOverOnceOnAnyExceptionPolicy());
  
  assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningString());
  assertEquals("impl2", unreliable.succeedsOnceThenFailsReturningString());
  try {
    unreliable.succeedsOnceThenFailsReturningString();
    fail("should not have succeeded more than twice");
  } catch (UnreliableException e) {
    // expected
  }
}
 
Example #17
Source File: RetryPolicies.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public RetryAction shouldRetry(Exception e, int retries,
    int failovers, boolean isIdempotentOrAtMostOnce) throws Exception {
  if (failovers >= maxFailovers) {
    return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
        "failovers (" + failovers + ") exceeded maximum allowed ("
        + maxFailovers + ")");
  }
  if (retries - failovers > maxRetries) {
    return new RetryAction(RetryAction.RetryDecision.FAIL, 0, "retries ("
        + retries + ") exceeded maximum allowed (" + maxRetries + ")");
  }
  
  if (e instanceof ConnectException ||
      e instanceof NoRouteToHostException ||
      e instanceof UnknownHostException ||
      e instanceof StandbyException ||
      e instanceof ConnectTimeoutException ||
      isWrappedStandbyException(e)) {
    return new RetryAction(RetryAction.RetryDecision.FAILOVER_AND_RETRY,
        getFailoverOrRetrySleepTime(failovers));
  } else if (e instanceof RetriableException
      || getWrappedRetriableException(e) != null) {
    // RetriableException or RetriableException wrapped 
    return new RetryAction(RetryAction.RetryDecision.RETRY,
          getFailoverOrRetrySleepTime(retries));
  } else if (e instanceof SocketException
      || (e instanceof IOException && !(e instanceof RemoteException))) {
    if (isIdempotentOrAtMostOnce) {
      return RetryAction.FAILOVER_AND_RETRY;
    } else {
      return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
          "the invoked method is not idempotent, and unable to determine "
              + "whether it was invoked");
    }
  } else {
      return fallbackPolicy.shouldRetry(e, retries, failovers,
          isIdempotentOrAtMostOnce);
  }
}
 
Example #18
Source File: StandbyState.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void checkOperation(HAContext context, OperationCategory op)
    throws StandbyException {
  if (op == OperationCategory.UNCHECKED ||
      (op == OperationCategory.READ && context.allowStaleReads())) {
    return;
  }
  String msg = "Operation category " + op + " is not supported in state "
      + context.getState();
  throw new StandbyException(msg);
}
 
Example #19
Source File: BackupNode.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override // NameNodeHAContext
public void checkOperation(OperationCategory op)
    throws StandbyException {
  if (op == OperationCategory.UNCHECKED ||
      op == OperationCategory.CHECKPOINT) {
    return;
  }
  if (OperationCategory.JOURNAL != op &&
      !(OperationCategory.READ == op && !isRole(NamenodeRole.CHECKPOINT))) {
    String msg = "Operation category " + op
        + " is not supported at " + getRole();
    throw new StandbyException(msg);
  }
}
 
Example #20
Source File: AdminService.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void checkRMStatus(String user, String argName, String msg)
    throws StandbyException {
  if (!isRMActive()) {
    RMAuditLogger.logFailure(user, argName, "", 
        "AdminService", "ResourceManager is not active. Can not " + msg);
    throwStandbyException();
  }
}
 
Example #21
Source File: TestFailoverProxy.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testSucceedsTenTimesThenFailOver() throws UnreliableException,
    IOException, StandbyException {
  UnreliableInterface unreliable = (UnreliableInterface)RetryProxy.create(
      UnreliableInterface.class,
      newFlipFlopProxyProvider(),
      new FailOverOnceOnAnyExceptionPolicy());
  
  for (int i = 0; i < 10; i++) {
    assertEquals("impl1", unreliable.succeedsTenTimesThenFailsReturningString());
  }
  assertEquals("impl2", unreliable.succeedsTenTimesThenFailsReturningString());
}
 
Example #22
Source File: UnreliableImplementation.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void nonIdempotentVoidFailsIfIdentifierDoesntMatch(String identifier)
    throws UnreliableException, StandbyException, IOException {
  if (this.identifier.equals(identifier)) {
    return;
  } else {
    String message = "expected '" + this.identifier + "' but received '" +
        identifier + "'";
    throwAppropriateException(exceptionToFailWith, message);
  }
}
 
Example #23
Source File: TestFailoverProxy.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Ensure that when all configured services are throwing StandbyException
 * that we fail over back and forth between them until one is no longer
 * throwing StandbyException.
 */
@Test
public void testFailoverBetweenMultipleStandbys()
    throws UnreliableException, StandbyException, IOException {
  
  final long millisToSleep = 10000;
  
  final UnreliableImplementation impl1 = new UnreliableImplementation("impl1",
      TypeOfExceptionToFailWith.STANDBY_EXCEPTION);
  FlipFlopProxyProvider<UnreliableInterface> proxyProvider
      = new FlipFlopProxyProvider<UnreliableInterface>(
      UnreliableInterface.class,
      impl1,
      new UnreliableImplementation("impl2",
          TypeOfExceptionToFailWith.STANDBY_EXCEPTION));
  
  final UnreliableInterface unreliable = (UnreliableInterface)RetryProxy
    .create(UnreliableInterface.class, proxyProvider,
        RetryPolicies.failoverOnNetworkException(
            RetryPolicies.TRY_ONCE_THEN_FAIL, 10, 1000, 10000));
  
  new Thread() {
    @Override
    public void run() {
      ThreadUtil.sleepAtLeastIgnoreInterrupts(millisToSleep);
      impl1.setIdentifier("renamed-impl1");
    }
  }.start();
  
  String result = unreliable.failsIfIdentifierDoesntMatch("renamed-impl1");
  assertEquals("renamed-impl1", result);
}
 
Example #24
Source File: TestFailoverProxy.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public String failsIfIdentifierDoesntMatch(String identifier)
    throws UnreliableException, StandbyException, IOException {
  // Wait until all threads are trying to invoke this method
  methodLatch.countDown();
  try {
    methodLatch.await();
  } catch (InterruptedException e) {
    throw new RuntimeException(e);
  }
  return super.failsIfIdentifierDoesntMatch(identifier);
}
 
Example #25
Source File: UnreliableImplementation.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public String succeedsTenTimesThenFailsReturningString()
    throws UnreliableException, IOException, StandbyException {
  if (succeedsTenTimesThenFailsCount++ < 10) {
    return identifier;
  } else {
    throwAppropriateException(exceptionToFailWith, identifier);
    return null;
  }
}
 
Example #26
Source File: TestFailoverProxy.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testFailoverOnStandbyException()
    throws UnreliableException, IOException, StandbyException {
  UnreliableInterface unreliable = (UnreliableInterface)RetryProxy.create(
      UnreliableInterface.class,
      newFlipFlopProxyProvider(),
      RetryPolicies.failoverOnNetworkException(1));
  
  assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningString());
  try {
    unreliable.succeedsOnceThenFailsReturningString();
    fail("should not have succeeded twice");
  } catch (UnreliableException e) {
    // Make sure there was no failover on normal exception.
    assertEquals("impl1", e.getMessage());
  }
  
  unreliable = (UnreliableInterface)RetryProxy
  .create(UnreliableInterface.class,
      newFlipFlopProxyProvider(
          TypeOfExceptionToFailWith.STANDBY_EXCEPTION,
          TypeOfExceptionToFailWith.UNRELIABLE_EXCEPTION),
      RetryPolicies.failoverOnNetworkException(1));
  
  assertEquals("impl1", unreliable.succeedsOnceThenFailsReturningString());
  // Make sure we fail over since the first implementation threw a StandbyException
  assertEquals("impl2", unreliable.succeedsOnceThenFailsReturningString());
}
 
Example #27
Source File: UnreliableImplementation.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public String succeedsOnceThenFailsReturningStringIdempotent()
    throws UnreliableException, StandbyException, IOException {
  if (succeedsOnceThenFailsIdempotentCount++ < 1) {
    return identifier;
  } else {
    throwAppropriateException(exceptionToFailWith, identifier);
    return null;
  }
}
 
Example #28
Source File: UnreliableImplementation.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public String failsIfIdentifierDoesntMatch(String identifier)
    throws UnreliableException, StandbyException, IOException {
  if (this.identifier.equals(identifier)) {
    return identifier;
  } else {
    String message = "expected '" + this.identifier + "' but received '" +
        identifier + "'";
    throwAppropriateException(exceptionToFailWith, message);
    return null;
  }
}
 
Example #29
Source File: TestFailoverProxy.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testSucceedsTenTimesThenFailOver() throws UnreliableException,
    IOException, StandbyException {
  UnreliableInterface unreliable = (UnreliableInterface)RetryProxy.create(
      UnreliableInterface.class,
      newFlipFlopProxyProvider(),
      new FailOverOnceOnAnyExceptionPolicy());
  
  for (int i = 0; i < 10; i++) {
    assertEquals("impl1", unreliable.succeedsTenTimesThenFailsReturningString());
  }
  assertEquals("impl2", unreliable.succeedsTenTimesThenFailsReturningString());
}
 
Example #30
Source File: TestFailoverProxy.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Ensure that when all configured services are throwing StandbyException
 * that we fail over back and forth between them until one is no longer
 * throwing StandbyException.
 */
@Test
public void testFailoverBetweenMultipleStandbys()
    throws UnreliableException, StandbyException, IOException {
  
  final long millisToSleep = 10000;
  
  final UnreliableImplementation impl1 = new UnreliableImplementation("impl1",
      TypeOfExceptionToFailWith.STANDBY_EXCEPTION);
  FlipFlopProxyProvider<UnreliableInterface> proxyProvider
      = new FlipFlopProxyProvider<UnreliableInterface>(
      UnreliableInterface.class,
      impl1,
      new UnreliableImplementation("impl2",
          TypeOfExceptionToFailWith.STANDBY_EXCEPTION));
  
  final UnreliableInterface unreliable = (UnreliableInterface)RetryProxy
    .create(UnreliableInterface.class, proxyProvider,
        RetryPolicies.failoverOnNetworkException(
            RetryPolicies.TRY_ONCE_THEN_FAIL, 10, 1000, 10000));
  
  new Thread() {
    @Override
    public void run() {
      ThreadUtil.sleepAtLeastIgnoreInterrupts(millisToSleep);
      impl1.setIdentifier("renamed-impl1");
    }
  }.start();
  
  String result = unreliable.failsIfIdentifierDoesntMatch("renamed-impl1");
  assertEquals("renamed-impl1", result);
}