org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl Java Examples

The following examples show how to use org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestAMRMClientAsync.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 10000)
public void testAMRMClientAsyncShutDown() throws Exception {
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  createAllocateResponse(new ArrayList<ContainerStatus>(),
    new ArrayList<Container>(), null);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  asyncClient.registerApplicationMaster("localhost", 1234, null);

  Thread.sleep(50);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 
Example #2
Source File: TestAMRMClientAsync.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 10000)
public void testAMRMClientAsyncShutDown() throws Exception {
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  createAllocateResponse(new ArrayList<ContainerStatus>(),
    new ArrayList<Container>(), null);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  asyncClient.registerApplicationMaster("localhost", 1234, null);

  Thread.sleep(50);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 
Example #3
Source File: TestTezAMRMClient.java    From tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Before
public void setup() {
  amrmClient = new TezAMRMClientAsync(new AMRMClientImpl(),
    1000, mock(AMRMClientAsync.CallbackHandler.class));
  RackResolver.init(new Configuration());
}
 
Example #4
Source File: TestAMRMClientAsync.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor()
    throws YarnException, IOException, InterruptedException {
  Configuration conf = new Configuration();
  final TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  asyncClient.init(conf);
  asyncClient.start();

  Supplier<Boolean> checker = new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return callbackHandler.notify;
    }
  };

  asyncClient.registerApplicationMaster("localhost", 1234, null);
  asyncClient.waitFor(checker);
  Assert.assertTrue(checker.get());
}
 
Example #5
Source File: TestAMRMClientAsync.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandler()
    throws YarnException, IOException, InterruptedException {
  Configuration conf = new Configuration();
  TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  asyncClient.init(conf);
  asyncClient.start();

  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.notify == false) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
}
 
Example #6
Source File: TestAMRMClientAsync.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 10000)
public void testAMRMClientAsyncShutDownWithWaitFor() throws Exception {
  Configuration conf = new Configuration();
  final TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  Supplier<Boolean> checker = new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return callbackHandler.reboot;
    }
  };

  asyncClient.registerApplicationMaster("localhost", 1234, null);
  asyncClient.waitFor(checker);

  asyncClient.stop();
  // stopping should have joined all threads and completed all callbacks
  Assert.assertTrue(callbackHandler.callbackCount == 0);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 
Example #7
Source File: TestAMRMClientAsync.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void runHeartBeatThrowOutException(Exception ex) throws Exception{
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
  when(client.allocate(anyFloat())).thenThrow(ex);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();
  
  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.savedException == null) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
  Assert.assertTrue(callbackHandler.savedException.getMessage().contains(
      ex.getMessage()));
  
  asyncClient.stop();
  // stopping should have joined all threads and completed all callbacks
  Assert.assertTrue(callbackHandler.callbackCount == 0);
}
 
Example #8
Source File: TestAMRMClientAsync.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor()
    throws YarnException, IOException, InterruptedException {
  Configuration conf = new Configuration();
  final TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  asyncClient.init(conf);
  asyncClient.start();

  Supplier<Boolean> checker = new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return callbackHandler.notify;
    }
  };

  asyncClient.registerApplicationMaster("localhost", 1234, null);
  asyncClient.waitFor(checker);
  Assert.assertTrue(checker.get());
}
 
Example #9
Source File: TestAMRMClientAsync.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandler()
    throws YarnException, IOException, InterruptedException {
  Configuration conf = new Configuration();
  TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  asyncClient.init(conf);
  asyncClient.start();

  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.notify == false) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
}
 
Example #10
Source File: TestAMRMClientAsync.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 10000)
public void testAMRMClientAsyncShutDownWithWaitFor() throws Exception {
  Configuration conf = new Configuration();
  final TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  Supplier<Boolean> checker = new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return callbackHandler.reboot;
    }
  };

  asyncClient.registerApplicationMaster("localhost", 1234, null);
  asyncClient.waitFor(checker);

  asyncClient.stop();
  // stopping should have joined all threads and completed all callbacks
  Assert.assertTrue(callbackHandler.callbackCount == 0);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 
Example #11
Source File: TestAMRMClientAsync.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void runHeartBeatThrowOutException(Exception ex) throws Exception{
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
  when(client.allocate(anyFloat())).thenThrow(ex);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();
  
  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.savedException == null) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
  Assert.assertTrue(callbackHandler.savedException.getMessage().contains(
      ex.getMessage()));
  
  asyncClient.stop();
  // stopping should have joined all threads and completed all callbacks
  Assert.assertTrue(callbackHandler.callbackCount == 0);
}
 
Example #12
Source File: TezAMRMClientAsync.java    From tez with Apache License 2.0 4 votes vote down vote up
public TezAMRMClientAsync(int intervalMs, CallbackHandler callbackHandler) {
  super(new AMRMClientImpl<T>(), intervalMs, callbackHandler);
}
 
Example #13
Source File: TestTezAMRMClient.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@Before
public void setup() {
  amrmClient = new TezAMRMClientAsync(new AMRMClientImpl(),
    1000, mock(AMRMClientAsync.CallbackHandler.class));
  RackResolver.init(new Configuration());
}
 
Example #14
Source File: TezAMRMClientAsync.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
public TezAMRMClientAsync(int intervalMs, CallbackHandler callbackHandler) {
  super(new AMRMClientImpl<T>(), intervalMs, callbackHandler);
}
 
Example #15
Source File: TestAMRMClientAsync.java    From big-c with Apache License 2.0 4 votes vote down vote up
void runCallBackThrowOutException(TestCallbackHandler2 callbackHandler) throws
      InterruptedException, YarnException, IOException {
  Configuration conf = new Configuration();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);
  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  callbackHandler.throwOutException = true;
  asyncClient.init(conf);
  asyncClient.start();

  // call register and wait for error callback and stop
  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.notify == false) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
  // verify error invoked
  verify(callbackHandler, times(0)).getProgress();
  verify(callbackHandler, times(1)).onError(any(Exception.class));
  // sleep to wait for a few heartbeat calls that can trigger callbacks
  Thread.sleep(50);
  // verify no more invocations after the first one.
  // ie. callback thread has stopped
  verify(callbackHandler, times(0)).getProgress();
  verify(callbackHandler, times(1)).onError(any(Exception.class));
}
 
Example #16
Source File: DagAwareYarnTaskScheduler.java    From tez with Apache License 2.0 4 votes vote down vote up
@Override
public void initialize() throws Exception {
  initialize(new AMRMClientAsyncWrapper(new AMRMClientImpl<TaskRequest>(), 1000, this));
}
 
Example #17
Source File: AMRMClientAsyncImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
public AMRMClientAsyncImpl(int intervalMs, CallbackHandler callbackHandler) {
  this(new AMRMClientImpl<T>(), intervalMs, callbackHandler);
}
 
Example #18
Source File: AMRMClientAsync.java    From big-c with Apache License 2.0 4 votes vote down vote up
protected AMRMClientAsync(int intervalMs, CallbackHandler callbackHandler) {
  this(new AMRMClientImpl<T>(), intervalMs, callbackHandler);
}
 
Example #19
Source File: TestAMRMClientAsync.java    From hadoop with Apache License 2.0 4 votes vote down vote up
void runCallBackThrowOutException(TestCallbackHandler2 callbackHandler) throws
      InterruptedException, YarnException, IOException {
  Configuration conf = new Configuration();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);
  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  callbackHandler.throwOutException = true;
  asyncClient.init(conf);
  asyncClient.start();

  // call register and wait for error callback and stop
  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.notify == false) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
  // verify error invoked
  verify(callbackHandler, times(0)).getProgress();
  verify(callbackHandler, times(1)).onError(any(Exception.class));
  // sleep to wait for a few heartbeat calls that can trigger callbacks
  Thread.sleep(50);
  // verify no more invocations after the first one.
  // ie. callback thread has stopped
  verify(callbackHandler, times(0)).getProgress();
  verify(callbackHandler, times(1)).onError(any(Exception.class));
}
 
Example #20
Source File: AMRMClientAsyncImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public AMRMClientAsyncImpl(int intervalMs, CallbackHandler callbackHandler) {
  this(new AMRMClientImpl<T>(), intervalMs, callbackHandler);
}
 
Example #21
Source File: AMRMClientAsync.java    From hadoop with Apache License 2.0 4 votes vote down vote up
protected AMRMClientAsync(int intervalMs, CallbackHandler callbackHandler) {
  this(new AMRMClientImpl<T>(), intervalMs, callbackHandler);
}
 
Example #22
Source File: AMRMClient.java    From hadoop with Apache License 2.0 2 votes vote down vote up
/**
 * Create a new instance of AMRMClient.
 * For usage:
 * <pre>
 * {@code
 * AMRMClient.<T>createAMRMClientContainerRequest()
 * }</pre>
 * @return the newly create AMRMClient instance.
 */
@Public
public static <T extends ContainerRequest> AMRMClient<T> createAMRMClient() {
  AMRMClient<T> client = new AMRMClientImpl<T>();
  return client;
}
 
Example #23
Source File: AMRMClient.java    From big-c with Apache License 2.0 2 votes vote down vote up
/**
 * Create a new instance of AMRMClient.
 * For usage:
 * <pre>
 * {@code
 * AMRMClient.<T>createAMRMClientContainerRequest()
 * }</pre>
 * @return the newly create AMRMClient instance.
 */
@Public
public static <T extends ContainerRequest> AMRMClient<T> createAMRMClient() {
  AMRMClient<T> client = new AMRMClientImpl<T>();
  return client;
}