org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest Java Examples

The following examples show how to use org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestAMRMClientContainerRequest.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (expected = InvalidContainerRequestException.class)
public void testDifferentLocalityRelaxationSamePriority() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  ContainerRequest request2 =
      new ContainerRequest(capability, new String[] {"host3"},
          null, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
}
 
Example #2
Source File: JstormMaster.java    From PoseidonX with Apache License 2.0 6 votes vote down vote up
/**
 * JstormAM 的构造函数
 */
public JstormMaster() throws IOException {
    conf = new YarnConfiguration();
    Path jstormYarnConfPath = new Path("jstorm-yarn.xml");
    conf.addResource(jstormYarnConfPath);

    JstormAMContext.serviceUserName = RegistryUtils.currentUser();
    LOG.error("### JstormAMContext.serviceUserName:["+JstormAMContext.serviceUserName+"]");
    registryClient = RegistryClient.init(JstormAMConstant.REGISTRY_JSTORM_YARN,JstormAMContext.serviceUserName,conf);
    slotPortsView = new SlotPortsView(registryClient);
    slotPortsView.setMinPort(JstormAMConstant.SUPERVISOR_MIN_PORT);
    slotPortsView.setMaxPort(JstormAMConstant.SUPERVISOR_MAX_PORT);
    JstormAMContext.requestBlockingQueue = new LinkedBlockingQueue<ContainerRequest>();

    JstormAMContext.currentNimbusNum = new AtomicInteger(0);
    JstormAMContext.currentSupervisorNum = new AtomicInteger(0);
}
 
Example #3
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testWaitFor() throws InterruptedException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  CountDownSupplier countDownChecker = new CountDownSupplier();

  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
            .<ContainerRequest> createAMRMClient();
    amClient.init(new YarnConfiguration());
    amClient.start();
    amClient.waitFor(countDownChecker, 1000);
    assertEquals(3, countDownChecker.counter);
  } finally {
    if (amClient != null) {
      amClient.stop();
    }
  }
}
 
Example #4
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 6 votes vote down vote up
private int getAllocatedContainersNumber(
    AMRMClientImpl<ContainerRequest> amClient, int iterationsLeft)
    throws YarnException, IOException {
  int allocatedContainerCount = 0;
  while (iterationsLeft-- > 0) {
    Log.info(" == alloc " + allocatedContainerCount + " it left " + iterationsLeft);
    AllocateResponse allocResponse = amClient.allocate(0.1f);
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
      
    assertEquals(nodeCount, amClient.getClusterNodeCount());
    allocatedContainerCount += allocResponse.getAllocatedContainers().size();
      
    if(allocatedContainerCount == 0) {
      // sleep to let NM's heartbeat to RM and trigger allocations
      sleep(100);
    }
  }
  return allocatedContainerCount;
}
 
Example #5
Source File: TestAMRMClientAsync.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 10000)
public void testAMRMClientAsyncShutDown() throws Exception {
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  createAllocateResponse(new ArrayList<ContainerStatus>(),
    new ArrayList<Container>(), null);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  asyncClient.registerApplicationMaster("localhost", 1234, null);

  Thread.sleep(50);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 
Example #6
Source File: TestAMRMClientContainerRequest.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFillInRacks() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
 
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          new String[] {"/rack2"}, Priority.newInstance(1));
  client.addContainerRequest(request);
  verifyResourceRequest(client, request, "host1", true);
  verifyResourceRequest(client, request, "host2", true);
  verifyResourceRequest(client, request, "/rack1", true);
  verifyResourceRequest(client, request, "/rack2", true);
  verifyResourceRequest(client, request, ResourceRequest.ANY, true);
}
 
Example #7
Source File: ResourceRequestHandler.java    From Bats with Apache License 2.0 6 votes vote down vote up
/**
 * Setup the request(s) that will be sent to the RM for the container ask.
 */
public ContainerRequest createContainerRequest(ContainerStartRequest csr, boolean first)
{
  int priority = csr.container.getResourceRequestPriority();
  // check for node locality constraint
  String[] nodes = null;
  String[] racks = null;

  String host = getHost(csr, first);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(csr.container.getRequiredMemoryMB());
  capability.setVirtualCores(csr.container.getRequiredVCores());
  if (host == INVALID_HOST) {
    return null;
  }
  if (host != null) {
    nodes = new String[]{host};
    // in order to request a host, we don't have to set the rack if the locality is false
    /*
     * if(this.nodeToRack.get(host) != null){ racks = new String[] { this.nodeToRack.get(host) }; }
     */
    return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority), false);
  }
  // For now, only memory is supported so we set memory requirements
  return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority));
}
 
Example #8
Source File: TestAMRMClientContainerRequest.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (expected = InvalidContainerRequestException.class)
public void testLocalityRelaxationDifferentLevels() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  ContainerRequest request2 =
      new ContainerRequest(capability, null,
          new String[] {"rack1"}, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
}
 
Example #9
Source File: TestAMRMClientContainerRequest.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (expected = InvalidContainerRequestException.class)
public void testLocalityRelaxationDifferentLevels() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  ContainerRequest request2 =
      new ContainerRequest(capability, null,
          new String[] {"rack1"}, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
}
 
Example #10
Source File: TestAMRMClientContainerRequest.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (expected = InvalidContainerRequestException.class)
public void testDifferentLocalityRelaxationSamePriority() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  ContainerRequest request2 =
      new ContainerRequest(capability, new String[] {"host3"},
          null, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
}
 
Example #11
Source File: ResourceRequestHandler.java    From Bats with Apache License 2.0 6 votes vote down vote up
/**
 * Issue requests to AM RM Client again if previous container requests expired and were not allocated by Yarn
 * @param amRmClient
 * @param requestedResources
 * @param loopCounter
 * @param resourceRequestor
 * @param containerRequests
 * @param removedContainerRequests
 */
public void reissueContainerRequests(AMRMClient<ContainerRequest> amRmClient, Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources, int loopCounter, ResourceRequestHandler resourceRequestor, List<ContainerRequest> containerRequests, List<ContainerRequest> removedContainerRequests)
{
  if (!requestedResources.isEmpty()) {
    for (Map.Entry<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> entry : requestedResources.entrySet()) {
      /*
       * Create container requests again if pending requests were not allocated by Yarn till timeout.
       */
      if ((loopCounter - entry.getValue().getKey()) > NUMBER_MISSED_HEARTBEATS) {
        StreamingContainerAgent.ContainerStartRequest csr = entry.getKey();
        LOG.debug("Request for container {} timed out. Re-requesting container", csr.container);
        removedContainerRequests.add(entry.getValue().getRight());
        ContainerRequest cr = resourceRequestor.createContainerRequest(csr, false);
        entry.getValue().setLeft(loopCounter);
        entry.getValue().setRight(cr);
        containerRequests.add(cr);
      }
    }
  }
}
 
Example #12
Source File: TestAMRMClientContainerRequest.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testFillInRacks() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
 
  Resource capability = Resource.newInstance(1024, 1);
  ContainerRequest request =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          new String[] {"/rack2"}, Priority.newInstance(1));
  client.addContainerRequest(request);
  verifyResourceRequest(client, request, "host1", true);
  verifyResourceRequest(client, request, "host2", true);
  verifyResourceRequest(client, request, "/rack1", true);
  verifyResourceRequest(client, request, "/rack2", true);
  verifyResourceRequest(client, request, ResourceRequest.ANY, true);
}
 
Example #13
Source File: ResourceRequestHandler.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
/**
 * Setup the request(s) that will be sent to the RM for the container ask.
 */
public ContainerRequest createContainerRequest(ContainerStartRequest csr, boolean first)
{
  int priority = csr.container.getResourceRequestPriority();
  // check for node locality constraint
  String[] nodes = null;
  String[] racks = null;

  String host = getHost(csr, first);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(csr.container.getRequiredMemoryMB());
  capability.setVirtualCores(csr.container.getRequiredVCores());
  if (host == INVALID_HOST) {
    return null;
  }
  if (host != null) {
    nodes = new String[]{host};
    // in order to request a host, we don't have to set the rack if the locality is false
    /*
     * if(this.nodeToRack.get(host) != null){ racks = new String[] { this.nodeToRack.get(host) }; }
     */
    return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority), false);
  }
  // For now, only memory is supported so we set memory requirements
  return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority));
}
 
Example #14
Source File: ResourceRequestHandler.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
/**
 * Issue requests to AM RM Client again if previous container requests expired and were not allocated by Yarn
 * @param amRmClient
 * @param requestedResources
 * @param loopCounter
 * @param resourceRequestor
 * @param containerRequests
 * @param removedContainerRequests
 */
public void reissueContainerRequests(AMRMClient<ContainerRequest> amRmClient, Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources, int loopCounter, ResourceRequestHandler resourceRequestor, List<ContainerRequest> containerRequests, List<ContainerRequest> removedContainerRequests)
{
  if (!requestedResources.isEmpty()) {
    for (Map.Entry<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> entry : requestedResources.entrySet()) {
      /*
       * Create container requests again if pending requests were not allocated by Yarn till timeout.
       */
      if ((loopCounter - entry.getValue().getKey()) > NUMBER_MISSED_HEARTBEATS) {
        StreamingContainerAgent.ContainerStartRequest csr = entry.getKey();
        LOG.debug("Request for container {} timed out. Re-requesting container", csr.container);
        removedContainerRequests.add(entry.getValue().getRight());
        ContainerRequest cr = resourceRequestor.createContainerRequest(csr, false);
        entry.getValue().setLeft(loopCounter);
        entry.getValue().setRight(cr);
        containerRequests.add(cr);
      }
    }
  }
}
 
Example #15
Source File: TestAMRMClient.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private int getAllocatedContainersNumber(
    AMRMClientImpl<ContainerRequest> amClient, int iterationsLeft)
    throws YarnException, IOException {
  int allocatedContainerCount = 0;
  while (iterationsLeft-- > 0) {
    Log.info(" == alloc " + allocatedContainerCount + " it left " + iterationsLeft);
    AllocateResponse allocResponse = amClient.allocate(0.1f);
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
      
    assertEquals(nodeCount, amClient.getClusterNodeCount());
    allocatedContainerCount += allocResponse.getAllocatedContainers().size();
      
    if(allocatedContainerCount == 0) {
      // sleep to let NM's heartbeat to RM and trigger allocations
      sleep(100);
    }
  }
  return allocatedContainerCount;
}
 
Example #16
Source File: ApplicationMaster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Setup the request that will be sent to the RM for the container ask.
 *
 * @return the setup ResourceRequest to be sent to RM
 */
private ContainerRequest setupContainerAskForRM() {
  // setup requirements for hosts
  // using * as any host will do for the distributed shell app
  // set the priority for the request
  // TODO - what is the range for priority? how to decide?
  Priority pri = Priority.newInstance(requestPriority);

  // Set up resource type requirements
  // For now, memory and CPU are supported so we set memory and cpu requirements
  Resource capability = Resource.newInstance(containerMemory,
    containerVirtualCores, containerGpuCores);

  ContainerRequest request = new ContainerRequest(capability, null, null,
      pri);
  LOG.info("Requested container ask: " + request.toString());
  return request;
}
 
Example #17
Source File: ApplicationMaster.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Setup the request that will be sent to the RM for the container ask.
 *
 * @return the setup ResourceRequest to be sent to RM
 */
private ContainerRequest setupContainerAskForRM() {
  // setup requirements for hosts
  // using * as any host will do for the distributed shell app
  // set the priority for the request
  // TODO - what is the range for priority? how to decide?
  Priority pri = Priority.newInstance(requestPriority);

  // Set up resource type requirements
  // For now, memory and CPU are supported so we set memory and cpu requirements
  Resource capability = Resource.newInstance(containerMemory,
    containerVirtualCores);

  ContainerRequest request = new ContainerRequest(capability, null, null,
      pri);
  LOG.info("Requested container ask: " + request.toString());
  return request;
}
 
Example #18
Source File: TestAMRMClient.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testWaitFor() throws InterruptedException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  CountDownSupplier countDownChecker = new CountDownSupplier();

  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
            .<ContainerRequest> createAMRMClient();
    amClient.init(new YarnConfiguration());
    amClient.start();
    amClient.waitFor(countDownChecker, 1000);
    assertEquals(3, countDownChecker.counter);
  } finally {
    if (amClient != null) {
      amClient.stop();
    }
  }
}
 
Example #19
Source File: TestAMRMClientAsync.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 10000)
public void testAMRMClientAsyncShutDown() throws Exception {
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  createAllocateResponse(new ArrayList<ContainerStatus>(),
    new ArrayList<Container>(), null);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  asyncClient.registerApplicationMaster("localhost", 1234, null);

  Thread.sleep(50);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 
Example #20
Source File: TestAMRMClientAsync.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 10000)
public void testAMRMClientAsyncShutDownWithWaitFor() throws Exception {
  Configuration conf = new Configuration();
  final TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  Supplier<Boolean> checker = new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return callbackHandler.reboot;
    }
  };

  asyncClient.registerApplicationMaster("localhost", 1234, null);
  asyncClient.waitFor(checker);

  asyncClient.stop();
  // stopping should have joined all threads and completed all callbacks
  Assert.assertTrue(callbackHandler.callbackCount == 0);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 
Example #21
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=30000)
public void testAskWithInvalidNodeLabels() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();

  // specified exp with more than one node labels
  verifyAddRequestFailed(client,
      new ContainerRequest(Resource.newInstance(1024, 1), null, null,
          Priority.UNDEFINED, true, "x && y"));
}
 
Example #22
Source File: TestAMRMClientContainerRequest.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void verifyResourceRequest(
    AMRMClientImpl<ContainerRequest> client, ContainerRequest request,
    String location, boolean expectedRelaxLocality) {
  ResourceRequest ask =  client.remoteRequestsTable.get(request.getPriority())
      .get(location).get(request.getCapability()).remoteRequest;
  assertEquals(location, ask.getResourceName());
  assertEquals(1, ask.getNumContainers());
  assertEquals(expectedRelaxLocality, ask.getRelaxLocality());
}
 
Example #23
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void verifyAddRequestFailed(AMRMClient<ContainerRequest> client,
    ContainerRequest request) {
  try {
    client.addContainerRequest(request);
  } catch (InvalidContainerRequestException e) {
    return;
  }
  Assert.fail();
}
 
Example #24
Source File: TestAMRMClientAsync.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor()
    throws YarnException, IOException, InterruptedException {
  Configuration conf = new Configuration();
  final TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  asyncClient.init(conf);
  asyncClient.start();

  Supplier<Boolean> checker = new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return callbackHandler.notify;
    }
  };

  asyncClient.registerApplicationMaster("localhost", 1234, null);
  asyncClient.waitFor(checker);
  Assert.assertTrue(checker.get());
}
 
Example #25
Source File: TestAMRMClientContainerRequest.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testInvalidValidWhenOldRemoved() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  
  client.removeContainerRequest(request1);

  ContainerRequest request2 =
      new ContainerRequest(capability, new String[] {"host3"},
          null, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
  
  client.removeContainerRequest(request2);
  
  ContainerRequest request3 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request3);
  
  client.removeContainerRequest(request3);
  
  ContainerRequest request4 =
      new ContainerRequest(capability, null,
          new String[] {"rack1"}, Priority.newInstance(1), true);
  client.addContainerRequest(request4);

}
 
Example #26
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout=60000)
public void testAMRMClient() throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.<ContainerRequest>createAMRMClient();

    //setting an instance NMTokenCache
    amClient.setNMTokenCache(new NMTokenCache());
    //asserting we are not using the singleton instance cache
    Assert.assertNotSame(NMTokenCache.getSingleton(), 
        amClient.getNMTokenCache());

    amClient.init(conf);
    amClient.start();

    amClient.registerApplicationMaster("Host", 10000, "");

    testAllocation((AMRMClientImpl<ContainerRequest>)amClient);

    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 
Example #27
Source File: TestAMRMClientAsync.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor()
    throws YarnException, IOException, InterruptedException {
  Configuration conf = new Configuration();
  final TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  asyncClient.init(conf);
  asyncClient.start();

  Supplier<Boolean> checker = new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return callbackHandler.notify;
    }
  };

  asyncClient.registerApplicationMaster("localhost", 1234, null);
  asyncClient.waitFor(checker);
  Assert.assertTrue(checker.get());
}
 
Example #28
Source File: TestAMRMClientAsync.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandler()
    throws YarnException, IOException, InterruptedException {
  Configuration conf = new Configuration();
  TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  asyncClient.init(conf);
  asyncClient.start();

  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.notify == false) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
}
 
Example #29
Source File: TestAMRMClientAsync.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandler()
    throws YarnException, IOException, InterruptedException {
  Configuration conf = new Configuration();
  TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  asyncClient.init(conf);
  asyncClient.start();

  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.notify == false) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
}
 
Example #30
Source File: TestAMRMClient.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=30000)
public void testAskWithNodeLabels() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();

  // add exp=x to ANY
  client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
      1), null, null, Priority.UNDEFINED, true, "x"));
  Assert.assertEquals(1, client.ask.size());
  Assert.assertEquals("x", client.ask.iterator().next()
      .getNodeLabelExpression());

  // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
  client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
      1), null, null, Priority.UNDEFINED, true, "x"));
  client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
      1), null, null, Priority.UNDEFINED, true, "a"));
  Assert.assertEquals(1, client.ask.size());
  Assert.assertEquals("a", client.ask.iterator().next()
      .getNodeLabelExpression());
  
  // add exp=x to ANY, rack and node, only resource request has ANY resource
  // name will be assigned the label expression
  // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
  client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
      1), null, null, Priority.UNDEFINED, true,
      "y"));
  Assert.assertEquals(1, client.ask.size());
  for (ResourceRequest req : client.ask) {
    if (ResourceRequest.ANY.equals(req.getResourceName())) {
      Assert.assertEquals("y", req.getNodeLabelExpression());
    } else {
      Assert.assertNull(req.getNodeLabelExpression());
    }
  }
}