Java Code Examples for org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest

The following examples show how to use org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Bats   Source File: ResourceRequestHandler.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Issue requests to AM RM Client again if previous container requests expired and were not allocated by Yarn
 * @param amRmClient
 * @param requestedResources
 * @param loopCounter
 * @param resourceRequestor
 * @param containerRequests
 * @param removedContainerRequests
 */
public void reissueContainerRequests(AMRMClient<ContainerRequest> amRmClient, Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources, int loopCounter, ResourceRequestHandler resourceRequestor, List<ContainerRequest> containerRequests, List<ContainerRequest> removedContainerRequests)
{
  if (!requestedResources.isEmpty()) {
    for (Map.Entry<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> entry : requestedResources.entrySet()) {
      /*
       * Create container requests again if pending requests were not allocated by Yarn till timeout.
       */
      if ((loopCounter - entry.getValue().getKey()) > NUMBER_MISSED_HEARTBEATS) {
        StreamingContainerAgent.ContainerStartRequest csr = entry.getKey();
        LOG.debug("Request for container {} timed out. Re-requesting container", csr.container);
        removedContainerRequests.add(entry.getValue().getRight());
        ContainerRequest cr = resourceRequestor.createContainerRequest(csr, false);
        entry.getValue().setLeft(loopCounter);
        entry.getValue().setRight(cr);
        containerRequests.add(cr);
      }
    }
  }
}
 
Example 2
Source Project: Bats   Source File: ResourceRequestHandler.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Setup the request(s) that will be sent to the RM for the container ask.
 */
public ContainerRequest createContainerRequest(ContainerStartRequest csr, boolean first)
{
  int priority = csr.container.getResourceRequestPriority();
  // check for node locality constraint
  String[] nodes = null;
  String[] racks = null;

  String host = getHost(csr, first);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(csr.container.getRequiredMemoryMB());
  capability.setVirtualCores(csr.container.getRequiredVCores());
  if (host == INVALID_HOST) {
    return null;
  }
  if (host != null) {
    nodes = new String[]{host};
    // in order to request a host, we don't have to set the rack if the locality is false
    /*
     * if(this.nodeToRack.get(host) != null){ racks = new String[] { this.nodeToRack.get(host) }; }
     */
    return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority), false);
  }
  // For now, only memory is supported so we set memory requirements
  return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority));
}
 
Example 3
Source Project: PoseidonX   Source File: JstormMaster.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * JstormAM 的构造函数
 */
public JstormMaster() throws IOException {
    conf = new YarnConfiguration();
    Path jstormYarnConfPath = new Path("jstorm-yarn.xml");
    conf.addResource(jstormYarnConfPath);

    JstormAMContext.serviceUserName = RegistryUtils.currentUser();
    LOG.error("### JstormAMContext.serviceUserName:["+JstormAMContext.serviceUserName+"]");
    registryClient = RegistryClient.init(JstormAMConstant.REGISTRY_JSTORM_YARN,JstormAMContext.serviceUserName,conf);
    slotPortsView = new SlotPortsView(registryClient);
    slotPortsView.setMinPort(JstormAMConstant.SUPERVISOR_MIN_PORT);
    slotPortsView.setMaxPort(JstormAMConstant.SUPERVISOR_MAX_PORT);
    JstormAMContext.requestBlockingQueue = new LinkedBlockingQueue<ContainerRequest>();

    JstormAMContext.currentNimbusNum = new AtomicInteger(0);
    JstormAMContext.currentSupervisorNum = new AtomicInteger(0);
}
 
Example 4
Source Project: hadoop   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFillInRacks() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
 
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          new String[] {"/rack2"}, Priority.newInstance(1));
  client.addContainerRequest(request);
  verifyResourceRequest(client, request, "host1", true);
  verifyResourceRequest(client, request, "host2", true);
  verifyResourceRequest(client, request, "/rack1", true);
  verifyResourceRequest(client, request, "/rack2", true);
  verifyResourceRequest(client, request, ResourceRequest.ANY, true);
}
 
Example 5
Source Project: hadoop   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test (expected = InvalidContainerRequestException.class)
public void testDifferentLocalityRelaxationSamePriority() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  ContainerRequest request2 =
      new ContainerRequest(capability, new String[] {"host3"},
          null, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
}
 
Example 6
Source Project: hadoop   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test (expected = InvalidContainerRequestException.class)
public void testLocalityRelaxationDifferentLevels() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  ContainerRequest request2 =
      new ContainerRequest(capability, null,
          new String[] {"rack1"}, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
}
 
Example 7
Source Project: hadoop   Source File: TestAMRMClient.java    License: Apache License 2.0 6 votes vote down vote up
private int getAllocatedContainersNumber(
    AMRMClientImpl<ContainerRequest> amClient, int iterationsLeft)
    throws YarnException, IOException {
  int allocatedContainerCount = 0;
  while (iterationsLeft-- > 0) {
    Log.info(" == alloc " + allocatedContainerCount + " it left " + iterationsLeft);
    AllocateResponse allocResponse = amClient.allocate(0.1f);
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
      
    assertEquals(nodeCount, amClient.getClusterNodeCount());
    allocatedContainerCount += allocResponse.getAllocatedContainers().size();
      
    if(allocatedContainerCount == 0) {
      // sleep to let NM's heartbeat to RM and trigger allocations
      sleep(100);
    }
  }
  return allocatedContainerCount;
}
 
Example 8
Source Project: hadoop   Source File: TestAMRMClient.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWaitFor() throws InterruptedException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  CountDownSupplier countDownChecker = new CountDownSupplier();

  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
            .<ContainerRequest> createAMRMClient();
    amClient.init(new YarnConfiguration());
    amClient.start();
    amClient.waitFor(countDownChecker, 1000);
    assertEquals(3, countDownChecker.counter);
  } finally {
    if (amClient != null) {
      amClient.stop();
    }
  }
}
 
Example 9
Source Project: hadoop   Source File: TestAMRMClientAsync.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 10000)
public void testAMRMClientAsyncShutDown() throws Exception {
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  createAllocateResponse(new ArrayList<ContainerStatus>(),
    new ArrayList<Container>(), null);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  asyncClient.registerApplicationMaster("localhost", 1234, null);

  Thread.sleep(50);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 
Example 10
Source Project: hadoop   Source File: ApplicationMaster.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Setup the request that will be sent to the RM for the container ask.
 *
 * @return the setup ResourceRequest to be sent to RM
 */
private ContainerRequest setupContainerAskForRM() {
  // setup requirements for hosts
  // using * as any host will do for the distributed shell app
  // set the priority for the request
  // TODO - what is the range for priority? how to decide?
  Priority pri = Priority.newInstance(requestPriority);

  // Set up resource type requirements
  // For now, memory and CPU are supported so we set memory and cpu requirements
  Resource capability = Resource.newInstance(containerMemory,
    containerVirtualCores, containerGpuCores);

  ContainerRequest request = new ContainerRequest(capability, null, null,
      pri);
  LOG.info("Requested container ask: " + request.toString());
  return request;
}
 
Example 11
Source Project: big-c   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFillInRacks() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
 
  Resource capability = Resource.newInstance(1024, 1);
  ContainerRequest request =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          new String[] {"/rack2"}, Priority.newInstance(1));
  client.addContainerRequest(request);
  verifyResourceRequest(client, request, "host1", true);
  verifyResourceRequest(client, request, "host2", true);
  verifyResourceRequest(client, request, "/rack1", true);
  verifyResourceRequest(client, request, "/rack2", true);
  verifyResourceRequest(client, request, ResourceRequest.ANY, true);
}
 
Example 12
Source Project: big-c   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test (expected = InvalidContainerRequestException.class)
public void testDifferentLocalityRelaxationSamePriority() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  ContainerRequest request2 =
      new ContainerRequest(capability, new String[] {"host3"},
          null, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
}
 
Example 13
Source Project: big-c   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Test (expected = InvalidContainerRequestException.class)
public void testLocalityRelaxationDifferentLevels() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  ContainerRequest request2 =
      new ContainerRequest(capability, null,
          new String[] {"rack1"}, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
}
 
Example 14
Source Project: big-c   Source File: TestAMRMClient.java    License: Apache License 2.0 6 votes vote down vote up
private int getAllocatedContainersNumber(
    AMRMClientImpl<ContainerRequest> amClient, int iterationsLeft)
    throws YarnException, IOException {
  int allocatedContainerCount = 0;
  while (iterationsLeft-- > 0) {
    Log.info(" == alloc " + allocatedContainerCount + " it left " + iterationsLeft);
    AllocateResponse allocResponse = amClient.allocate(0.1f);
    assertEquals(0, amClient.ask.size());
    assertEquals(0, amClient.release.size());
      
    assertEquals(nodeCount, amClient.getClusterNodeCount());
    allocatedContainerCount += allocResponse.getAllocatedContainers().size();
      
    if(allocatedContainerCount == 0) {
      // sleep to let NM's heartbeat to RM and trigger allocations
      sleep(100);
    }
  }
  return allocatedContainerCount;
}
 
Example 15
Source Project: big-c   Source File: TestAMRMClient.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWaitFor() throws InterruptedException {
  AMRMClientImpl<ContainerRequest> amClient = null;
  CountDownSupplier countDownChecker = new CountDownSupplier();

  try {
    // start am rm client
    amClient =
        (AMRMClientImpl<ContainerRequest>) AMRMClient
            .<ContainerRequest> createAMRMClient();
    amClient.init(new YarnConfiguration());
    amClient.start();
    amClient.waitFor(countDownChecker, 1000);
    assertEquals(3, countDownChecker.counter);
  } finally {
    if (amClient != null) {
      amClient.stop();
    }
  }
}
 
Example 16
Source Project: big-c   Source File: TestAMRMClientAsync.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 10000)
public void testAMRMClientAsyncShutDown() throws Exception {
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  createAllocateResponse(new ArrayList<ContainerStatus>(),
    new ArrayList<Container>(), null);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  asyncClient.registerApplicationMaster("localhost", 1234, null);

  Thread.sleep(50);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 
Example 17
Source Project: big-c   Source File: ApplicationMaster.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Setup the request that will be sent to the RM for the container ask.
 *
 * @return the setup ResourceRequest to be sent to RM
 */
private ContainerRequest setupContainerAskForRM() {
  // setup requirements for hosts
  // using * as any host will do for the distributed shell app
  // set the priority for the request
  // TODO - what is the range for priority? how to decide?
  Priority pri = Priority.newInstance(requestPriority);

  // Set up resource type requirements
  // For now, memory and CPU are supported so we set memory and cpu requirements
  Resource capability = Resource.newInstance(containerMemory,
    containerVirtualCores);

  ContainerRequest request = new ContainerRequest(capability, null, null,
      pri);
  LOG.info("Requested container ask: " + request.toString());
  return request;
}
 
Example 18
Source Project: attic-apex-core   Source File: ResourceRequestHandler.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Issue requests to AM RM Client again if previous container requests expired and were not allocated by Yarn
 * @param amRmClient
 * @param requestedResources
 * @param loopCounter
 * @param resourceRequestor
 * @param containerRequests
 * @param removedContainerRequests
 */
public void reissueContainerRequests(AMRMClient<ContainerRequest> amRmClient, Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources, int loopCounter, ResourceRequestHandler resourceRequestor, List<ContainerRequest> containerRequests, List<ContainerRequest> removedContainerRequests)
{
  if (!requestedResources.isEmpty()) {
    for (Map.Entry<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> entry : requestedResources.entrySet()) {
      /*
       * Create container requests again if pending requests were not allocated by Yarn till timeout.
       */
      if ((loopCounter - entry.getValue().getKey()) > NUMBER_MISSED_HEARTBEATS) {
        StreamingContainerAgent.ContainerStartRequest csr = entry.getKey();
        LOG.debug("Request for container {} timed out. Re-requesting container", csr.container);
        removedContainerRequests.add(entry.getValue().getRight());
        ContainerRequest cr = resourceRequestor.createContainerRequest(csr, false);
        entry.getValue().setLeft(loopCounter);
        entry.getValue().setRight(cr);
        containerRequests.add(cr);
      }
    }
  }
}
 
Example 19
Source Project: attic-apex-core   Source File: ResourceRequestHandler.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Setup the request(s) that will be sent to the RM for the container ask.
 */
public ContainerRequest createContainerRequest(ContainerStartRequest csr, boolean first)
{
  int priority = csr.container.getResourceRequestPriority();
  // check for node locality constraint
  String[] nodes = null;
  String[] racks = null;

  String host = getHost(csr, first);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(csr.container.getRequiredMemoryMB());
  capability.setVirtualCores(csr.container.getRequiredVCores());
  if (host == INVALID_HOST) {
    return null;
  }
  if (host != null) {
    nodes = new String[]{host};
    // in order to request a host, we don't have to set the rack if the locality is false
    /*
     * if(this.nodeToRack.get(host) != null){ racks = new String[] { this.nodeToRack.get(host) }; }
     */
    return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority), false);
  }
  // For now, only memory is supported so we set memory requirements
  return new ContainerRequest(capability, nodes, racks, Priority.newInstance(priority));
}
 
Example 20
public void onContainersCompleted(List<ContainerStatus> statuses) {
  LOG.info("onContainersCompleted");
  for (ContainerStatus status : statuses) {
    assert (status.getState() == ContainerState.COMPLETE);

    int exitStatus = status.getExitStatus();
    if (exitStatus != ContainerExitStatus.SUCCESS) {
      if (exitStatus != ContainerExitStatus.ABORTED) {
        failedContainerCount.incrementAndGet();
      }
      allocatedContainerCount.decrementAndGet();
      requestedContainerCount.decrementAndGet();
      recordFailedCommand(status.getContainerId());
    } else {
      completedContainerCount.incrementAndGet();
    }
  }

  int askAgainCount = totalContainerCount - requestedContainerCount.get();
  requestedContainerCount.addAndGet(askAgainCount);

  if (askAgainCount > 0) {
    // need to reallocate failed containers
    for (int i = 0; i < askAgainCount; i++) {
      ContainerRequest req = setupContainerReqForRM();
      resourceManager.addContainerRequest(req);
    }
  }

  if (completedContainerCount.get() == totalContainerCount) {
    done = true;
  }
}
 
Example 21
Source Project: Bats   Source File: BlacklistBasedResourceRequestHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void addContainerRequest(Map<StreamingContainerAgent.ContainerStartRequest, MutablePair<Integer, ContainerRequest>> requestedResources, int loopCounter, List<ContainerRequest> containerRequests, StreamingContainerAgent.ContainerStartRequest csr, ContainerRequest cr)
{
  if (cr.getNodes() != null && !cr.getNodes().isEmpty()) {
    // Put it in a Map to check if multiple requests can be combined
    addHostSpecificRequest(csr, cr);
  } else {
    LOG.info("No node specific request ", cr);
    otherContainerRequests.put(cr, csr);
  }
}
 
Example 22
Source Project: Bats   Source File: BlacklistBasedResourceRequestHandler.java    License: Apache License 2.0 5 votes vote down vote up
private void addHostSpecificRequest(StreamingContainerAgent.ContainerStartRequest csr, ContainerRequest cr)
{
  String hostKey = StringUtils.join(cr.getNodes(), ":");
  List<ContainerRequest> requests;
  if (hostSpecificRequestsMap.containsKey(hostKey)) {
    requests = hostSpecificRequestsMap.get(hostKey);
  } else {
    requests = new ArrayList<>();
  }
  requests.add(cr);
  hostSpecificRequestsMap.put(hostKey, requests);
  LOG.info("Requesting container for node {} request = {}", cr.getNodes(), cr);
  hostSpecificRequests.put(cr, csr);
}
 
Example 23
Source Project: TensorFlowOnYARN   Source File: ApplicationMaster.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Setup the request that will be sent to the RM for the container ask.
 *
 * @return the setup ResourceRequest to be sent to RM
 */
private ContainerRequest setupContainerAskForRM() {
  // Set up resource type requirements
  // For now, memory and CPU are supported so we set memory and cpu requirements
  Resource capability = Resource.newInstance(containerMemory, containerVCores);
  Priority priority = Priority.newInstance(0);

  return new ContainerRequest(capability, null, null, priority);
}
 
Example 24
Source Project: hadoop   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInvalidValidWhenOldRemoved() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();
  Configuration conf = new Configuration();
  conf.setClass(
      CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
      MyResolver.class, DNSToSwitchMapping.class);
  client.init(conf);
  
  Resource capability = Resource.newInstance(1024, 1, 1);
  ContainerRequest request1 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request1);
  
  client.removeContainerRequest(request1);

  ContainerRequest request2 =
      new ContainerRequest(capability, new String[] {"host3"},
          null, Priority.newInstance(1), true);
  client.addContainerRequest(request2);
  
  client.removeContainerRequest(request2);
  
  ContainerRequest request3 =
      new ContainerRequest(capability, new String[] {"host1", "host2"},
          null, Priority.newInstance(1), false);
  client.addContainerRequest(request3);
  
  client.removeContainerRequest(request3);
  
  ContainerRequest request4 =
      new ContainerRequest(capability, null,
          new String[] {"rack1"}, Priority.newInstance(1), true);
  client.addContainerRequest(request4);

}
 
Example 25
Source Project: hadoop   Source File: TestAMRMClientContainerRequest.java    License: Apache License 2.0 5 votes vote down vote up
private void verifyResourceRequest(
    AMRMClientImpl<ContainerRequest> client, ContainerRequest request,
    String location, boolean expectedRelaxLocality) {
  ResourceRequest ask =  client.remoteRequestsTable.get(request.getPriority())
      .get(location).get(request.getCapability()).remoteRequest;
  assertEquals(location, ask.getResourceName());
  assertEquals(1, ask.getNumContainers());
  assertEquals(expectedRelaxLocality, ask.getRelaxLocality());
}
 
Example 26
Source Project: hadoop   Source File: TestAMRMClient.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout=60000)
public void testAMRMClient() throws YarnException, IOException {
  AMRMClient<ContainerRequest> amClient = null;
  try {
    // start am rm client
    amClient = AMRMClient.<ContainerRequest>createAMRMClient();

    //setting an instance NMTokenCache
    amClient.setNMTokenCache(new NMTokenCache());
    //asserting we are not using the singleton instance cache
    Assert.assertNotSame(NMTokenCache.getSingleton(), 
        amClient.getNMTokenCache());

    amClient.init(conf);
    amClient.start();

    amClient.registerApplicationMaster("Host", 10000, "");

    testAllocation((AMRMClientImpl<ContainerRequest>)amClient);

    amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
        null, null);

  } finally {
    if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
      amClient.stop();
    }
  }
}
 
Example 27
Source Project: hadoop   Source File: TestAMRMClient.java    License: Apache License 2.0 5 votes vote down vote up
private void verifyAddRequestFailed(AMRMClient<ContainerRequest> client,
    ContainerRequest request) {
  try {
    client.addContainerRequest(request);
  } catch (InvalidContainerRequestException e) {
    return;
  }
  Assert.fail();
}
 
Example 28
Source Project: hadoop   Source File: TestAMRMClient.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=30000)
public void testAskWithInvalidNodeLabels() {
  AMRMClientImpl<ContainerRequest> client =
      new AMRMClientImpl<ContainerRequest>();

  // specified exp with more than one node labels
  verifyAddRequestFailed(client,
      new ContainerRequest(Resource.newInstance(1024, 1), null, null,
          Priority.UNDEFINED, true, "x && y"));
}
 
Example 29
Source Project: hadoop   Source File: TestAMRMClientAsync.java    License: Apache License 2.0 5 votes vote down vote up
private void runHeartBeatThrowOutException(Exception ex) throws Exception{
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
  when(client.allocate(anyFloat())).thenThrow(ex);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();
  
  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.savedException == null) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
  Assert.assertTrue(callbackHandler.savedException.getMessage().contains(
      ex.getMessage()));
  
  asyncClient.stop();
  // stopping should have joined all threads and completed all callbacks
  Assert.assertTrue(callbackHandler.callbackCount == 0);
}
 
Example 30
Source Project: hadoop   Source File: TestAMRMClientAsync.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 10000)
public void testAMRMClientAsyncShutDownWithWaitFor() throws Exception {
  Configuration conf = new Configuration();
  final TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  Supplier<Boolean> checker = new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return callbackHandler.reboot;
    }
  };

  asyncClient.registerApplicationMaster("localhost", 1234, null);
  asyncClient.waitFor(checker);

  asyncClient.stop();
  // stopping should have joined all threads and completed all callbacks
  Assert.assertTrue(callbackHandler.callbackCount == 0);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}