Java Code Examples for org.apache.hadoop.yarn.server.utils.BuilderUtils#newNodeId()

The following examples show how to use org.apache.hadoop.yarn.server.utils.BuilderUtils#newNodeId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: MockNM.java From big-c with Apache License 2.0

5 votes

public MockNM(String nodeIdStr, int memory, int vcores,
    ResourceTrackerService resourceTracker, String version) {
  this.memory = memory;
  this.vCores = vcores;
  this.resourceTracker = resourceTracker;
  this.version = version;
  String[] splits = nodeIdStr.split(":");
  nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1]));
}

Example 2

Source File: TestRMNodeTransitions.java From big-c with Apache License 2.0

5 votes

private RMNodeImpl getRebootedNode() {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  Resource capability = Resource.newInstance(4096, 4);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
      null, capability, null);
  node.handle(new RMNodeStartedEvent(node.getNodeID(), null, null));
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.REBOOTING));
  Assert.assertEquals(NodeState.REBOOTED, node.getState());
  return node;
}

Example 3

Source File: TestRMNodeTransitions.java From hadoop with Apache License 2.0

5 votes

private RMNodeImpl getRunningNode(String nmVersion) {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  Resource capability = Resource.newInstance(4096, 4, 4);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
      null, capability, nmVersion);
  node.handle(new RMNodeStartedEvent(node.getNodeID(), null, null));
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  return node;
}

Example 4

Source File: TestRMNodeTransitions.java From hadoop with Apache License 2.0

5 votes

private RMNodeImpl getRebootedNode() {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  Resource capability = Resource.newInstance(4096, 4, 4);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
      null, capability, null);
  node.handle(new RMNodeStartedEvent(node.getNodeID(), null, null));
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  node.handle(new RMNodeEvent(node.getNodeID(), RMNodeEventType.REBOOTING));
  Assert.assertEquals(NodeState.REBOOTED, node.getState());
  return node;
}

Example 5

Source File: MockNM.java From hadoop with Apache License 2.0

5 votes

public MockNM(String nodeIdStr, int memory, int vcores, int gcores,
    ResourceTrackerService resourceTracker, String version) {
  this.memory = memory;
  this.vCores = vcores;
  this.gCores = gcores;
  this.resourceTracker = resourceTracker;
  this.version = version;
  String[] splits = nodeIdStr.split(":");
  nodeId = BuilderUtils.newNodeId(splits[0], Integer.parseInt(splits[1]));
}

Example 6

Source File: TestRMNodeTransitions.java From big-c with Apache License 2.0

5 votes

private RMNodeImpl getRunningNode(String nmVersion) {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  Resource capability = Resource.newInstance(4096, 4);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext,null, 0, 0,
      null, capability, nmVersion);
  node.handle(new RMNodeStartedEvent(node.getNodeID(), null, null));
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  return node;
}

Example 7

Source File: TestRMContainerImpl.java From big-c with Apache License 2.0

4 votes

@Test
public void testReleaseWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);
  ConcurrentMap<ApplicationId, RMApp> rmApps =
      spy(new ConcurrentHashMap<ApplicationId, RMApp>());
  RMApp rmApp = mock(RMApp.class);
  when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
  Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any());

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getRMApps()).thenReturn(rmApps);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify RELEASED and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.RELEASED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.RELEASED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
  assertEquals(SchedulerUtils.RELEASED_CONTAINER,
      rmContainer.getDiagnosticsInfo());
  assertEquals(ContainerExitStatus.ABORTED,
      rmContainer.getContainerExitStatus());
  assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState());
  verify(writer).containerFinished(any(RMContainer.class));
  verify(publisher).containerFinished(any(RMContainer.class), anyLong());

  ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor
      .forClass(RMAppAttemptContainerFinishedEvent.class);
  verify(appAttemptEventHandler).handle(captor.capture());
  RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue();
  assertEquals(appAttemptId, cfEvent.getApplicationAttemptId());
  assertEquals(containerStatus, cfEvent.getContainerStatus());
  assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType());
  
  // In RELEASED state. A FINIHSED event may come in.
  rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils
      .createAbnormalContainerStatus(containerId, "FinishedContainer"),
      RMContainerEventType.FINISHED));
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
}

Example 8

Source File: TestResourceTrackerService.java From big-c with Apache License 2.0

4 votes

@Test
public void testNodeRegistrationWithMinimumAllocations() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4");
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService
    = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = BuilderUtils.newNodeId("host", 1234);
  req.setNodeId(nodeId);

  Resource capability = BuilderUtils.newResource(1024, 1);
  req.setResource(capability);
  RegisterNodeManagerResponse response1 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(1);
  req.setResource(capability);
  RegisterNodeManagerResponse response2 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction());
  
  capability.setMemory(1024);
  capability.setVirtualCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response3 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response4 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response4.getNodeAction());
}

Example 9

Source File: TestRMNodeTransitions.java From big-c with Apache License 2.0

4 votes

private RMNodeImpl getNewNode(Resource capability) {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, 
      capability, null);
  return node;
}

Example 10

Source File: TestRMNodeTransitions.java From big-c with Apache License 2.0

4 votes

private RMNodeImpl getNewNode() {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
  return node;
}

Example 11

Source File: TestRMNodeTransitions.java From big-c with Apache License 2.0

4 votes

@Test (timeout = 5000)
public void testContainerUpdate() throws InterruptedException{
  //Start the node
  node.handle(new RMNodeStartedEvent(null, null, null));
  
  NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
  RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
  node2.handle(new RMNodeStartedEvent(null, null, null));
  
  ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(0, 0), 0), 0);
  ContainerId completedContainerIdFromNode2_1 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(1, 1), 1), 1);
  ContainerId completedContainerIdFromNode2_2 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(1, 1), 1), 2);
 
  RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent();
  RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent();
  RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent();
  
  ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class);
  ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class);
  ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class);

  doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1)
      .getContainerId();
  doReturn(Collections.singletonList(containerStatusFromNode1))
      .when(statusEventFromNode1).getContainers();
  node.handle(statusEventFromNode1);
  Assert.assertEquals(1, completedContainers.size());
  Assert.assertEquals(completedContainerIdFromNode1,
      completedContainers.get(0).getContainerId());

  completedContainers.clear();

  doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1)
      .getContainerId();
  doReturn(Collections.singletonList(containerStatusFromNode2_1))
      .when(statusEventFromNode2_1).getContainers();

  doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2)
      .getContainerId();
  doReturn(Collections.singletonList(containerStatusFromNode2_2))
      .when(statusEventFromNode2_2).getContainers();

  node2.setNextHeartBeat(false);
  node2.handle(statusEventFromNode2_1);
  node2.setNextHeartBeat(true);
  node2.handle(statusEventFromNode2_2);

  Assert.assertEquals(2, completedContainers.size());
  Assert.assertEquals(completedContainerIdFromNode2_1,completedContainers.get(0)
      .getContainerId()); 
  Assert.assertEquals(completedContainerIdFromNode2_2,completedContainers.get(1)
      .getContainerId());   
}

Example 12

Source File: TestRMNodeTransitions.java From big-c with Apache License 2.0

4 votes

@Before
public void setUp() throws Exception {
  InlineDispatcher rmDispatcher = new InlineDispatcher();
  
  rmContext =
      new RMContextImpl(rmDispatcher, null, null, null,
          mock(DelegationTokenRenewer.class), null, null, null, null, null);
  NodesListManager nodesListManager = mock(NodesListManager.class);
  HostsFileReader reader = mock(HostsFileReader.class);
  when(nodesListManager.getHostsReader()).thenReturn(reader);
  ((RMContextImpl) rmContext).setNodesListManager(nodesListManager);
  scheduler = mock(YarnScheduler.class);
  doAnswer(
      new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
          final SchedulerEvent event = (SchedulerEvent)(invocation.getArguments()[0]);
          eventType = event.getType();
          if (eventType == SchedulerEventType.NODE_UPDATE) {
            List<UpdatedContainerInfo> lastestContainersInfoList = 
                ((NodeUpdateSchedulerEvent)event).getRMNode().pullContainerUpdates();
            for(UpdatedContainerInfo lastestContainersInfo : lastestContainersInfoList) {
          	  completedContainers.addAll(lastestContainersInfo.getCompletedContainers()); 
            }
          }
          return null;
        }
      }
      ).when(scheduler).handle(any(SchedulerEvent.class));
  
  rmDispatcher.register(SchedulerEventType.class, 
      new TestSchedulerEventDispatcher());
  
  rmDispatcher.register(NodesListManagerEventType.class,
      new TestNodeListManagerEventDispatcher());

  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
  nodesListManagerEvent =  null;

}

Example 13

Source File: TestRMContainerImpl.java From big-c with Apache License 2.0

4 votes

@Test
public void testExpireWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify EXPIRE and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.EXPIRED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.EXPIRE));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  verify(writer, never()).containerFinished(any(RMContainer.class));
  verify(publisher, never()).containerFinished(any(RMContainer.class),
      anyLong());
}

Example 14

Source File: TestRMContainerImpl.java From hadoop with Apache License 2.0

4 votes

@Test
public void testReleaseWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);
  ConcurrentMap<ApplicationId, RMApp> rmApps =
      spy(new ConcurrentHashMap<ApplicationId, RMApp>());
  RMApp rmApp = mock(RMApp.class);
  when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
  Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any());

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getRMApps()).thenReturn(rmApps);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify RELEASED and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.RELEASED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.RELEASED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
  assertEquals(SchedulerUtils.RELEASED_CONTAINER,
      rmContainer.getDiagnosticsInfo());
  assertEquals(ContainerExitStatus.ABORTED,
      rmContainer.getContainerExitStatus());
  assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState());
  verify(writer).containerFinished(any(RMContainer.class));
  verify(publisher).containerFinished(any(RMContainer.class), anyLong());

  ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor
      .forClass(RMAppAttemptContainerFinishedEvent.class);
  verify(appAttemptEventHandler).handle(captor.capture());
  RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue();
  assertEquals(appAttemptId, cfEvent.getApplicationAttemptId());
  assertEquals(containerStatus, cfEvent.getContainerStatus());
  assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType());
  
  // In RELEASED state. A FINIHSED event may come in.
  rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils
      .createAbnormalContainerStatus(containerId, "FinishedContainer"),
      RMContainerEventType.FINISHED));
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
}

Example 15

Source File: TestResourceTrackerService.java From hadoop with Apache License 2.0

4 votes

@Test
public void testNodeRegistrationWithMinimumAllocations() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, "2048");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, "4");
  conf.set(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_GCORES, "4");
  rm = new MockRM(conf);
  rm.start();

  ResourceTrackerService resourceTrackerService
    = rm.getResourceTrackerService();
  RegisterNodeManagerRequest req = Records.newRecord(
      RegisterNodeManagerRequest.class);
  NodeId nodeId = BuilderUtils.newNodeId("host", 1234);
  req.setNodeId(nodeId);

  Resource capability = BuilderUtils.newResource(1024, 1, 1);
  req.setResource(capability);
  RegisterNodeManagerResponse response1 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response1.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(1);
  capability.setGpuCores(1);
  req.setResource(capability);
  RegisterNodeManagerResponse response2 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response2.getNodeAction());
  
  capability.setMemory(1024);
  capability.setVirtualCores(4);
  capability.setGpuCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response3 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.SHUTDOWN,response3.getNodeAction());
  
  capability.setMemory(2048);
  capability.setVirtualCores(4);
  capability.setGpuCores(4);
  req.setResource(capability);
  RegisterNodeManagerResponse response4 =
      resourceTrackerService.registerNodeManager(req);
  Assert.assertEquals(NodeAction.NORMAL,response4.getNodeAction());
}

Example 16

Source File: TestRMNodeTransitions.java From hadoop with Apache License 2.0

4 votes

private RMNodeImpl getNewNode(Resource capability) {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, 
      capability, null);
  return node;
}

Example 17

Source File: TestRMNodeTransitions.java From hadoop with Apache License 2.0

4 votes

private RMNodeImpl getNewNode() {
  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  RMNodeImpl node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
  return node;
}

Example 18

Source File: TestRMNodeTransitions.java From hadoop with Apache License 2.0

4 votes

@Test (timeout = 5000)
public void testContainerUpdate() throws InterruptedException{
  //Start the node
  node.handle(new RMNodeStartedEvent(null, null, null));
  
  NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
  RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
  node2.handle(new RMNodeStartedEvent(null, null, null));
  
  ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(0, 0), 0), 0);
  ContainerId completedContainerIdFromNode2_1 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(1, 1), 1), 1);
  ContainerId completedContainerIdFromNode2_2 = BuilderUtils.newContainerId(
      BuilderUtils.newApplicationAttemptId(
          BuilderUtils.newApplicationId(1, 1), 1), 2);
 
  RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent();
  RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent();
  RMNodeStatusEvent statusEventFromNode2_2 = getMockRMNodeStatusEvent();
  
  ContainerStatus containerStatusFromNode1 = mock(ContainerStatus.class);
  ContainerStatus containerStatusFromNode2_1 = mock(ContainerStatus.class);
  ContainerStatus containerStatusFromNode2_2 = mock(ContainerStatus.class);

  doReturn(completedContainerIdFromNode1).when(containerStatusFromNode1)
      .getContainerId();
  doReturn(Collections.singletonList(containerStatusFromNode1))
      .when(statusEventFromNode1).getContainers();
  node.handle(statusEventFromNode1);
  Assert.assertEquals(1, completedContainers.size());
  Assert.assertEquals(completedContainerIdFromNode1,
      completedContainers.get(0).getContainerId());

  completedContainers.clear();

  doReturn(completedContainerIdFromNode2_1).when(containerStatusFromNode2_1)
      .getContainerId();
  doReturn(Collections.singletonList(containerStatusFromNode2_1))
      .when(statusEventFromNode2_1).getContainers();

  doReturn(completedContainerIdFromNode2_2).when(containerStatusFromNode2_2)
      .getContainerId();
  doReturn(Collections.singletonList(containerStatusFromNode2_2))
      .when(statusEventFromNode2_2).getContainers();

  node2.setNextHeartBeat(false);
  node2.handle(statusEventFromNode2_1);
  node2.setNextHeartBeat(true);
  node2.handle(statusEventFromNode2_2);

  Assert.assertEquals(2, completedContainers.size());
  Assert.assertEquals(completedContainerIdFromNode2_1,completedContainers.get(0)
      .getContainerId()); 
  Assert.assertEquals(completedContainerIdFromNode2_2,completedContainers.get(1)
      .getContainerId());   
}

Example 19

Source File: TestRMNodeTransitions.java From hadoop with Apache License 2.0

4 votes

@Before
public void setUp() throws Exception {
  InlineDispatcher rmDispatcher = new InlineDispatcher();
  
  rmContext =
      new RMContextImpl(rmDispatcher, null, null, null,
          mock(DelegationTokenRenewer.class), null, null, null, null, null);
  NodesListManager nodesListManager = mock(NodesListManager.class);
  HostsFileReader reader = mock(HostsFileReader.class);
  when(nodesListManager.getHostsReader()).thenReturn(reader);
  ((RMContextImpl) rmContext).setNodesListManager(nodesListManager);
  scheduler = mock(YarnScheduler.class);
  doAnswer(
      new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
          final SchedulerEvent event = (SchedulerEvent)(invocation.getArguments()[0]);
          eventType = event.getType();
          if (eventType == SchedulerEventType.NODE_UPDATE) {
            List<UpdatedContainerInfo> lastestContainersInfoList = 
                ((NodeUpdateSchedulerEvent)event).getRMNode().pullContainerUpdates();
            for(UpdatedContainerInfo lastestContainersInfo : lastestContainersInfoList) {
          	  completedContainers.addAll(lastestContainersInfo.getCompletedContainers()); 
            }
          }
          return null;
        }
      }
      ).when(scheduler).handle(any(SchedulerEvent.class));
  
  rmDispatcher.register(SchedulerEventType.class, 
      new TestSchedulerEventDispatcher());
  
  rmDispatcher.register(NodesListManagerEventType.class,
      new TestNodeListManagerEventDispatcher());

  NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
  node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
  nodesListManagerEvent =  null;

}

Example 20

Source File: TestRMContainerImpl.java From hadoop with Apache License 2.0

4 votes

@Test
public void testExpireWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify EXPIRE and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.EXPIRED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.EXPIRE));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  verify(writer, never()).containerFinished(any(RMContainer.class));
  verify(publisher, never()).containerFinished(any(RMContainer.class),
      anyLong());
}