Java Code Examples for org.apache.hadoop.yarn.api.records.ResourceOption

The following examples show how to use org.apache.hadoop.yarn.api.records.ResourceOption. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnRunningNode() {
  RMNodeImpl node = getRunningNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2, 2),
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  assertEquals("GPU resource is not match.", newCapacity.getGpuCores(), 2);
  
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  Assert.assertNotNull(nodesListManagerEvent);
  Assert.assertEquals(NodesListManagerEventType.NODE_USABLE,
      nodesListManagerEvent.getType());
}
 
Example 2
Source Project: hadoop   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnNewNode() {
  RMNodeImpl node = getNewNode(Resource.newInstance(4096, 4, 4));
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2, 2),
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  assertEquals("GPU resource is not match.", newCapacity.getGpuCores(), 2);
  
  Assert.assertEquals(NodeState.NEW, node.getState());
}
 
Example 3
Source Project: hadoop   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnRebootedNode() {
  RMNodeImpl node = getRebootedNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2, 2),
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  assertEquals("GPU resource is not match.", newCapacity.getGpuCores(), 2);
  
  Assert.assertEquals(NodeState.REBOOTED, node.getState());
}
 
Example 4
Source Project: big-c   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnRunningNode() {
  RMNodeImpl node = getRunningNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2), 
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  Assert.assertNotNull(nodesListManagerEvent);
  Assert.assertEquals(NodesListManagerEventType.NODE_USABLE,
      nodesListManagerEvent.getType());
}
 
Example 5
Source Project: hadoop   Source File: UpdateNodeResourceRequestPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setNodeResourceMap(Map<NodeId, ResourceOption> nodeResourceMap) {
  if (nodeResourceMap == null) {
    return;
  }
  initNodeResourceMap();
  this.nodeResourceMap.clear();
  this.nodeResourceMap.putAll(nodeResourceMap);
}
 
Example 6
Source Project: hadoop   Source File: UpdateNodeResourceRequestPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void initNodeResourceMap() {
  if (this.nodeResourceMap != null) {
    return;
  }
  UpdateNodeResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
  List<NodeResourceMapProto> list = p.getNodeResourceMapList();
  this.nodeResourceMap = new HashMap<NodeId, ResourceOption>(list
      .size());
  for (NodeResourceMapProto nodeResourceProto : list) {
    this.nodeResourceMap.put(convertFromProtoFormat(nodeResourceProto.getNodeId()), 
        convertFromProtoFormat(nodeResourceProto.getResourceOption()));
  }
}
 
Example 7
Source Project: hadoop   Source File: FairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node and update Queue.
 */
@Override
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  super.updateNodeResource(nm, resourceOption);
  updateRootQueueMetrics();
  queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
  queueMgr.getRootQueue().recomputeSteadyShares();
}
 
Example 8
Source Project: hadoop   Source File: AbstractYarnScheduler.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node.
 */
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  SchedulerNode node = getSchedulerNode(nm.getNodeID());
  Resource newResource = resourceOption.getResource();
  Resource oldResource = node.getTotalResource();
  if(!oldResource.equals(newResource)) {
    // Notify NodeLabelsManager about this change
    rmContext.getNodeLabelManager().updateNodeResource(nm.getNodeID(),
        newResource);

    // Log resource change
    LOG.info("Update resource on node: " + node.getNodeName()
        + " from: " + oldResource + ", to: "
        + newResource);

    nodes.remove(nm.getNodeID());
    updateMaximumAllocation(node, false);

    // update resource to node
    node.setTotalResource(newResource);

    nodes.put(nm.getNodeID(), (N)node);
    updateMaximumAllocation(node, true);

    // update resource to clusterResource
    Resources.subtractFrom(clusterResource, oldResource);
    Resources.addTo(clusterResource, newResource);
  } else {
    // Log resource change
    LOG.warn("Update resource on node: " + node.getNodeName() 
        + " with the same resource: " + newResource);
  }
}
 
Example 9
Source Project: hadoop   Source File: CapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node.
 */
private synchronized void updateNodeAndQueueResource(RMNode nm, 
    ResourceOption resourceOption) {
  updateNodeResource(nm, resourceOption);
  root.updateClusterResource(clusterResource, new ResourceLimits(
      clusterResource));
}
 
Example 10
Source Project: hadoop   Source File: UpdateNodeResourceRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Evolving
public static UpdateNodeResourceRequest newInstance(
    Map<NodeId, ResourceOption> nodeResourceMap) {
  UpdateNodeResourceRequest request =
      Records.newRecord(UpdateNodeResourceRequest.class);
  request.setNodeResourceMap(nodeResourceMap);
  return request;
}
 
Example 11
Source Project: big-c   Source File: UpdateNodeResourceRequestPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setNodeResourceMap(Map<NodeId, ResourceOption> nodeResourceMap) {
  if (nodeResourceMap == null) {
    return;
  }
  initNodeResourceMap();
  this.nodeResourceMap.clear();
  this.nodeResourceMap.putAll(nodeResourceMap);
}
 
Example 12
Source Project: big-c   Source File: UpdateNodeResourceRequestPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void initNodeResourceMap() {
  if (this.nodeResourceMap != null) {
    return;
  }
  UpdateNodeResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
  List<NodeResourceMapProto> list = p.getNodeResourceMapList();
  this.nodeResourceMap = new HashMap<NodeId, ResourceOption>(list
      .size());
  for (NodeResourceMapProto nodeResourceProto : list) {
    this.nodeResourceMap.put(convertFromProtoFormat(nodeResourceProto.getNodeId()), 
        convertFromProtoFormat(nodeResourceProto.getResourceOption()));
  }
}
 
Example 13
Source Project: big-c   Source File: RMNodeImpl.java    License: Apache License 2.0 5 votes vote down vote up
private static void updateNodeResourceFromEvent(RMNodeImpl rmNode, 
   RMNodeResourceUpdateEvent event){
    ResourceOption resourceOption = event.getResourceOption();
    // Set resource on RMNode
    rmNode.totalCapability = resourceOption.getResource();
    
}
 
Example 14
Source Project: big-c   Source File: FairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node and update Queue.
 */
@Override
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  super.updateNodeResource(nm, resourceOption);
  updateRootQueueMetrics();
  queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
  queueMgr.getRootQueue().recomputeSteadyShares();
}
 
Example 15
Source Project: big-c   Source File: AbstractYarnScheduler.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node.
 */
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  SchedulerNode node = getSchedulerNode(nm.getNodeID());
  Resource newResource = resourceOption.getResource();
  Resource oldResource = node.getTotalResource();
  if(!oldResource.equals(newResource)) {
    // Log resource change
    LOG.info("Update resource on node: " + node.getNodeName()
        + " from: " + oldResource + ", to: "
        + newResource);

    nodes.remove(nm.getNodeID());
    updateMaximumAllocation(node, false);

    // update resource to node
    node.setTotalResource(newResource);

    nodes.put(nm.getNodeID(), (N)node);
    updateMaximumAllocation(node, true);

    // update resource to clusterResource
    Resources.subtractFrom(clusterResource, oldResource);
    Resources.addTo(clusterResource, newResource);
  } else {
    // Log resource change
    LOG.warn("Update resource on node: " + node.getNodeName() 
        + " with the same resource: " + newResource);
  }
}
 
Example 16
Source Project: big-c   Source File: CapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node.
 */
private synchronized void updateNodeAndQueueResource(RMNode nm, 
    ResourceOption resourceOption) {
  updateNodeResource(nm, resourceOption);
  root.updateClusterResource(clusterResource, new ResourceLimits(
      clusterResource));
}
 
Example 17
Source Project: big-c   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testResourceUpdateOnNewNode() {
  RMNodeImpl node = getNewNode(Resource.newInstance(4096, 4));
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2), 
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  
  Assert.assertEquals(NodeState.NEW, node.getState());
}
 
Example 18
Source Project: big-c   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testResourceUpdateOnRebootedNode() {
  RMNodeImpl node = getRebootedNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2), 
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  
  Assert.assertEquals(NodeState.REBOOTED, node.getState());
}
 
Example 19
Source Project: big-c   Source File: UpdateNodeResourceRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Evolving
public static UpdateNodeResourceRequest newInstance(
    Map<NodeId, ResourceOption> nodeResourceMap) {
  UpdateNodeResourceRequest request =
      Records.newRecord(UpdateNodeResourceRequest.class);
  request.setNodeResourceMap(nodeResourceMap);
  return request;
}
 
Example 20
Source Project: incubator-myriad   Source File: YarnNodeCapacityManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 1. Updates {@link RMNode#getTotalCapability()} with newCapacity.
 * 2. Sends out a {@link NodeResourceUpdateSchedulerEvent} that's handled by YARN's scheduler.
 * The scheduler updates the corresponding {@link SchedulerNode} with the newCapacity.
 *
 * @param rmNode
 * @param newCapacity
 */
@SuppressWarnings("unchecked")
public void setNodeCapacity(RMNode rmNode, Resource newCapacity) {
  //NOOP prevent YARN warning changing to same size
  if ((Resources.equals(rmNode.getTotalCapability(), newCapacity))) {
    return;
  }
  if (yarnScheduler.getSchedulerNode(rmNode.getNodeID()) == null) {
    LOGGER.info("Yarn Scheduler doesn't have node {}, probably UNHEALTHY", rmNode.getNodeID());
    return;
  }
  yarnSchedulerLock.lock();
  try {
    if (newCapacity.getMemory() < 0 || newCapacity.getVirtualCores() < 0) {
      Resource zeroed = ResourceUtils.componentwiseMax(ZERO_RESOURCE, newCapacity);
      rmNode.getTotalCapability().setMemory(zeroed.getMemory());
      rmNode.getTotalCapability().setVirtualCores(zeroed.getVirtualCores());
      LOGGER.warn("Asked to set Node {} to a value less than zero!  Had {}, setting to {}.",
          rmNode.getHttpAddress(), rmNode.getTotalCapability().toString(), zeroed.toString());
    } else {
      rmNode.getTotalCapability().setMemory(newCapacity.getMemory());
      rmNode.getTotalCapability().setVirtualCores(newCapacity.getVirtualCores());
      if (LOGGER.isInfoEnabled()) {
        LOGGER.info("Setting capacity for node {} to {}", rmNode.getHostName(), newCapacity);
      }
    }
    // updates the scheduler with the new capacity for the NM.
    // the event is handled by the scheduler asynchronously
    rmContext.getDispatcher().getEventHandler().handle(new NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption.newInstance(
        rmNode.getTotalCapability(), RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  } finally {
    yarnSchedulerLock.unlock();
  }
}
 
Example 21
Source Project: hadoop   Source File: UpdateNodeResourceRequestPBImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Map<NodeId, ResourceOption> getNodeResourceMap() {
  initNodeResourceMap();
  return this.nodeResourceMap;
}
 
Example 22
Source Project: hadoop   Source File: UpdateNodeResourceRequestPBImpl.java    License: Apache License 2.0 4 votes vote down vote up
private ResourceOptionProto convertToProtoFormat(ResourceOption c) {
  return ((ResourceOptionPBImpl)c).getProto();
}
 
Example 23
Source Project: hadoop   Source File: RMNodeResourceUpdateEvent.java    License: Apache License 2.0 4 votes vote down vote up
public RMNodeResourceUpdateEvent(NodeId nodeId, ResourceOption resourceOption) {
  super(nodeId, RMNodeEventType.RESOURCE_UPDATE);
  this.resourceOption = resourceOption;
}
 
Example 24
Source Project: hadoop   Source File: RMNodeResourceUpdateEvent.java    License: Apache License 2.0 4 votes vote down vote up
public ResourceOption getResourceOption() {
  return resourceOption;
}
 
Example 25
Source Project: hadoop   Source File: RMNodeImpl.java    License: Apache License 2.0 4 votes vote down vote up
private static void updateNodeResourceFromEvent(RMNodeImpl rmNode, 
   RMNodeResourceUpdateEvent event){
    ResourceOption resourceOption = event.getResourceOption();
    // Set resource on RMNode
    rmNode.totalCapability = resourceOption.getResource();
}
 
Example 26
Source Project: hadoop   Source File: RMNodeImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void transition(RMNodeImpl rmNode, RMNodeEvent event) {
  RMNodeReconnectEvent reconnectEvent = (RMNodeReconnectEvent) event;
  RMNode newNode = reconnectEvent.getReconnectedNode();
  rmNode.nodeManagerVersion = newNode.getNodeManagerVersion();
  List<ApplicationId> runningApps = reconnectEvent.getRunningApplications();
  boolean noRunningApps = 
      (runningApps == null) || (runningApps.size() == 0);
  
  // No application running on the node, so send node-removal event with 
  // cleaning up old container info.
  if (noRunningApps) {
    rmNode.nodeUpdateQueue.clear();
    rmNode.context.getDispatcher().getEventHandler().handle(
        new NodeRemovedSchedulerEvent(rmNode));

    if (rmNode.getHttpPort() == newNode.getHttpPort()) {
      // Reset heartbeat ID since node just restarted.
      rmNode.getLastNodeHeartBeatResponse().setResponseId(0);
      if (rmNode.getState().equals(NodeState.RUNNING)) {
        // Only add new node if old state is RUNNING
        rmNode.context.getDispatcher().getEventHandler().handle(
            new NodeAddedSchedulerEvent(newNode));
      }
    } else {
      // Reconnected node differs, so replace old node and start new node
      switch (rmNode.getState()) {
        case RUNNING:
          ClusterMetrics.getMetrics().decrNumActiveNodes();
          break;
        case UNHEALTHY:
          ClusterMetrics.getMetrics().decrNumUnhealthyNMs();
          break;
        default:
          LOG.debug("Unexpected Rmnode state");
        }
        rmNode.context.getRMNodes().put(newNode.getNodeID(), newNode);
        rmNode.context.getDispatcher().getEventHandler().handle(
            new RMNodeStartedEvent(newNode.getNodeID(), null, null));
    }
  } else {
    rmNode.httpPort = newNode.getHttpPort();
    rmNode.httpAddress = newNode.getHttpAddress();
    boolean isCapabilityChanged = false;
    if (rmNode.getTotalCapability() != newNode.getTotalCapability()) {
      rmNode.totalCapability = newNode.getTotalCapability();
      isCapabilityChanged = true;
    }
  
    handleNMContainerStatus(reconnectEvent.getNMContainerStatuses(), rmNode);

    // Reset heartbeat ID since node just restarted.
    rmNode.getLastNodeHeartBeatResponse().setResponseId(0);

    for (ApplicationId appId : reconnectEvent.getRunningApplications()) {
      handleRunningAppOnNode(rmNode, rmNode.context, appId, rmNode.nodeId);
    }

    if (isCapabilityChanged
        && rmNode.getState().equals(NodeState.RUNNING)) {
      // Update scheduler node's capacity for reconnect node.
      rmNode.context
          .getDispatcher()
          .getEventHandler()
          .handle(
              new NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption
                  .newInstance(newNode.getTotalCapability(), -1)));
    }
  }
}
 
Example 27
Source Project: hadoop   Source File: NodeResourceUpdateSchedulerEvent.java    License: Apache License 2.0 4 votes vote down vote up
public NodeResourceUpdateSchedulerEvent(RMNode rmNode,
    ResourceOption resourceOption) {
  super(SchedulerEventType.NODE_RESOURCE_UPDATE);
  this.rmNode = rmNode;
  this.resourceOption = resourceOption;
}
 
Example 28
Source Project: hadoop   Source File: NodeResourceUpdateSchedulerEvent.java    License: Apache License 2.0 4 votes vote down vote up
public ResourceOption getResourceOption() {
  return resourceOption;
}
 
Example 29
Source Project: hadoop   Source File: TestFifoScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testResourceOverCommit() throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();
  
  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB);
  
  RMApp app1 = rm.submitApp(2048);
  // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
  am1.registerAppAttempt();
  SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
      nm1.getNodeId());
  // check node report, 2 GB used and 2 GB available
  Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory());
  Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemory());

  // add request for containers
  am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
  AllocateResponse alloc1Response = am1.schedule(); // send the request

  // kick the scheduler, 2 GB given to AM1, resource remaining 0
  nm1.nodeHeartbeat(true);
  while (alloc1Response.getAllocatedContainers().size() < 1) {
    LOG.info("Waiting for containers to be created for app 1...");
    Thread.sleep(1000);
    alloc1Response = am1.schedule();
  }

  List<Container> allocated1 = alloc1Response.getAllocatedContainers();
  Assert.assertEquals(1, allocated1.size());
  Assert.assertEquals(2 * GB, allocated1.get(0).getResource().getMemory());
  Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId());
  
  report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
  // check node report, 4 GB used and 0 GB available
  Assert.assertEquals(0, report_nm1.getAvailableResource().getMemory());
  Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory());

  // check container is assigned with 2 GB.
  Container c1 = allocated1.get(0);
  Assert.assertEquals(2 * GB, c1.getResource().getMemory());
  
  // update node resource to 2 GB, so resource is over-consumed.
  Map<NodeId, ResourceOption> nodeResourceMap = 
      new HashMap<NodeId, ResourceOption>();
  nodeResourceMap.put(nm1.getNodeId(), 
      ResourceOption.newInstance(Resource.newInstance(2 * GB, 1, 1), -1));
  UpdateNodeResourceRequest request = 
      UpdateNodeResourceRequest.newInstance(nodeResourceMap);
  AdminService as = rm.adminService;
  as.updateNodeResource(request);
  
  // Now, the used resource is still 4 GB, and available resource is minus value.
  report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
  Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory());
  Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemory());
  
  // Check container can complete successfully in case of resource over-commitment.
  ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
      c1.getId(), ContainerState.COMPLETE, "", 0);
  nm1.containerStatus(containerStatus);
  int waitCount = 0;
  while (attempt1.getJustFinishedContainers().size() < 1
      && waitCount++ != 20) {
    LOG.info("Waiting for containers to be finished for app 1... Tried "
        + waitCount + " times already..");
    Thread.sleep(100);
  }
  Assert.assertEquals(1, attempt1.getJustFinishedContainers().size());
  Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size());
  report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
  Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory());
  // As container return 2 GB back, the available resource becomes 0 again.
  Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemory());
  rm.stop();
}
 
Example 30
Source Project: hadoop   Source File: TestFifoScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=2000)
public void testUpdateResourceOnNode() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  Configuration conf = new Configuration();
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.rollMasterKey();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  
  FifoScheduler scheduler = new FifoScheduler(){
    @SuppressWarnings("unused")
    public Map<NodeId, FiCaSchedulerNode> getNodes(){
      return nodes;
    }
  };
  RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
      null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
      scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(new Configuration(), rmContext);
  RMNode node0 = MockNodes.newNodeInfo(1,
      Resources.createResource(2048, 4, 4), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
  scheduler.handle(nodeEvent1);
  
  Method method = scheduler.getClass().getDeclaredMethod("getNodes");
  @SuppressWarnings("unchecked")
  Map<NodeId, FiCaSchedulerNode> schedulerNodes = 
      (Map<NodeId, FiCaSchedulerNode>) method.invoke(scheduler);
  assertEquals(schedulerNodes.values().size(), 1);
  
  Resource newResource = Resources.createResource(1024, 4, 4);
  
  NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new 
      NodeResourceUpdateSchedulerEvent(node0, ResourceOption.newInstance(
          newResource, RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT));
  scheduler.handle(node0ResourceUpdate);
  
  // SchedulerNode's total resource and available resource are changed.
  assertEquals(schedulerNodes.get(node0.getNodeID()).getTotalResource()
      .getMemory(), 1024);
  assertEquals(schedulerNodes.get(node0.getNodeID()).
      getAvailableResource().getMemory(), 1024);
  QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
  Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
  
  int _appId = 1;
  int _appAttemptId = 1;
  ApplicationAttemptId appAttemptId = createAppAttemptId(_appId,
      _appAttemptId);
  createMockRMApp(appAttemptId, rmContext);

  AppAddedSchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1",
        "user1");
  scheduler.handle(appEvent);
  AppAttemptAddedSchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  int memory = 1024;
  int priority = 1;

  List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
  ResourceRequest nodeLocal = createResourceRequest(memory,
      node0.getHostName(), priority, 1);
  ResourceRequest rackLocal = createResourceRequest(memory,
      node0.getRackName(), priority, 1);
  ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
      1);
  ask.add(nodeLocal);
  ask.add(rackLocal);
  ask.add(any);
  scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);

  // Before the node update event, there are one local request
  Assert.assertEquals(1, nodeLocal.getNumContainers());

  NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
  // Now schedule.
  scheduler.handle(node0Update);

  // After the node update event, check no local request
  Assert.assertEquals(0, nodeLocal.getNumContainers());
  // Also check that one container was scheduled
  SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
  Assert.assertEquals(1, info.getLiveContainers().size());
  // And check the default Queue now is full.
  queueInfo = scheduler.getQueueInfo(null, false, false);
  Assert.assertEquals(1.0f, queueInfo.getCurrentCapacity(), 0.0f);
}