org.apache.hadoop.yarn.api.records.ResourceOption Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.ResourceOption. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnRunningNode() {
  RMNodeImpl node = getRunningNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2), 
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  Assert.assertNotNull(nodesListManagerEvent);
  Assert.assertEquals(NodesListManagerEventType.NODE_USABLE,
      nodesListManagerEvent.getType());
}
 
Example #2
Source File: TestRMNodeTransitions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnRebootedNode() {
  RMNodeImpl node = getRebootedNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2, 2),
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  assertEquals("GPU resource is not match.", newCapacity.getGpuCores(), 2);
  
  Assert.assertEquals(NodeState.REBOOTED, node.getState());
}
 
Example #3
Source File: TestRMNodeTransitions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnNewNode() {
  RMNodeImpl node = getNewNode(Resource.newInstance(4096, 4, 4));
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2, 2),
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  assertEquals("GPU resource is not match.", newCapacity.getGpuCores(), 2);
  
  Assert.assertEquals(NodeState.NEW, node.getState());
}
 
Example #4
Source File: TestRMNodeTransitions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnRunningNode() {
  RMNodeImpl node = getRunningNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2, 2),
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  assertEquals("GPU resource is not match.", newCapacity.getGpuCores(), 2);
  
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  Assert.assertNotNull(nodesListManagerEvent);
  Assert.assertEquals(NodesListManagerEventType.NODE_USABLE,
      nodesListManagerEvent.getType());
}
 
Example #5
Source File: UpdateNodeResourceRequestPBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void setNodeResourceMap(Map<NodeId, ResourceOption> nodeResourceMap) {
  if (nodeResourceMap == null) {
    return;
  }
  initNodeResourceMap();
  this.nodeResourceMap.clear();
  this.nodeResourceMap.putAll(nodeResourceMap);
}
 
Example #6
Source File: YarnNodeCapacityManager.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
/**
 * 1. Updates {@link RMNode#getTotalCapability()} with newCapacity.
 * 2. Sends out a {@link NodeResourceUpdateSchedulerEvent} that's handled by YARN's scheduler.
 * The scheduler updates the corresponding {@link SchedulerNode} with the newCapacity.
 *
 * @param rmNode
 * @param newCapacity
 */
@SuppressWarnings("unchecked")
public void setNodeCapacity(RMNode rmNode, Resource newCapacity) {
  //NOOP prevent YARN warning changing to same size
  if ((Resources.equals(rmNode.getTotalCapability(), newCapacity))) {
    return;
  }
  if (yarnScheduler.getSchedulerNode(rmNode.getNodeID()) == null) {
    LOGGER.info("Yarn Scheduler doesn't have node {}, probably UNHEALTHY", rmNode.getNodeID());
    return;
  }
  yarnSchedulerLock.lock();
  try {
    if (newCapacity.getMemory() < 0 || newCapacity.getVirtualCores() < 0) {
      Resource zeroed = ResourceUtils.componentwiseMax(ZERO_RESOURCE, newCapacity);
      rmNode.getTotalCapability().setMemory(zeroed.getMemory());
      rmNode.getTotalCapability().setVirtualCores(zeroed.getVirtualCores());
      LOGGER.warn("Asked to set Node {} to a value less than zero!  Had {}, setting to {}.",
          rmNode.getHttpAddress(), rmNode.getTotalCapability().toString(), zeroed.toString());
    } else {
      rmNode.getTotalCapability().setMemory(newCapacity.getMemory());
      rmNode.getTotalCapability().setVirtualCores(newCapacity.getVirtualCores());
      if (LOGGER.isInfoEnabled()) {
        LOGGER.info("Setting capacity for node {} to {}", rmNode.getHostName(), newCapacity);
      }
    }
    // updates the scheduler with the new capacity for the NM.
    // the event is handled by the scheduler asynchronously
    rmContext.getDispatcher().getEventHandler().handle(new NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption.newInstance(
        rmNode.getTotalCapability(), RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  } finally {
    yarnSchedulerLock.unlock();
  }
}
 
Example #7
Source File: UpdateNodeResourceRequest.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Public
@Evolving
public static UpdateNodeResourceRequest newInstance(
    Map<NodeId, ResourceOption> nodeResourceMap) {
  UpdateNodeResourceRequest request =
      Records.newRecord(UpdateNodeResourceRequest.class);
  request.setNodeResourceMap(nodeResourceMap);
  return request;
}
 
Example #8
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testResourceUpdateOnRebootedNode() {
  RMNodeImpl node = getRebootedNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2), 
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  
  Assert.assertEquals(NodeState.REBOOTED, node.getState());
}
 
Example #9
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testResourceUpdateOnNewNode() {
  RMNodeImpl node = getNewNode(Resource.newInstance(4096, 4));
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2), 
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  
  Assert.assertEquals(NodeState.NEW, node.getState());
}
 
Example #10
Source File: CapacityScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node.
 */
private synchronized void updateNodeAndQueueResource(RMNode nm, 
    ResourceOption resourceOption) {
  updateNodeResource(nm, resourceOption);
  root.updateClusterResource(clusterResource, new ResourceLimits(
      clusterResource));
}
 
Example #11
Source File: AbstractYarnScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node.
 */
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  SchedulerNode node = getSchedulerNode(nm.getNodeID());
  Resource newResource = resourceOption.getResource();
  Resource oldResource = node.getTotalResource();
  if(!oldResource.equals(newResource)) {
    // Log resource change
    LOG.info("Update resource on node: " + node.getNodeName()
        + " from: " + oldResource + ", to: "
        + newResource);

    nodes.remove(nm.getNodeID());
    updateMaximumAllocation(node, false);

    // update resource to node
    node.setTotalResource(newResource);

    nodes.put(nm.getNodeID(), (N)node);
    updateMaximumAllocation(node, true);

    // update resource to clusterResource
    Resources.subtractFrom(clusterResource, oldResource);
    Resources.addTo(clusterResource, newResource);
  } else {
    // Log resource change
    LOG.warn("Update resource on node: " + node.getNodeName() 
        + " with the same resource: " + newResource);
  }
}
 
Example #12
Source File: FairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node and update Queue.
 */
@Override
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  super.updateNodeResource(nm, resourceOption);
  updateRootQueueMetrics();
  queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
  queueMgr.getRootQueue().recomputeSteadyShares();
}
 
Example #13
Source File: RMNodeImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static void updateNodeResourceFromEvent(RMNodeImpl rmNode, 
   RMNodeResourceUpdateEvent event){
    ResourceOption resourceOption = event.getResourceOption();
    // Set resource on RMNode
    rmNode.totalCapability = resourceOption.getResource();
    
}
 
Example #14
Source File: UpdateNodeResourceRequestPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void initNodeResourceMap() {
  if (this.nodeResourceMap != null) {
    return;
  }
  UpdateNodeResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
  List<NodeResourceMapProto> list = p.getNodeResourceMapList();
  this.nodeResourceMap = new HashMap<NodeId, ResourceOption>(list
      .size());
  for (NodeResourceMapProto nodeResourceProto : list) {
    this.nodeResourceMap.put(convertFromProtoFormat(nodeResourceProto.getNodeId()), 
        convertFromProtoFormat(nodeResourceProto.getResourceOption()));
  }
}
 
Example #15
Source File: UpdateNodeResourceRequestPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void setNodeResourceMap(Map<NodeId, ResourceOption> nodeResourceMap) {
  if (nodeResourceMap == null) {
    return;
  }
  initNodeResourceMap();
  this.nodeResourceMap.clear();
  this.nodeResourceMap.putAll(nodeResourceMap);
}
 
Example #16
Source File: UpdateNodeResourceRequest.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Public
@Evolving
public static UpdateNodeResourceRequest newInstance(
    Map<NodeId, ResourceOption> nodeResourceMap) {
  UpdateNodeResourceRequest request =
      Records.newRecord(UpdateNodeResourceRequest.class);
  request.setNodeResourceMap(nodeResourceMap);
  return request;
}
 
Example #17
Source File: UpdateNodeResourceRequestPBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void initNodeResourceMap() {
  if (this.nodeResourceMap != null) {
    return;
  }
  UpdateNodeResourceRequestProtoOrBuilder p = viaProto ? proto : builder;
  List<NodeResourceMapProto> list = p.getNodeResourceMapList();
  this.nodeResourceMap = new HashMap<NodeId, ResourceOption>(list
      .size());
  for (NodeResourceMapProto nodeResourceProto : list) {
    this.nodeResourceMap.put(convertFromProtoFormat(nodeResourceProto.getNodeId()), 
        convertFromProtoFormat(nodeResourceProto.getResourceOption()));
  }
}
 
Example #18
Source File: CapacityScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node.
 */
private synchronized void updateNodeAndQueueResource(RMNode nm, 
    ResourceOption resourceOption) {
  updateNodeResource(nm, resourceOption);
  root.updateClusterResource(clusterResource, new ResourceLimits(
      clusterResource));
}
 
Example #19
Source File: AbstractYarnScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node.
 */
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  SchedulerNode node = getSchedulerNode(nm.getNodeID());
  Resource newResource = resourceOption.getResource();
  Resource oldResource = node.getTotalResource();
  if(!oldResource.equals(newResource)) {
    // Notify NodeLabelsManager about this change
    rmContext.getNodeLabelManager().updateNodeResource(nm.getNodeID(),
        newResource);

    // Log resource change
    LOG.info("Update resource on node: " + node.getNodeName()
        + " from: " + oldResource + ", to: "
        + newResource);

    nodes.remove(nm.getNodeID());
    updateMaximumAllocation(node, false);

    // update resource to node
    node.setTotalResource(newResource);

    nodes.put(nm.getNodeID(), (N)node);
    updateMaximumAllocation(node, true);

    // update resource to clusterResource
    Resources.subtractFrom(clusterResource, oldResource);
    Resources.addTo(clusterResource, newResource);
  } else {
    // Log resource change
    LOG.warn("Update resource on node: " + node.getNodeName() 
        + " with the same resource: " + newResource);
  }
}
 
Example #20
Source File: FairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node and update Queue.
 */
@Override
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  super.updateNodeResource(nm, resourceOption);
  updateRootQueueMetrics();
  queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
  queueMgr.getRootQueue().recomputeSteadyShares();
}
 
Example #21
Source File: UpdateNodeResourceRequestPBImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public Map<NodeId, ResourceOption> getNodeResourceMap() {
  initNodeResourceMap();
  return this.nodeResourceMap;
}
 
Example #22
Source File: TestAbstractYarnScheduler.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testMaxAllocationAfterUpdateNodeResource() throws IOException {
  final int configuredMaxVCores = 20;
  final int configuredMaxMemory = 10 * 1024;
  Resource configuredMaximumResource = Resource.newInstance
      (configuredMaxMemory, configuredMaxVCores, configuredMaxVCores);

  configureScheduler();
  YarnConfiguration conf = getConf();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
      configuredMaxVCores);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
      configuredMaxMemory);
  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      0);

  MockRM rm = new MockRM(conf);
  try {
    rm.start();
    AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm
        .getResourceScheduler();
    verifyMaximumResourceCapability(configuredMaximumResource, scheduler);

    Resource resource1 = Resource.newInstance(2048, 5, 5);
    Resource resource2 = Resource.newInstance(4096, 10, 10);
    Resource resource3 = Resource.newInstance(512, 1, 1);
    Resource resource4 = Resource.newInstance(1024, 2, 2);

    RMNode node1 = MockNodes.newNodeInfo(
        0, resource1, 1, "127.0.0.2");
    scheduler.handle(new NodeAddedSchedulerEvent(node1));
    RMNode node2 = MockNodes.newNodeInfo(
        0, resource3, 2, "127.0.0.3");
    scheduler.handle(new NodeAddedSchedulerEvent(node2));
    verifyMaximumResourceCapability(resource1, scheduler);

    // increase node1 resource
    scheduler.updateNodeResource(node1, ResourceOption.newInstance(
        resource2, 0));
    verifyMaximumResourceCapability(resource2, scheduler);

    // decrease node1 resource
    scheduler.updateNodeResource(node1, ResourceOption.newInstance(
        resource1, 0));
    verifyMaximumResourceCapability(resource1, scheduler);

    // increase node2 resource
    scheduler.updateNodeResource(node2, ResourceOption.newInstance(
        resource4, 0));
    verifyMaximumResourceCapability(resource1, scheduler);

    // decrease node2 resource
    scheduler.updateNodeResource(node2, ResourceOption.newInstance(
        resource3, 0));
    verifyMaximumResourceCapability(resource1, scheduler);
  } finally {
    rm.stop();
  }
}
 
Example #23
Source File: UpdateNodeResourceRequestPBImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private ResourceOptionProto convertToProtoFormat(ResourceOption c) {
  return ((ResourceOptionPBImpl)c).getProto();
}
 
Example #24
Source File: RMNodeResourceUpdateEvent.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public RMNodeResourceUpdateEvent(NodeId nodeId, ResourceOption resourceOption) {
  super(nodeId, RMNodeEventType.RESOURCE_UPDATE);
  this.resourceOption = resourceOption;
}
 
Example #25
Source File: RMNodeResourceUpdateEvent.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public ResourceOption getResourceOption() {
  return resourceOption;
}
 
Example #26
Source File: TestAbstractYarnScheduler.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testMaxAllocationAfterUpdateNodeResource() throws IOException {
  final int configuredMaxVCores = 20;
  final int configuredMaxMemory = 10 * 1024;
  Resource configuredMaximumResource = Resource.newInstance
      (configuredMaxMemory, configuredMaxVCores);

  configureScheduler();
  YarnConfiguration conf = getConf();
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
      configuredMaxVCores);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
      configuredMaxMemory);
  conf.setLong(
      YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
      0);

  MockRM rm = new MockRM(conf);
  try {
    rm.start();
    AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm
        .getResourceScheduler();
    verifyMaximumResourceCapability(configuredMaximumResource, scheduler);

    Resource resource1 = Resource.newInstance(2048, 5);
    Resource resource2 = Resource.newInstance(4096, 10);
    Resource resource3 = Resource.newInstance(512, 1);
    Resource resource4 = Resource.newInstance(1024, 2);

    RMNode node1 = MockNodes.newNodeInfo(
        0, resource1, 1, "127.0.0.2");
    scheduler.handle(new NodeAddedSchedulerEvent(node1));
    RMNode node2 = MockNodes.newNodeInfo(
        0, resource3, 2, "127.0.0.3");
    scheduler.handle(new NodeAddedSchedulerEvent(node2));
    verifyMaximumResourceCapability(resource1, scheduler);

    // increase node1 resource
    scheduler.updateNodeResource(node1, ResourceOption.newInstance(
        resource2, 0));
    verifyMaximumResourceCapability(resource2, scheduler);

    // decrease node1 resource
    scheduler.updateNodeResource(node1, ResourceOption.newInstance(
        resource1, 0));
    verifyMaximumResourceCapability(resource1, scheduler);

    // increase node2 resource
    scheduler.updateNodeResource(node2, ResourceOption.newInstance(
        resource4, 0));
    verifyMaximumResourceCapability(resource1, scheduler);

    // decrease node2 resource
    scheduler.updateNodeResource(node2, ResourceOption.newInstance(
        resource3, 0));
    verifyMaximumResourceCapability(resource1, scheduler);
  } finally {
    rm.stop();
  }
}
 
Example #27
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testResourceOverCommit() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  
  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB);
  RMApp app1 = rm.submitApp(2048);
  // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
  am1.registerAppAttempt();
  SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
      nm1.getNodeId());
  // check node report, 2 GB used and 2 GB available
  Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory());
  Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemory());

  // add request for containers
  am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
  AllocateResponse alloc1Response = am1.schedule(); // send the request

  // kick the scheduler, 2 GB given to AM1, resource remaining 0
  nm1.nodeHeartbeat(true);
  while (alloc1Response.getAllocatedContainers().size() < 1) {
    LOG.info("Waiting for containers to be created for app 1...");
    Thread.sleep(100);
    alloc1Response = am1.schedule();
  }

  List<Container> allocated1 = alloc1Response.getAllocatedContainers();
  Assert.assertEquals(1, allocated1.size());
  Assert.assertEquals(2 * GB, allocated1.get(0).getResource().getMemory());
  Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId());
  
  report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
  // check node report, 4 GB used and 0 GB available
  Assert.assertEquals(0, report_nm1.getAvailableResource().getMemory());
  Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory());

  // check container is assigned with 2 GB.
  Container c1 = allocated1.get(0);
  Assert.assertEquals(2 * GB, c1.getResource().getMemory());
  
  // update node resource to 2 GB, so resource is over-consumed.
  Map<NodeId, ResourceOption> nodeResourceMap = 
      new HashMap<NodeId, ResourceOption>();
  nodeResourceMap.put(nm1.getNodeId(), 
      ResourceOption.newInstance(Resource.newInstance(2 * GB, 1), -1));
  UpdateNodeResourceRequest request = 
      UpdateNodeResourceRequest.newInstance(nodeResourceMap);
  AdminService as = ((MockRM)rm).getAdminService();
  as.updateNodeResource(request);
  
  // Now, the used resource is still 4 GB, and available resource is minus value.
  report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
  Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory());
  Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemory());
  
  // Check container can complete successfully in case of resource over-commitment.
  ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
      c1.getId(), ContainerState.COMPLETE, "", 0);
  nm1.containerStatus(containerStatus);
  int waitCount = 0;
  while (attempt1.getJustFinishedContainers().size() < 1
      && waitCount++ != 20) {
    LOG.info("Waiting for containers to be finished for app 1... Tried "
        + waitCount + " times already..");
    Thread.sleep(100);
  }
  Assert.assertEquals(1, attempt1.getJustFinishedContainers().size());
  Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size());
  report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
  Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory());
  // As container return 2 GB back, the available resource becomes 0 again.
  Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemory());
  
  // Verify no NPE is trigger in schedule after resource is updated.
  am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 1, 1);
  alloc1Response = am1.schedule();
  Assert.assertEquals("Shouldn't have enough resource to allocate containers",
      0, alloc1Response.getAllocatedContainers().size());
  int times = 0;
  // try 10 times as scheduling is async process.
  while (alloc1Response.getAllocatedContainers().size() < 1
      && times++ < 10) {
    LOG.info("Waiting for containers to be allocated for app 1... Tried "
        + times + " times already..");
    Thread.sleep(100);
  }
  Assert.assertEquals("Shouldn't have enough resource to allocate containers",
      0, alloc1Response.getAllocatedContainers().size());
  rm.stop();
}
 
Example #28
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(timeout=2000)
public void testUpdateResourceOnNode() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  Configuration conf = new Configuration();
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.rollMasterKey();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  
  FifoScheduler scheduler = new FifoScheduler(){
    @SuppressWarnings("unused")
    public Map<NodeId, FiCaSchedulerNode> getNodes(){
      return nodes;
    }
  };
  RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
      null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
      scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(new Configuration(), rmContext);
  RMNode node0 = MockNodes.newNodeInfo(1,
      Resources.createResource(2048, 4), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
  scheduler.handle(nodeEvent1);
  
  Method method = scheduler.getClass().getDeclaredMethod("getNodes");
  @SuppressWarnings("unchecked")
  Map<NodeId, FiCaSchedulerNode> schedulerNodes = 
      (Map<NodeId, FiCaSchedulerNode>) method.invoke(scheduler);
  assertEquals(schedulerNodes.values().size(), 1);
  
  Resource newResource = Resources.createResource(1024, 4);
  
  NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new 
      NodeResourceUpdateSchedulerEvent(node0, ResourceOption.newInstance(
          newResource, RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT));
  scheduler.handle(node0ResourceUpdate);
  
  // SchedulerNode's total resource and available resource are changed.
  assertEquals(schedulerNodes.get(node0.getNodeID()).getTotalResource()
      .getMemory(), 1024);
  assertEquals(schedulerNodes.get(node0.getNodeID()).
      getAvailableResource().getMemory(), 1024);
  QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
  Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
  
  int _appId = 1;
  int _appAttemptId = 1;
  ApplicationAttemptId appAttemptId = createAppAttemptId(_appId,
      _appAttemptId);
  createMockRMApp(appAttemptId, rmContext);

  AppAddedSchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1",
        "user1");
  scheduler.handle(appEvent);
  AppAttemptAddedSchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  int memory = 1024;
  int priority = 1;

  List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
  ResourceRequest nodeLocal = createResourceRequest(memory,
      node0.getHostName(), priority, 1);
  ResourceRequest rackLocal = createResourceRequest(memory,
      node0.getRackName(), priority, 1);
  ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
      1);
  ask.add(nodeLocal);
  ask.add(rackLocal);
  ask.add(any);
  scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);

  // Before the node update event, there are one local request
  Assert.assertEquals(1, nodeLocal.getNumContainers());

  NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
  // Now schedule.
  scheduler.handle(node0Update);

  // After the node update event, check no local request
  Assert.assertEquals(0, nodeLocal.getNumContainers());
  // Also check that one container was scheduled
  SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
  Assert.assertEquals(1, info.getLiveContainers().size());
  // And check the default Queue now is full.
  queueInfo = scheduler.getQueueInfo(null, false, false);
  Assert.assertEquals(1.0f, queueInfo.getCurrentCapacity(), 0.0f);
}
 
Example #29
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testResourceOverCommit() throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();
  
  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB);
  
  RMApp app1 = rm.submitApp(2048);
  // kick the scheduling, 2 GB given to AM1, remaining 2GB on nm1
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
  am1.registerAppAttempt();
  SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
      nm1.getNodeId());
  // check node report, 2 GB used and 2 GB available
  Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory());
  Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemory());

  // add request for containers
  am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
  AllocateResponse alloc1Response = am1.schedule(); // send the request

  // kick the scheduler, 2 GB given to AM1, resource remaining 0
  nm1.nodeHeartbeat(true);
  while (alloc1Response.getAllocatedContainers().size() < 1) {
    LOG.info("Waiting for containers to be created for app 1...");
    Thread.sleep(1000);
    alloc1Response = am1.schedule();
  }

  List<Container> allocated1 = alloc1Response.getAllocatedContainers();
  Assert.assertEquals(1, allocated1.size());
  Assert.assertEquals(2 * GB, allocated1.get(0).getResource().getMemory());
  Assert.assertEquals(nm1.getNodeId(), allocated1.get(0).getNodeId());
  
  report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
  // check node report, 4 GB used and 0 GB available
  Assert.assertEquals(0, report_nm1.getAvailableResource().getMemory());
  Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory());

  // check container is assigned with 2 GB.
  Container c1 = allocated1.get(0);
  Assert.assertEquals(2 * GB, c1.getResource().getMemory());
  
  // update node resource to 2 GB, so resource is over-consumed.
  Map<NodeId, ResourceOption> nodeResourceMap = 
      new HashMap<NodeId, ResourceOption>();
  nodeResourceMap.put(nm1.getNodeId(), 
      ResourceOption.newInstance(Resource.newInstance(2 * GB, 1), -1));
  UpdateNodeResourceRequest request = 
      UpdateNodeResourceRequest.newInstance(nodeResourceMap);
  AdminService as = rm.adminService;
  as.updateNodeResource(request);
  
  // Now, the used resource is still 4 GB, and available resource is minus value.
  report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
  Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemory());
  Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemory());
  
  // Check container can complete successfully in case of resource over-commitment.
  ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
      c1.getId(), ContainerState.COMPLETE, "", 0);
  nm1.containerStatus(containerStatus);
  int waitCount = 0;
  while (attempt1.getJustFinishedContainers().size() < 1
      && waitCount++ != 20) {
    LOG.info("Waiting for containers to be finished for app 1... Tried "
        + waitCount + " times already..");
    Thread.sleep(100);
  }
  Assert.assertEquals(1, attempt1.getJustFinishedContainers().size());
  Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size());
  report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
  Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory());
  // As container return 2 GB back, the available resource becomes 0 again.
  Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemory());
  rm.stop();
}
 
Example #30
Source File: RMNodeImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private static void updateNodeResourceFromEvent(RMNodeImpl rmNode, 
   RMNodeResourceUpdateEvent event){
    ResourceOption resourceOption = event.getResourceOption();
    // Set resource on RMNode
    rmNode.totalCapability = resourceOption.getResource();
}