org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestFairScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testAggregateCapacityTracking() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // Add a node
  RMNode node1 =
      MockNodes
          .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  assertEquals(1024, scheduler.getClusterResource().getMemory());

  // Add another node
  RMNode node2 =
      MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2");
  NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
  scheduler.handle(nodeEvent2);
  assertEquals(1536, scheduler.getClusterResource().getMemory());

  // Remove the first node
  NodeRemovedSchedulerEvent nodeEvent3 = new NodeRemovedSchedulerEvent(node1);
  scheduler.handle(nodeEvent3);
  assertEquals(512, scheduler.getClusterResource().getMemory());
}
 
Example #2
Source File: CapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
private synchronized void addNode(RMNode nodeManager) {
  FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager,
      usePortForNodeName, nodeManager.getNodeLabels());
  this.nodes.put(nodeManager.getNodeID(), schedulerNode);
  Resources.addTo(clusterResource, nodeManager.getTotalCapability());

  // update this node to node label manager
  if (labelManager != null) {
    labelManager.activateNode(nodeManager.getNodeID(),
        nodeManager.getTotalCapability());
  }
  
  root.updateClusterResource(clusterResource, new ResourceLimits(
      clusterResource));
  int numNodes = numNodeManagers.incrementAndGet();
  updateMaximumAllocation(schedulerNode, true);
  
  LOG.info("Added node " + nodeManager.getNodeAddress() + 
      " clusterResource: " + clusterResource);

  if (scheduleAsynchronously && numNodes == 1) {
    asyncSchedulerThread.beginSchedule();
  }
}
 
Example #3
Source File: ClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request)
    throws YarnException {
  GetClusterNodesResponse response = 
    recordFactory.newRecordInstance(GetClusterNodesResponse.class);
  EnumSet<NodeState> nodeStates = request.getNodeStates();
  if (nodeStates == null || nodeStates.isEmpty()) {
    nodeStates = EnumSet.allOf(NodeState.class);
  }
  Collection<RMNode> nodes = RMServerUtils.queryRMNodes(rmContext,
      nodeStates);
  
  List<NodeReport> nodeReports = new ArrayList<NodeReport>(nodes.size());
  for (RMNode nodeInfo : nodes) {
    nodeReports.add(createNodeReports(nodeInfo));
  }
  response.setNodeReports(nodeReports);
  return response;
}
 
Example #4
Source File: FifoScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private synchronized void removeNode(RMNode nodeInfo) {
  FiCaSchedulerNode node = getNode(nodeInfo.getNodeID());
  if (node == null) {
    return;
  }
  // Kill running containers
  for(RMContainer container : node.getRunningContainers()) {
    completedContainer(container, 
        SchedulerUtils.createAbnormalContainerStatus(
            container.getContainerId(), 
            SchedulerUtils.LOST_CONTAINER),
            RMContainerEventType.KILL);
  }
  
  //Remove the node
  this.nodes.remove(nodeInfo.getNodeID());
  updateMaximumAllocation(node, false);
  
  // Update cluster metrics
  Resources.subtractFrom(clusterResource, node.getRMNode().getTotalCapability());
}
 
Example #5
Source File: TestRMNodeTransitions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnRebootedNode() {
  RMNodeImpl node = getRebootedNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2, 2),
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  assertEquals("GPU resource is not match.", newCapacity.getGpuCores(), 2);
  
  Assert.assertEquals(NodeState.REBOOTED, node.getState());
}
 
Example #6
Source File: ClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up
private NodeReport createNodeReports(RMNode rmNode) {    
  SchedulerNodeReport schedulerNodeReport = 
      scheduler.getNodeReport(rmNode.getNodeID());
  Resource used = BuilderUtils.newResource(0, 0);
  int numContainers = 0;
  if (schedulerNodeReport != null) {
    used = schedulerNodeReport.getUsedResource();
    numContainers = schedulerNodeReport.getNumContainers();
  } 
  
  NodeReport report =
      BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
          rmNode.getHttpAddress(), rmNode.getRackName(), used,
          rmNode.getTotalCapability(), numContainers,
          rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
          rmNode.getNodeLabels());

  return report;
}
 
Example #7
Source File: TestRMNodeTransitions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnRunningNode() {
  RMNodeImpl node = getRunningNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2, 2),
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  assertEquals("GPU resource is not match.", newCapacity.getGpuCores(), 2);
  
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  Assert.assertNotNull(nodesListManagerEvent);
  Assert.assertEquals(NodesListManagerEventType.NODE_USABLE,
      nodesListManagerEvent.getType());
}
 
Example #8
Source File: MockNodes.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static RMNode buildRMNode(int rack, final Resource perNode,
    NodeState state, String httpAddr, int hostnum, String hostName, int port,
    Set<String> labels) {
  final String rackName = "rack"+ rack;
  final int nid = hostnum;
  final String nodeAddr = hostName + ":" + nid;
  if (hostName == null) {
    hostName = "host"+ nid;
  }
  final NodeId nodeID = NodeId.newInstance(hostName, port);

  final String httpAddress = httpAddr;
  String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe";
  return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode,
      rackName, healthReport, 0, nid, hostName, state, labels);
}
 
Example #9
Source File: TestUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static FiCaSchedulerNode getMockNode(
    String host, String rack, int port, int capability) {
  NodeId nodeId = mock(NodeId.class);
  when(nodeId.getHost()).thenReturn(host);
  when(nodeId.getPort()).thenReturn(port);
  RMNode rmNode = mock(RMNode.class);
  when(rmNode.getNodeID()).thenReturn(nodeId);
  when(rmNode.getTotalCapability()).thenReturn(
      Resources.createResource(capability, 1));
  when(rmNode.getNodeAddress()).thenReturn(host+":"+port);
  when(rmNode.getHostName()).thenReturn(host);
  when(rmNode.getRackName()).thenReturn(rack);
  
  FiCaSchedulerNode node = spy(new FiCaSchedulerNode(rmNode, false));
  LOG.info("node = " + host + " avail=" + node.getAvailableResource());
  return node;
}
 
Example #10
Source File: CapacityScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private synchronized void nodeUpdate(RMNode nm) {
  if (LOG.isDebugEnabled()) {
    LOG.debug("nodeUpdate: " + nm + " clusterResources: " + clusterResource);
  }

  FiCaSchedulerNode node = getNode(nm.getNodeID());
  
  List<UpdatedContainerInfo> containerInfoList = nm.pullContainerUpdates();
  List<ContainerStatus> newlyLaunchedContainers = new ArrayList<ContainerStatus>();
  List<ContainerStatus> completedContainers = new ArrayList<ContainerStatus>();
  for(UpdatedContainerInfo containerInfo : containerInfoList) {
    newlyLaunchedContainers.addAll(containerInfo.getNewlyLaunchedContainers());
    completedContainers.addAll(containerInfo.getCompletedContainers());
  }
  
  // Processing the newly launched containers
  for (ContainerStatus launchedContainer : newlyLaunchedContainers) {
    containerLaunchedOnNode(launchedContainer, node);
    LOG.info("Container LAUNCHED:"+launchedContainer.getContainerId());
  }

  // Process completed containers
  for (ContainerStatus completedContainer : completedContainers) {
    ContainerId containerId = completedContainer.getContainerId();
    LOG.info("Container FINISHED: " + containerId);
    completedContainer(getRMContainer(containerId), 
        completedContainer, RMContainerEventType.FINISHED);
  }
  
  // Now node data structures are up to date and ready for scheduling.
  if(LOG.isDebugEnabled()) 
  {
    LOG.info("Node being looked for scheduling " + nm
      + " availableResource: " + node.getAvailableResource());
  }
}
 
Example #11
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testNumClusterNodes() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(conf);
  RMContext rmContext = TestUtils.getMockRMContext();
  cs.setRMContext(rmContext);
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  cs.init(csConf);
  cs.start();
  assertEquals(0, cs.getNumClusterNodes());

  RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
  RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);
  cs.handle(new NodeAddedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n2));
  assertEquals(2, cs.getNumClusterNodes());

  cs.handle(new NodeRemovedSchedulerEvent(n1));
  assertEquals(1, cs.getNumClusterNodes());
  cs.handle(new NodeAddedSchedulerEvent(n1));
  assertEquals(2, cs.getNumClusterNodes());
  cs.handle(new NodeRemovedSchedulerEvent(n2));
  cs.handle(new NodeRemovedSchedulerEvent(n1));
  assertEquals(0, cs.getNumClusterNodes());

  cs.stop();
}
 
Example #12
Source File: ResourceManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void handle(RMNodeEvent event) {
  NodeId nodeId = event.getNodeId();
  RMNode node = this.rmContext.getRMNodes().get(nodeId);
  if (node != null) {
    try {
      ((EventHandler<RMNodeEvent>) node).handle(event);
    } catch (Throwable t) {
      LOG.error("Error in handling event type " + event.getType()
          + " for node " + nodeId, t);
    }
  }
}
 
Example #13
Source File: TestFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testMultipleContainersWaitingForReservation() throws IOException {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // Add a node
  RMNode node1 =
      MockNodes
          .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  // Request full capacity of node
  createSchedulingRequest(1024, "queue1", "user1", 1);
  scheduler.update();
  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
  scheduler.handle(updateEvent);

  ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue2", "user2", 1);
  ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue3", "user3", 1);
  
  scheduler.update();
  scheduler.handle(updateEvent);
  
  // One container should get reservation and the other should get nothing
  assertEquals(1024,
      scheduler.getSchedulerApp(attId1).getCurrentReservation().getMemory());
  assertEquals(0,
      scheduler.getSchedulerApp(attId2).getCurrentReservation().getMemory());
}
 
Example #14
Source File: RMAppImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public int pullRMNodeUpdates(Collection<RMNode> updatedNodes) {
  this.writeLock.lock();
  try {
    int updatedNodeCount = this.updatedNodes.size();
    updatedNodes.addAll(this.updatedNodes);
    this.updatedNodes.clear();
    return updatedNodeCount;
  } finally {
    this.writeLock.unlock();
  }
}
 
Example #15
Source File: TestRMWebServicesNodes.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testNodesQueryStateLost() throws JSONException, Exception {
  WebResource r = resource();
  MockNM nm1 = rm.registerNode("h1:1234", 5120);
  MockNM nm2 = rm.registerNode("h2:1234", 5120);
  rm.sendNodeStarted(nm1);
  rm.sendNodeStarted(nm2);
  rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
  rm.NMwaitForState(nm2.getNodeId(), NodeState.RUNNING);
  rm.sendNodeLost(nm1);
  rm.sendNodeLost(nm2);

  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("nodes").queryParam("states", NodeState.LOST.toString())
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);

  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  JSONObject nodes = json.getJSONObject("nodes");
  assertEquals("incorrect number of elements", 1, nodes.length());
  JSONArray nodeArray = nodes.getJSONArray("node");
  assertEquals("incorrect number of elements", 2, nodeArray.length());
  for (int i = 0; i < nodeArray.length(); ++i) {
    JSONObject info = nodeArray.getJSONObject(i);
    String host = info.get("id").toString().split(":")[0];
    RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get(host);
    WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "",
        info.getString("nodeHTTPAddress"));
    WebServicesTestUtils.checkStringMatch("state", rmNode.getState()
        .toString(), info.getString("state"));
  }
}
 
Example #16
Source File: RMWebServices.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Returns all nodes in the cluster. If the states param is given, returns
 * all nodes that are in the comma-separated list of states.
 */
@GET
@Path("/nodes")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodesInfo getNodes(@QueryParam("states") String states) {
  init();
  ResourceScheduler sched = this.rm.getResourceScheduler();
  if (sched == null) {
    throw new NotFoundException("Null ResourceScheduler instance");
  }
  
  EnumSet<NodeState> acceptedStates;
  if (states == null) {
    acceptedStates = EnumSet.allOf(NodeState.class);
  } else {
    acceptedStates = EnumSet.noneOf(NodeState.class);
    for (String stateStr : states.split(",")) {
      acceptedStates.add(
          NodeState.valueOf(StringUtils.toUpperCase(stateStr)));
    }
  }
  
  Collection<RMNode> rmNodes = RMServerUtils.queryRMNodes(this.rm.getRMContext(),
      acceptedStates);
  NodesInfo nodesInfo = new NodesInfo();
  for (RMNode rmNode : rmNodes) {
    NodeInfo nodeInfo = new NodeInfo(rmNode, sched);
    if (EnumSet.of(NodeState.LOST, NodeState.DECOMMISSIONED, NodeState.REBOOTED)
        .contains(rmNode.getState())) {
      nodeInfo.setNodeHTTPAddress(EMPTY);
    }
    nodesInfo.add(nodeInfo);
  }
  
  return nodesInfo;
}
 
Example #17
Source File: FairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private synchronized void removeNode(RMNode rmNode) {
  FSSchedulerNode node = getFSSchedulerNode(rmNode.getNodeID());
  // This can occur when an UNHEALTHY node reconnects
  if (node == null) {
    return;
  }
  Resources.subtractFrom(clusterResource, rmNode.getTotalCapability());
  updateRootQueueMetrics();

  // Remove running containers
  List<RMContainer> runningContainers = node.getRunningContainers();
  for (RMContainer container : runningContainers) {
    completedContainer(container,
        SchedulerUtils.createAbnormalContainerStatus(
            container.getContainerId(),
            SchedulerUtils.LOST_CONTAINER),
        RMContainerEventType.KILL);
  }

  // Remove reservations, if any
  RMContainer reservedContainer = node.getReservedContainer();
  if (reservedContainer != null) {
    completedContainer(reservedContainer,
        SchedulerUtils.createAbnormalContainerStatus(
            reservedContainer.getContainerId(),
            SchedulerUtils.LOST_CONTAINER),
        RMContainerEventType.KILL);
  }

  nodes.remove(rmNode.getNodeID());
  queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
  queueMgr.getRootQueue().recomputeSteadyShares();
  updateMaximumAllocation(node, false);
  LOG.info("Removed node " + rmNode.getNodeAddress() +
      " cluster capacity: " + clusterResource);
}
 
Example #18
Source File: FairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node and update Queue.
 */
@Override
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  super.updateNodeResource(nm, resourceOption);
  updateRootQueueMetrics();
  queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
  queueMgr.getRootQueue().recomputeSteadyShares();
}
 
Example #19
Source File: RMAppImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public int pullRMNodeUpdates(Collection<RMNode> updatedNodes) {
  this.writeLock.lock();
  try {
    int updatedNodeCount = this.updatedNodes.size();
    updatedNodes.addAll(this.updatedNodes);
    this.updatedNodes.clear();
    return updatedNodeCount;
  } finally {
    this.writeLock.unlock();
  }
}
 
Example #20
Source File: TestFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testSchedulerRootQueueMetrics() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // Add a node
  RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  // Queue 1 requests full capacity of node
  createSchedulingRequest(1024, "queue1", "user1", 1);
  scheduler.update();
  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
  scheduler.handle(updateEvent);

  // Now queue 2 requests likewise
  createSchedulingRequest(1024, "queue2", "user1", 1);
  scheduler.update();
  scheduler.handle(updateEvent);

  // Make sure reserved memory gets updated correctly
  assertEquals(1024, scheduler.rootMetrics.getReservedMB());
  
  // Now another node checks in with capacity
  RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
  NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
  NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2);
  scheduler.handle(nodeEvent2);
  scheduler.handle(updateEvent2);


  // The old reservation should still be there...
  assertEquals(1024, scheduler.rootMetrics.getReservedMB());

  // ... but it should disappear when we update the first node.
  scheduler.handle(updateEvent);
  assertEquals(0, scheduler.rootMetrics.getReservedMB());
}
 
Example #21
Source File: RMWebServices.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Returns all nodes in the cluster. If the states param is given, returns
 * all nodes that are in the comma-separated list of states.
 */
@GET
@Path("/nodes")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodesInfo getNodes(@QueryParam("states") String states) {
  init();
  ResourceScheduler sched = this.rm.getResourceScheduler();
  if (sched == null) {
    throw new NotFoundException("Null ResourceScheduler instance");
  }
  
  EnumSet<NodeState> acceptedStates;
  if (states == null) {
    acceptedStates = EnumSet.allOf(NodeState.class);
  } else {
    acceptedStates = EnumSet.noneOf(NodeState.class);
    for (String stateStr : states.split(",")) {
      acceptedStates.add(
          NodeState.valueOf(StringUtils.toUpperCase(stateStr)));
    }
  }
  
  Collection<RMNode> rmNodes = RMServerUtils.queryRMNodes(this.rm.getRMContext(),
      acceptedStates);
  NodesInfo nodesInfo = new NodesInfo();
  for (RMNode rmNode : rmNodes) {
    NodeInfo nodeInfo = new NodeInfo(rmNode, sched);
    if (EnumSet.of(NodeState.LOST, NodeState.DECOMMISSIONED, NodeState.REBOOTED)
        .contains(rmNode.getState())) {
      nodeInfo.setNodeHTTPAddress(EMPTY);
    }
    nodesInfo.add(nodeInfo);
  }
  
  return nodesInfo;
}
 
Example #22
Source File: AbstractYarnScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node.
 */
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  SchedulerNode node = getSchedulerNode(nm.getNodeID());
  Resource newResource = resourceOption.getResource();
  Resource oldResource = node.getTotalResource();
  if(!oldResource.equals(newResource)) {
    // Notify NodeLabelsManager about this change
    rmContext.getNodeLabelManager().updateNodeResource(nm.getNodeID(),
        newResource);

    // Log resource change
    LOG.info("Update resource on node: " + node.getNodeName()
        + " from: " + oldResource + ", to: "
        + newResource);

    nodes.remove(nm.getNodeID());
    updateMaximumAllocation(node, false);

    // update resource to node
    node.setTotalResource(newResource);

    nodes.put(nm.getNodeID(), (N)node);
    updateMaximumAllocation(node, true);

    // update resource to clusterResource
    Resources.subtractFrom(clusterResource, oldResource);
    Resources.addTo(clusterResource, newResource);
  } else {
    // Log resource change
    LOG.warn("Update resource on node: " + node.getNodeName() 
        + " with the same resource: " + newResource);
  }
}
 
Example #23
Source File: CapacityScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node.
 */
private synchronized void updateNodeAndQueueResource(RMNode nm, 
    ResourceOption resourceOption) {
  updateNodeResource(nm, resourceOption);
  root.updateClusterResource(clusterResource, new ResourceLimits(
      clusterResource));
}
 
Example #24
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testResourceUpdateOnNewNode() {
  RMNodeImpl node = getNewNode(Resource.newInstance(4096, 4));
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2), 
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  
  Assert.assertEquals(NodeState.NEW, node.getState());
}
 
Example #25
Source File: NMHeartBeatHandler.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected void handleStatusUpdate(RMNodeEvent event, RMContext context) {
  if (!(event instanceof RMNodeStatusEvent)) {
    logger.error("{} not an instance of {}", event.getClass().getName(), RMNodeStatusEvent.class.getName());
    return;
  }

  RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;
  RMNode rmNode = context.getRMNodes().get(event.getNodeId());
  String hostName = rmNode.getNodeID().getHost();

  Node host = nodeStore.getNode(hostName);
  if (host != null) {
    host.snapshotRunningContainers();
  }

  /*
   * Set the new node capacity which is the sum of the current node resources plus those offered by Mesos. 
   * If the sum is greater than the max capacity of the node, reject the offer.
   */
  Resource offeredResources = getNewResourcesOfferedByMesos(hostName);
  Resource currentResources = getResourcesUnderUse(statusEvent);
  
  if (offerWithinResourceLimits(currentResources, offeredResources)) {
    yarnNodeCapacityMgr.setNodeCapacity(rmNode, Resources.add(currentResources, offeredResources));
    logger.info("Updated resources for {} with {} cores and {} memory", rmNode.getNode().getName(), 
            offeredResources.getVirtualCores(), offeredResources.getMemory());
  } else {
    logger.info("Did not update {} with {} cores and {} memory, over max cpu cores and/or max memory", 
            rmNode.getNode().getName(), offeredResources.getVirtualCores(), offeredResources.getMemory());
  }
}
 
Example #26
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testNodeUpdateBeforeAppAttemptInit() throws Exception {
  FifoScheduler scheduler = new FifoScheduler();
  MockRM rm = new MockRM(conf);
  scheduler.setRMContext(rm.getRMContext());
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rm.getRMContext());

  RMNode node = MockNodes.newNodeInfo(1,
          Resources.createResource(1024, 4, 4), 1, "127.0.0.1");
  scheduler.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = ApplicationId.newInstance(0, 1);
  scheduler.addApplication(appId, "queue1", "user1", false);

  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  try {
    scheduler.handle(updateEvent);
  } catch (NullPointerException e) {
      Assert.fail();
  }

  ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 1);
  scheduler.addApplicationAttempt(attId, false, false);

  rm.stop();
}
 
Example #27
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testReconnectedNode() throws Exception {
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(csConf);
  cs.start();
  cs.reinitialize(csConf, new RMContextImpl(null, null, null, null,
    null, null, new RMContainerTokenSecretManager(csConf),
    new NMTokenSecretManagerInRM(csConf),
    new ClientToAMTokenSecretManagerInRM(), null));

  RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
  RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);

  cs.handle(new NodeAddedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n2));

  Assert.assertEquals(6 * GB, cs.getClusterResource().getMemory());

  // reconnect n1 with downgraded memory
  n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1);
  cs.handle(new NodeRemovedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n1));

  Assert.assertEquals(4 * GB, cs.getClusterResource().getMemory());
  cs.stop();
}
 
Example #28
Source File: MockNodes.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static List<RMNode> deactivatedNodes(int racks, int nodesPerRack,
    Resource perNode) {
  List<RMNode> list = Lists.newArrayList();
  for (int i = 0; i < racks; ++i) {
    for (int j = 0; j < nodesPerRack; ++j) {
      NodeState[] allStates = NodeState.values();
      list.add(nodeInfo(i, perNode, allStates[j % allStates.length]));
    }
  }
  return list;
}
 
Example #29
Source File: FifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private synchronized void addNode(RMNode nodeManager) {
  FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager,
      usePortForNodeName);
  this.nodes.put(nodeManager.getNodeID(), schedulerNode);
  Resources.addTo(clusterResource, nodeManager.getTotalCapability());
  updateMaximumAllocation(schedulerNode, true);
}
 
Example #30
Source File: TestContinuousScheduling.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 60000)
public void testSchedulingDelay() throws InterruptedException {
  // Add one node
  String host = "127.0.0.1";
  RMNode node1 = MockNodes.newNodeInfo(
      1, Resources.createResource(4096, 4, 4), 1, host);
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  NodeUpdateSchedulerEvent nodeUpdateEvent = new NodeUpdateSchedulerEvent(node1);
  scheduler.handle(nodeUpdateEvent);

  // Create one application and submit one each of node-local, rack-local
  // and ANY requests
  ApplicationAttemptId appAttemptId =
      createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
  createMockRMApp(appAttemptId);

  scheduler.addApplication(appAttemptId.getApplicationId(), "queue11", "user11", false);
  scheduler.addApplicationAttempt(appAttemptId, false, false);
  List<ResourceRequest> ask = new ArrayList<>();
  ask.add(createResourceRequest(1024, 1, 1, ResourceRequest.ANY, 1, 1, true));
  scheduler.allocate(
      appAttemptId, ask, new ArrayList<ContainerId>(), null, null);
  FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId);

  // Advance time and let continuous scheduling kick in
  mockClock.tick(1);
  while (1024 != app.getCurrentConsumption().getMemory()) {
    Thread.sleep(100);
  }
  assertEquals(1024, app.getCurrentConsumption().getMemory());
}