Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   Source File: MockNodes.java    License: Apache License 2.0 6 votes vote down vote up
private static RMNode buildRMNode(int rack, final Resource perNode,
    NodeState state, String httpAddr, int hostnum, String hostName, int port,
    Set<String> labels) {
  final String rackName = "rack"+ rack;
  final int nid = hostnum;
  final String nodeAddr = hostName + ":" + nid;
  if (hostName == null) {
    hostName = "host"+ nid;
  }
  final NodeId nodeID = NodeId.newInstance(hostName, port);

  final String httpAddress = httpAddr;
  String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe";
  return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode,
      rackName, healthReport, 0, nid, hostName, state, labels);
}
 
Example 2
Source Project: hadoop   Source File: ClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public GetClusterNodesResponse getClusterNodes(GetClusterNodesRequest request)
    throws YarnException {
  GetClusterNodesResponse response = 
    recordFactory.newRecordInstance(GetClusterNodesResponse.class);
  EnumSet<NodeState> nodeStates = request.getNodeStates();
  if (nodeStates == null || nodeStates.isEmpty()) {
    nodeStates = EnumSet.allOf(NodeState.class);
  }
  Collection<RMNode> nodes = RMServerUtils.queryRMNodes(rmContext,
      nodeStates);
  
  List<NodeReport> nodeReports = new ArrayList<NodeReport>(nodes.size());
  for (RMNode nodeInfo : nodes) {
    nodeReports.add(createNodeReports(nodeInfo));
  }
  response.setNodeReports(nodeReports);
  return response;
}
 
Example 3
Source Project: hadoop   Source File: FifoScheduler.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void removeNode(RMNode nodeInfo) {
  FiCaSchedulerNode node = getNode(nodeInfo.getNodeID());
  if (node == null) {
    return;
  }
  // Kill running containers
  for(RMContainer container : node.getRunningContainers()) {
    completedContainer(container, 
        SchedulerUtils.createAbnormalContainerStatus(
            container.getContainerId(), 
            SchedulerUtils.LOST_CONTAINER),
            RMContainerEventType.KILL);
  }
  
  //Remove the node
  this.nodes.remove(nodeInfo.getNodeID());
  updateMaximumAllocation(node, false);
  
  // Update cluster metrics
  Resources.subtractFrom(clusterResource, node.getRMNode().getTotalCapability());
}
 
Example 4
Source Project: big-c   Source File: CapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void addNode(RMNode nodeManager) {
  FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager,
      usePortForNodeName, nodeManager.getNodeLabels());
  this.nodes.put(nodeManager.getNodeID(), schedulerNode);
  Resources.addTo(clusterResource, nodeManager.getTotalCapability());

  // update this node to node label manager
  if (labelManager != null) {
    labelManager.activateNode(nodeManager.getNodeID(),
        nodeManager.getTotalCapability());
  }
  
  root.updateClusterResource(clusterResource, new ResourceLimits(
      clusterResource));
  int numNodes = numNodeManagers.incrementAndGet();
  updateMaximumAllocation(schedulerNode, true);
  
  LOG.info("Added node " + nodeManager.getNodeAddress() + 
      " clusterResource: " + clusterResource);

  if (scheduleAsynchronously && numNodes == 1) {
    asyncSchedulerThread.beginSchedule();
  }
}
 
Example 5
Source Project: hadoop   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnRebootedNode() {
  RMNodeImpl node = getRebootedNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2, 2),
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  assertEquals("GPU resource is not match.", newCapacity.getGpuCores(), 2);
  
  Assert.assertEquals(NodeState.REBOOTED, node.getState());
}
 
Example 6
Source Project: big-c   Source File: ClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
private NodeReport createNodeReports(RMNode rmNode) {    
  SchedulerNodeReport schedulerNodeReport = 
      scheduler.getNodeReport(rmNode.getNodeID());
  Resource used = BuilderUtils.newResource(0, 0);
  int numContainers = 0;
  if (schedulerNodeReport != null) {
    used = schedulerNodeReport.getUsedResource();
    numContainers = schedulerNodeReport.getNumContainers();
  } 
  
  NodeReport report =
      BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
          rmNode.getHttpAddress(), rmNode.getRackName(), used,
          rmNode.getTotalCapability(), numContainers,
          rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
          rmNode.getNodeLabels());

  return report;
}
 
Example 7
Source Project: hadoop   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceUpdateOnRunningNode() {
  RMNodeImpl node = getRunningNode();
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2, 2),
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  assertEquals("GPU resource is not match.", newCapacity.getGpuCores(), 2);
  
  Assert.assertEquals(NodeState.RUNNING, node.getState());
  Assert.assertNotNull(nodesListManagerEvent);
  Assert.assertEquals(NodesListManagerEventType.NODE_USABLE,
      nodesListManagerEvent.getType());
}
 
Example 8
Source Project: hadoop   Source File: TestUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static FiCaSchedulerNode getMockNode(
    String host, String rack, int port, int capability) {
  NodeId nodeId = mock(NodeId.class);
  when(nodeId.getHost()).thenReturn(host);
  when(nodeId.getPort()).thenReturn(port);
  RMNode rmNode = mock(RMNode.class);
  when(rmNode.getNodeID()).thenReturn(nodeId);
  when(rmNode.getTotalCapability()).thenReturn(
      Resources.createResource(capability, 1));
  when(rmNode.getNodeAddress()).thenReturn(host+":"+port);
  when(rmNode.getHostName()).thenReturn(host);
  when(rmNode.getRackName()).thenReturn(rack);
  
  FiCaSchedulerNode node = spy(new FiCaSchedulerNode(rmNode, false));
  LOG.info("node = " + host + " avail=" + node.getAvailableResource());
  return node;
}
 
Example 9
Source Project: hadoop   Source File: TestFairScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAggregateCapacityTracking() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // Add a node
  RMNode node1 =
      MockNodes
          .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  assertEquals(1024, scheduler.getClusterResource().getMemory());

  // Add another node
  RMNode node2 =
      MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2");
  NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
  scheduler.handle(nodeEvent2);
  assertEquals(1536, scheduler.getClusterResource().getMemory());

  // Remove the first node
  NodeRemovedSchedulerEvent nodeEvent3 = new NodeRemovedSchedulerEvent(node1);
  scheduler.handle(nodeEvent3);
  assertEquals(512, scheduler.getClusterResource().getMemory());
}
 
Example 10
Source Project: hadoop   Source File: RMWebServices.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns all nodes in the cluster. If the states param is given, returns
 * all nodes that are in the comma-separated list of states.
 */
@GET
@Path("/nodes")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodesInfo getNodes(@QueryParam("states") String states) {
  init();
  ResourceScheduler sched = this.rm.getResourceScheduler();
  if (sched == null) {
    throw new NotFoundException("Null ResourceScheduler instance");
  }
  
  EnumSet<NodeState> acceptedStates;
  if (states == null) {
    acceptedStates = EnumSet.allOf(NodeState.class);
  } else {
    acceptedStates = EnumSet.noneOf(NodeState.class);
    for (String stateStr : states.split(",")) {
      acceptedStates.add(
          NodeState.valueOf(StringUtils.toUpperCase(stateStr)));
    }
  }
  
  Collection<RMNode> rmNodes = RMServerUtils.queryRMNodes(this.rm.getRMContext(),
      acceptedStates);
  NodesInfo nodesInfo = new NodesInfo();
  for (RMNode rmNode : rmNodes) {
    NodeInfo nodeInfo = new NodeInfo(rmNode, sched);
    if (EnumSet.of(NodeState.LOST, NodeState.DECOMMISSIONED, NodeState.REBOOTED)
        .contains(rmNode.getState())) {
      nodeInfo.setNodeHTTPAddress(EMPTY);
    }
    nodesInfo.add(nodeInfo);
  }
  
  return nodesInfo;
}
 
Example 11
Source Project: big-c   Source File: RMWebServices.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns all nodes in the cluster. If the states param is given, returns
 * all nodes that are in the comma-separated list of states.
 */
@GET
@Path("/nodes")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodesInfo getNodes(@QueryParam("states") String states) {
  init();
  ResourceScheduler sched = this.rm.getResourceScheduler();
  if (sched == null) {
    throw new NotFoundException("Null ResourceScheduler instance");
  }
  
  EnumSet<NodeState> acceptedStates;
  if (states == null) {
    acceptedStates = EnumSet.allOf(NodeState.class);
  } else {
    acceptedStates = EnumSet.noneOf(NodeState.class);
    for (String stateStr : states.split(",")) {
      acceptedStates.add(
          NodeState.valueOf(StringUtils.toUpperCase(stateStr)));
    }
  }
  
  Collection<RMNode> rmNodes = RMServerUtils.queryRMNodes(this.rm.getRMContext(),
      acceptedStates);
  NodesInfo nodesInfo = new NodesInfo();
  for (RMNode rmNode : rmNodes) {
    NodeInfo nodeInfo = new NodeInfo(rmNode, sched);
    if (EnumSet.of(NodeState.LOST, NodeState.DECOMMISSIONED, NodeState.REBOOTED)
        .contains(rmNode.getState())) {
      nodeInfo.setNodeHTTPAddress(EMPTY);
    }
    nodesInfo.add(nodeInfo);
  }
  
  return nodesInfo;
}
 
Example 12
Source Project: hadoop   Source File: RMAppImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int pullRMNodeUpdates(Collection<RMNode> updatedNodes) {
  this.writeLock.lock();
  try {
    int updatedNodeCount = this.updatedNodes.size();
    updatedNodes.addAll(this.updatedNodes);
    this.updatedNodes.clear();
    return updatedNodeCount;
  } finally {
    this.writeLock.unlock();
  }
}
 
Example 13
Source Project: big-c   Source File: CapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void nodeUpdate(RMNode nm) {
  if (LOG.isDebugEnabled()) {
    LOG.debug("nodeUpdate: " + nm + " clusterResources: " + clusterResource);
  }

  FiCaSchedulerNode node = getNode(nm.getNodeID());
  
  List<UpdatedContainerInfo> containerInfoList = nm.pullContainerUpdates();
  List<ContainerStatus> newlyLaunchedContainers = new ArrayList<ContainerStatus>();
  List<ContainerStatus> completedContainers = new ArrayList<ContainerStatus>();
  for(UpdatedContainerInfo containerInfo : containerInfoList) {
    newlyLaunchedContainers.addAll(containerInfo.getNewlyLaunchedContainers());
    completedContainers.addAll(containerInfo.getCompletedContainers());
  }
  
  // Processing the newly launched containers
  for (ContainerStatus launchedContainer : newlyLaunchedContainers) {
    containerLaunchedOnNode(launchedContainer, node);
    LOG.info("Container LAUNCHED:"+launchedContainer.getContainerId());
  }

  // Process completed containers
  for (ContainerStatus completedContainer : completedContainers) {
    ContainerId containerId = completedContainer.getContainerId();
    LOG.info("Container FINISHED: " + containerId);
    completedContainer(getRMContainer(containerId), 
        completedContainer, RMContainerEventType.FINISHED);
  }
  
  // Now node data structures are up to date and ready for scheduling.
  if(LOG.isDebugEnabled()) 
  {
    LOG.info("Node being looked for scheduling " + nm
      + " availableResource: " + node.getAvailableResource());
  }
}
 
Example 14
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNumClusterNodes() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(conf);
  RMContext rmContext = TestUtils.getMockRMContext();
  cs.setRMContext(rmContext);
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  cs.init(csConf);
  cs.start();
  assertEquals(0, cs.getNumClusterNodes());

  RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
  RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);
  cs.handle(new NodeAddedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n2));
  assertEquals(2, cs.getNumClusterNodes());

  cs.handle(new NodeRemovedSchedulerEvent(n1));
  assertEquals(1, cs.getNumClusterNodes());
  cs.handle(new NodeAddedSchedulerEvent(n1));
  assertEquals(2, cs.getNumClusterNodes());
  cs.handle(new NodeRemovedSchedulerEvent(n2));
  cs.handle(new NodeRemovedSchedulerEvent(n1));
  assertEquals(0, cs.getNumClusterNodes());

  cs.stop();
}
 
Example 15
Source Project: hadoop   Source File: ResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void handle(RMNodeEvent event) {
  NodeId nodeId = event.getNodeId();
  RMNode node = this.rmContext.getRMNodes().get(nodeId);
  if (node != null) {
    try {
      ((EventHandler<RMNodeEvent>) node).handle(event);
    } catch (Throwable t) {
      LOG.error("Error in handling event type " + event.getType()
          + " for node " + nodeId, t);
    }
  }
}
 
Example 16
Source Project: big-c   Source File: TestRMWebServicesNodes.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNodesQueryStateLost() throws JSONException, Exception {
  WebResource r = resource();
  MockNM nm1 = rm.registerNode("h1:1234", 5120);
  MockNM nm2 = rm.registerNode("h2:1234", 5120);
  rm.sendNodeStarted(nm1);
  rm.sendNodeStarted(nm2);
  rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
  rm.NMwaitForState(nm2.getNodeId(), NodeState.RUNNING);
  rm.sendNodeLost(nm1);
  rm.sendNodeLost(nm2);

  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("nodes").queryParam("states", NodeState.LOST.toString())
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);

  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  JSONObject nodes = json.getJSONObject("nodes");
  assertEquals("incorrect number of elements", 1, nodes.length());
  JSONArray nodeArray = nodes.getJSONArray("node");
  assertEquals("incorrect number of elements", 2, nodeArray.length());
  for (int i = 0; i < nodeArray.length(); ++i) {
    JSONObject info = nodeArray.getJSONObject(i);
    String host = info.get("id").toString().split(":")[0];
    RMNode rmNode = rm.getRMContext().getInactiveRMNodes().get(host);
    WebServicesTestUtils.checkStringMatch("nodeHTTPAddress", "",
        info.getString("nodeHTTPAddress"));
    WebServicesTestUtils.checkStringMatch("state", rmNode.getState()
        .toString(), info.getString("state"));
  }
}
 
Example 17
Source Project: big-c   Source File: RMAppImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int pullRMNodeUpdates(Collection<RMNode> updatedNodes) {
  this.writeLock.lock();
  try {
    int updatedNodeCount = this.updatedNodes.size();
    updatedNodes.addAll(this.updatedNodes);
    this.updatedNodes.clear();
    return updatedNodeCount;
  } finally {
    this.writeLock.unlock();
  }
}
 
Example 18
Source Project: hadoop   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 3000)
public void testMaxAssignWithZeroMemoryContainers() throws Exception {
  conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
  
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  RMNode node =
      MockNodes.newNodeInfo(1, Resources.createResource(16384, 16, 16), 0,
          "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  scheduler.handle(nodeEvent);

  ApplicationAttemptId attId =
      createSchedulingRequest(0, 1, 1, "root.default", "user", 8);
  FSAppAttempt app = scheduler.getSchedulerApp(attId);

  // set maxAssign to 2: only 2 containers should be allocated
  scheduler.maxAssign = 2;
  scheduler.update();
  scheduler.handle(updateEvent);
  assertEquals("Incorrect number of containers allocated", 2, app
      .getLiveContainers().size());

  // set maxAssign to -1: all remaining containers should be allocated
  scheduler.maxAssign = -1;
  scheduler.update();
  scheduler.handle(updateEvent);
  assertEquals("Incorrect number of containers allocated", 8, app
      .getLiveContainers().size());
}
 
Example 19
Source Project: hadoop   Source File: FairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void removeNode(RMNode rmNode) {
  FSSchedulerNode node = getFSSchedulerNode(rmNode.getNodeID());
  // This can occur when an UNHEALTHY node reconnects
  if (node == null) {
    return;
  }
  Resources.subtractFrom(clusterResource, rmNode.getTotalCapability());
  updateRootQueueMetrics();

  // Remove running containers
  List<RMContainer> runningContainers = node.getRunningContainers();
  for (RMContainer container : runningContainers) {
    completedContainer(container,
        SchedulerUtils.createAbnormalContainerStatus(
            container.getContainerId(),
            SchedulerUtils.LOST_CONTAINER),
        RMContainerEventType.KILL);
  }

  // Remove reservations, if any
  RMContainer reservedContainer = node.getReservedContainer();
  if (reservedContainer != null) {
    completedContainer(reservedContainer,
        SchedulerUtils.createAbnormalContainerStatus(
            reservedContainer.getContainerId(),
            SchedulerUtils.LOST_CONTAINER),
        RMContainerEventType.KILL);
  }

  nodes.remove(rmNode.getNodeID());
  queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
  queueMgr.getRootQueue().recomputeSteadyShares();
  updateMaximumAllocation(node, false);
  LOG.info("Removed node " + rmNode.getNodeAddress() +
      " cluster capacity: " + clusterResource);
}
 
Example 20
Source Project: hadoop   Source File: FairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node and update Queue.
 */
@Override
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  super.updateNodeResource(nm, resourceOption);
  updateRootQueueMetrics();
  queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
  queueMgr.getRootQueue().recomputeSteadyShares();
}
 
Example 21
Source Project: hadoop   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSchedulerRootQueueMetrics() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // Add a node
  RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  // Queue 1 requests full capacity of node
  createSchedulingRequest(1024, "queue1", "user1", 1);
  scheduler.update();
  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
  scheduler.handle(updateEvent);

  // Now queue 2 requests likewise
  createSchedulingRequest(1024, "queue2", "user1", 1);
  scheduler.update();
  scheduler.handle(updateEvent);

  // Make sure reserved memory gets updated correctly
  assertEquals(1024, scheduler.rootMetrics.getReservedMB());
  
  // Now another node checks in with capacity
  RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(1024));
  NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
  NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2);
  scheduler.handle(nodeEvent2);
  scheduler.handle(updateEvent2);


  // The old reservation should still be there...
  assertEquals(1024, scheduler.rootMetrics.getReservedMB());

  // ... but it should disappear when we update the first node.
  scheduler.handle(updateEvent);
  assertEquals(0, scheduler.rootMetrics.getReservedMB());
}
 
Example 22
Source Project: hadoop   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testMultipleContainersWaitingForReservation() throws IOException {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // Add a node
  RMNode node1 =
      MockNodes
          .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  // Request full capacity of node
  createSchedulingRequest(1024, "queue1", "user1", 1);
  scheduler.update();
  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
  scheduler.handle(updateEvent);

  ApplicationAttemptId attId1 = createSchedulingRequest(1024, "queue2", "user2", 1);
  ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue3", "user3", 1);
  
  scheduler.update();
  scheduler.handle(updateEvent);
  
  // One container should get reservation and the other should get nothing
  assertEquals(1024,
      scheduler.getSchedulerApp(attId1).getCurrentReservation().getMemory());
  assertEquals(0,
      scheduler.getSchedulerApp(attId2).getCurrentReservation().getMemory());
}
 
Example 23
Source Project: hadoop   Source File: AbstractYarnScheduler.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node.
 */
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  SchedulerNode node = getSchedulerNode(nm.getNodeID());
  Resource newResource = resourceOption.getResource();
  Resource oldResource = node.getTotalResource();
  if(!oldResource.equals(newResource)) {
    // Notify NodeLabelsManager about this change
    rmContext.getNodeLabelManager().updateNodeResource(nm.getNodeID(),
        newResource);

    // Log resource change
    LOG.info("Update resource on node: " + node.getNodeName()
        + " from: " + oldResource + ", to: "
        + newResource);

    nodes.remove(nm.getNodeID());
    updateMaximumAllocation(node, false);

    // update resource to node
    node.setTotalResource(newResource);

    nodes.put(nm.getNodeID(), (N)node);
    updateMaximumAllocation(node, true);

    // update resource to clusterResource
    Resources.subtractFrom(clusterResource, oldResource);
    Resources.addTo(clusterResource, newResource);
  } else {
    // Log resource change
    LOG.warn("Update resource on node: " + node.getNodeName() 
        + " with the same resource: " + newResource);
  }
}
 
Example 24
Source Project: hadoop   Source File: CapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Process resource update on a node.
 */
private synchronized void updateNodeAndQueueResource(RMNode nm, 
    ResourceOption resourceOption) {
  updateNodeResource(nm, resourceOption);
  root.updateClusterResource(clusterResource, new ResourceLimits(
      clusterResource));
}
 
Example 25
Source Project: big-c   Source File: TestRMNodeTransitions.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testResourceUpdateOnNewNode() {
  RMNodeImpl node = getNewNode(Resource.newInstance(4096, 4));
  Resource oldCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", oldCapacity.getMemory(), 4096);
  assertEquals("CPU resource is not match.", oldCapacity.getVirtualCores(), 4);
  node.handle(new RMNodeResourceUpdateEvent(node.getNodeID(),
      ResourceOption.newInstance(Resource.newInstance(2048, 2), 
          RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  Resource newCapacity = node.getTotalCapability();
  assertEquals("Memory resource is not match.", newCapacity.getMemory(), 2048);
  assertEquals("CPU resource is not match.", newCapacity.getVirtualCores(), 2);
  
  Assert.assertEquals(NodeState.NEW, node.getState());
}
 
Example 26
Source Project: incubator-myriad   Source File: NMHeartBeatHandler.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected void handleStatusUpdate(RMNodeEvent event, RMContext context) {
  if (!(event instanceof RMNodeStatusEvent)) {
    logger.error("{} not an instance of {}", event.getClass().getName(), RMNodeStatusEvent.class.getName());
    return;
  }

  RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;
  RMNode rmNode = context.getRMNodes().get(event.getNodeId());
  String hostName = rmNode.getNodeID().getHost();

  Node host = nodeStore.getNode(hostName);
  if (host != null) {
    host.snapshotRunningContainers();
  }

  /*
   * Set the new node capacity which is the sum of the current node resources plus those offered by Mesos. 
   * If the sum is greater than the max capacity of the node, reject the offer.
   */
  Resource offeredResources = getNewResourcesOfferedByMesos(hostName);
  Resource currentResources = getResourcesUnderUse(statusEvent);
  
  if (offerWithinResourceLimits(currentResources, offeredResources)) {
    yarnNodeCapacityMgr.setNodeCapacity(rmNode, Resources.add(currentResources, offeredResources));
    logger.info("Updated resources for {} with {} cores and {} memory", rmNode.getNode().getName(), 
            offeredResources.getVirtualCores(), offeredResources.getMemory());
  } else {
    logger.info("Did not update {} with {} cores and {} memory, over max cpu cores and/or max memory", 
            rmNode.getNode().getName(), offeredResources.getVirtualCores(), offeredResources.getMemory());
  }
}
 
Example 27
Source Project: hadoop   Source File: TestFifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNodeUpdateBeforeAppAttemptInit() throws Exception {
  FifoScheduler scheduler = new FifoScheduler();
  MockRM rm = new MockRM(conf);
  scheduler.setRMContext(rm.getRMContext());
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rm.getRMContext());

  RMNode node = MockNodes.newNodeInfo(1,
          Resources.createResource(1024, 4, 4), 1, "127.0.0.1");
  scheduler.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = ApplicationId.newInstance(0, 1);
  scheduler.addApplication(appId, "queue1", "user1", false);

  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  try {
    scheduler.handle(updateEvent);
  } catch (NullPointerException e) {
      Assert.fail();
  }

  ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 1);
  scheduler.addApplicationAttempt(attId, false, false);

  rm.stop();
}
 
Example 28
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReconnectedNode() throws Exception {
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  CapacityScheduler cs = new CapacityScheduler();
  cs.setConf(new YarnConfiguration());
  cs.setRMContext(resourceManager.getRMContext());
  cs.init(csConf);
  cs.start();
  cs.reinitialize(csConf, new RMContextImpl(null, null, null, null,
    null, null, new RMContainerTokenSecretManager(csConf),
    new NMTokenSecretManagerInRM(csConf),
    new ClientToAMTokenSecretManagerInRM(), null));

  RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
  RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);

  cs.handle(new NodeAddedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n2));

  Assert.assertEquals(6 * GB, cs.getClusterResource().getMemory());

  // reconnect n1 with downgraded memory
  n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1);
  cs.handle(new NodeRemovedSchedulerEvent(n1));
  cs.handle(new NodeAddedSchedulerEvent(n1));

  Assert.assertEquals(4 * GB, cs.getClusterResource().getMemory());
  cs.stop();
}
 
Example 29
Source Project: hadoop   Source File: MockNodes.java    License: Apache License 2.0 5 votes vote down vote up
public static List<RMNode> deactivatedNodes(int racks, int nodesPerRack,
    Resource perNode) {
  List<RMNode> list = Lists.newArrayList();
  for (int i = 0; i < racks; ++i) {
    for (int j = 0; j < nodesPerRack; ++j) {
      NodeState[] allStates = NodeState.values();
      list.add(nodeInfo(i, perNode, allStates[j % allStates.length]));
    }
  }
  return list;
}
 
Example 30
Source Project: big-c   Source File: FifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void addNode(RMNode nodeManager) {
  FiCaSchedulerNode schedulerNode = new FiCaSchedulerNode(nodeManager,
      usePortForNodeName);
  this.nodes.put(nodeManager.getNodeID(), schedulerNode);
  Resources.addTo(clusterResource, nodeManager.getTotalCapability());
  updateMaximumAllocation(schedulerNode, true);
}