Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAddAndRemoveAppFromCapacityScheduler() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();

  SchedulerApplication<SchedulerApplicationAttempt> app =
      TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
        cs.getSchedulerApplications(), cs, "a1");
  Assert.assertEquals("a1", app.getQueue().getQueueName());
}
 
Example 2
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAddAndRemoveAppFromCapacityScheduler() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();

  SchedulerApplication<SchedulerApplicationAttempt> app =
      TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
        cs.getSchedulerApplications(), cs, "a1");
  Assert.assertEquals("a1", app.getQueue().getQueueName());
}
 
Example 3
Source Project: hadoop   Source File: TestFifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAddAndRemoveAppFromFiFoScheduler() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> fs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();
  TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
    fs.getSchedulerApplications(), fs, "queue");
}
 
Example 4
Source Project: hadoop   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAddAndRemoveAppFromFairScheduler() throws Exception {
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> scheduler =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) resourceManager
        .getResourceScheduler();
  TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
    scheduler.getSchedulerApplications(), scheduler, "default");
}
 
Example 5
Source Project: hadoop   Source File: ResourceSchedulerWrapper.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void serviceInit(Configuration conf) throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).init(conf);
  super.serviceInit(conf);
}
 
Example 6
Source Project: hadoop   Source File: ResourceSchedulerWrapper.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void serviceStart() throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).start();
  super.serviceStart();
}
 
Example 7
Source Project: hadoop   Source File: ResourceSchedulerWrapper.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void serviceStop() throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).stop();
  super.serviceStop();
}
 
Example 8
Source Project: big-c   Source File: TestFifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAddAndRemoveAppFromFiFoScheduler() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> fs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();
  TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
    fs.getSchedulerApplications(), fs, "queue");
}
 
Example 9
Source Project: big-c   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAddAndRemoveAppFromFairScheduler() throws Exception {
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> scheduler =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) resourceManager
        .getResourceScheduler();
  TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
    scheduler.getSchedulerApplications(), scheduler, "default");
}
 
Example 10
Source Project: big-c   Source File: ResourceSchedulerWrapper.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void serviceInit(Configuration conf) throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).init(conf);
  super.serviceInit(conf);
}
 
Example 11
Source Project: big-c   Source File: ResourceSchedulerWrapper.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void serviceStart() throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).start();
  super.serviceStart();
}
 
Example 12
Source Project: big-c   Source File: ResourceSchedulerWrapper.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void serviceStop() throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).stop();
  super.serviceStop();
}
 
Example 13
Source Project: incubator-myriad   Source File: YarnNodeCapacityManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void afterSchedulerEventHandled(SchedulerEvent event) {
  switch (event.getType()) {
    case NODE_ADDED:
      if (!(event instanceof NodeAddedSchedulerEvent)) {
        LOGGER.error("{} not an instance of {}", event.getClass().getName(), NodeAddedSchedulerEvent.class.getName());
        return;
      }

      NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent) event;
      NodeId nodeId = nodeAddedEvent.getAddedRMNode().getNodeID();
      String host = nodeId.getHost();

      SchedulerNode node = yarnScheduler.getSchedulerNode(nodeId);
      nodeStore.add(node);
      LOGGER.info("afterSchedulerEventHandled: NM registration from node {}", host);
      break;

    case NODE_UPDATE:
      if (!(event instanceof NodeUpdateSchedulerEvent)) {
        LOGGER.error("{} not an instance of {}", event.getClass().getName(), NodeUpdateSchedulerEvent.class.getName());
        return;
      }

      RMNode rmNode = ((NodeUpdateSchedulerEvent) event).getRMNode();
      handleContainerAllocation(rmNode);

      break;

    default:
      break;
  }
}
 
Example 14
@Before
public void setUp() throws Exception {
  NodeStore store = new NodeStore();
  NodeIdProto nodeId = NodeIdProto.newBuilder().setHost("localhost").setPort(8000).build();
  RMNode rmNode = new RMNodeImpl(new NodeIdPBImpl(nodeId), new MockRMContext(), "localhost", 8000, 8070, new NodeBase(),
          new ResourcePBImpl(), "1.0");
  SchedulerNode node = new FiCaSchedulerNode(rmNode, false);
  store.add(node);
  manager = new OfferLifecycleManager(store, new MyriadDriver(new MockSchedulerDriver()));
}
 
Example 15
Source Project: hadoop   Source File: MockRM.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("static-access")
public void clearQueueMetrics(RMApp app) {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) getResourceScheduler())
    .getSchedulerApplications().get(app.getApplicationId()).getQueue()
    .getMetrics().clearQueueMetrics();
}
 
Example 16
Source Project: big-c   Source File: MockRM.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("static-access")
public void clearQueueMetrics(RMApp app) {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) getResourceScheduler())
    .getSchedulerApplications().get(app.getApplicationId()).getQueue()
    .getMetrics().clearQueueMetrics();
}
 
Example 17
Source Project: incubator-myriad   Source File: LeastAMNodesFirstPolicy.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Sort the given list of tasks by the number of App Master containers running on the corresponding NM node.
 *
 * @param taskIDs
 */
@Override
public void apply(List<Protos.TaskID> taskIDs) {
  if (LOGGER.isDebugEnabled()) {
    for (SchedulerNode node : schedulerNodes.values()) {
      LOGGER.debug("Host {} is running {} containers including {} App Masters", node.getNodeID().getHost(),
          node.getRunningContainers().size(), getNumAMContainers(node.getRunningContainers()));
    }
  }
  // We need to lock the YARN scheduler here. If we don't do that, then the YARN scheduler can
  // process HBs from NodeManagers and the state of SchedulerNode objects might change while we
  // are in the middle of sorting them based on the least number of AM containers.
  synchronized (yarnScheduler) {
    Collections.sort(taskIDs, new Comparator<Protos.TaskID>() {
      @Override
      public int compare(Protos.TaskID t1, Protos.TaskID t2) {
        SchedulerNode o1 = schedulerNodes.get(schedulerState.getTask(t1).getHostname());
        SchedulerNode o2 = schedulerNodes.get(schedulerState.getTask(t2).getHostname());

        if (o1 == null) { // a NM was launched by Myriad, but it hasn't yet registered with RM
          if (o2 == null) {
            return 0;
          } else {
            return -1;
          }
        } else if (o2 == null) {
          return 1;
        } // else, both the NMs have registered with RM

        List<RMContainer> runningContainers1 = o1.getRunningContainers();
        List<RMContainer> runningContainers2 = o2.getRunningContainers();

        Integer numRunningAMs1 = getNumAMContainers(runningContainers1);
        Integer numRunningAMs2 = getNumAMContainers(runningContainers2);

        Integer numRunningContainers1 = runningContainers1.size();
        Integer numRunningContainers2 = runningContainers2.size();

        // If two NMs are running equal number of AMs, sort them based on total num of running containers
        if (numRunningAMs1.compareTo(numRunningAMs2) == 0) {
          return numRunningContainers1.compareTo(numRunningContainers2);
        }
        return numRunningAMs1.compareTo(numRunningAMs2);
      }
    });
  }
}
 
Example 18
Source Project: incubator-myriad   Source File: LeastAMNodesFirstPolicy.java    License: Apache License 2.0 4 votes vote down vote up
private void onNodeRemoved(NodeRemovedSchedulerEvent event) {
  SchedulerNode schedulerNode = schedulerNodes.get(event.getRemovedRMNode().getNodeID().getHost());
  if (schedulerNode != null && schedulerNode.getNodeID().equals(event.getRemovedRMNode().getNodeID())) {
    schedulerNodes.remove(schedulerNode.getNodeID().getHost());
  }
}
 
Example 19
Source Project: incubator-myriad   Source File: NodeStore.java    License: Apache License 2.0 4 votes vote down vote up
private String getKey(SchedulerNode schedNode) {
  return schedNode.getNodeID().getHost();
}
 
Example 20
Source Project: incubator-myriad   Source File: NodeStore.java    License: Apache License 2.0 4 votes vote down vote up
public void add(SchedulerNode schedNode) {
  nodeMap.put(getKey(schedNode), new Node(schedNode));
}
 
Example 21
Source Project: incubator-myriad   Source File: Node.java    License: Apache License 2.0 4 votes vote down vote up
public Node(SchedulerNode node) {
  this.node = node;
}
 
Example 22
Source Project: incubator-myriad   Source File: Node.java    License: Apache License 2.0 4 votes vote down vote up
public SchedulerNode getNode() {
  return node;
}
 
Example 23
Source Project: incubator-myriad   Source File: TestObjectFactory.java    License: Apache License 2.0 4 votes vote down vote up
public static SchedulerNode getSchedulerNode(String host, int port, int cores, int memory) {
  RMNode node = TestObjectFactory.getRMNode(host, port, cores, memory);
  return new FiCaSchedulerNode(node, false);
}
 
Example 24
Source Project: incubator-myriad   Source File: LeastAMNodesFirstPolicy.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Called whenever a NM HBs to RM. The NM's updates will already be recorded in the
 * SchedulerNode before this method is called.
 *
 * @param event
 */
private void onNodeUpdated(NodeUpdateSchedulerEvent event) {
  NodeId nodeID = event.getRMNode().getNodeID();
  SchedulerNode schedulerNode = yarnScheduler.getSchedulerNode(nodeID);
  schedulerNodes.put(nodeID.getHost(), schedulerNode); // keep track of only one node per host
}