Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: ResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void handle(SchedulerEvent event) {
  try {
    int qSize = eventQueue.size();
    if (qSize !=0 && qSize %1000 == 0) {
      LOG.info("Size of scheduler event-queue is " + qSize);
    }
    int remCapacity = eventQueue.remainingCapacity();
    if (remCapacity < 1000) {
      LOG.info("Very low remaining capacity on scheduler event queue: "
          + remCapacity);
    }
    this.eventQueue.put(event);
  } catch (InterruptedException e) {
    LOG.info("Interrupted. Trying to exit gracefully.");
  }
}
 
Example 2
Source Project: hadoop   Source File: TestSchedulerUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static SchedulerApplication<SchedulerApplicationAttempt>
    verifyAppAddedAndRemovedFromScheduler(
        Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>> applications,
        EventHandler<SchedulerEvent> handler, String queueName)
        throws Exception {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appId, queueName, "user");
  handler.handle(appAddedEvent);
  SchedulerApplication<SchedulerApplicationAttempt> app =
      applications.get(appId);
  // verify application is added.
  Assert.assertNotNull(app);
  Assert.assertEquals("user", app.getUser());

  AppRemovedSchedulerEvent appRemoveEvent =
      new AppRemovedSchedulerEvent(appId, RMAppState.FINISHED);
  handler.handle(appRemoveEvent);
  Assert.assertNull(applications.get(appId));
  return app;
}
 
Example 3
Source Project: big-c   Source File: ResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void handle(SchedulerEvent event) {
  try {
    int qSize = eventQueue.size();
    if (qSize !=0 && qSize %1000 == 0) {
      LOG.info("Size of scheduler event-queue is " + qSize);
    }
    int remCapacity = eventQueue.remainingCapacity();
    if (remCapacity < 1000) {
      LOG.info("Very low remaining capacity on scheduler event queue: "
          + remCapacity);
    }
    this.eventQueue.put(event);
  } catch (InterruptedException e) {
    LOG.info("Interrupted. Trying to exit gracefully.");
  }
}
 
Example 4
Source Project: big-c   Source File: TestSchedulerUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static SchedulerApplication<SchedulerApplicationAttempt>
    verifyAppAddedAndRemovedFromScheduler(
        Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>> applications,
        EventHandler<SchedulerEvent> handler, String queueName)
        throws Exception {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appId, queueName, "user");
  handler.handle(appAddedEvent);
  SchedulerApplication<SchedulerApplicationAttempt> app =
      applications.get(appId);
  // verify application is added.
  Assert.assertNotNull(app);
  Assert.assertEquals("user", app.getUser());

  AppRemovedSchedulerEvent appRemoveEvent =
      new AppRemovedSchedulerEvent(appId, RMAppState.FINISHED);
  handler.handle(appRemoveEvent);
  Assert.assertNull(applications.get(appId));
  return app;
}
 
Example 5
Source Project: incubator-myriad   Source File: LeastAMNodesFirstPolicy.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void afterSchedulerEventHandled(SchedulerEvent event) {

  try {
    switch (event.getType()) {
      case NODE_UPDATE:
        onNodeUpdated((NodeUpdateSchedulerEvent) event);
        break;

      case NODE_REMOVED:
        onNodeRemoved((NodeRemovedSchedulerEvent) event);
        break;

      default:
        break;
    }
  } catch (ClassCastException e) {
    LOGGER.error("incorrect event object", e);
  }
}
 
Example 6
Source Project: hadoop   Source File: ResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run() {

  SchedulerEvent event;

  while (!stopped && !Thread.currentThread().isInterrupted()) {
    try {
      event = eventQueue.take();
    } catch (InterruptedException e) {
      LOG.error("Returning, interrupted : " + e);
      return; // TODO: Kill RM.
    }

    try {
      scheduler.handle(event);
    } catch (Throwable t) {
      // An error occurred, but we are shutting down anyway.
      // If it was an InterruptedException, the very act of 
      // shutdown could have caused it and is probably harmless.
      if (stopped) {
        LOG.warn("Exception during shutdown: ", t);
        break;
      }
      LOG.fatal("Error in handling event type " + event.getType()
          + " to the scheduler", t);
      if (shouldExitOnError
          && !ShutdownHookManager.get().isShutdownInProgress()) {
        LOG.info("Exiting, bbye..");
        System.exit(-1);
      }
    }
  }
}
 
Example 7
Source Project: hadoop   Source File: TestAMRMRPCNodeUpdates.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  dispatcher = new DrainDispatcher();
  this.rm = new MockRM() {
    @Override
    public void init(Configuration conf) {
      conf.set(
        CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
        "1.0");
      super.init(conf);
    }
    @Override
    protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
      return new SchedulerEventDispatcher(this.scheduler) {
        @Override
        public void handle(SchedulerEvent event) {
          scheduler.handle(event);
        }
      };
    }

    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm.start();
  amService = rm.getApplicationMasterService();
}
 
Example 8
Source Project: hadoop   Source File: TestFifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=5000)
public void testAppAttemptMetrics() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  
  FifoScheduler scheduler = new FifoScheduler();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext rmContext = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, writer, scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  Configuration conf = new Configuration();
  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);
  QueueMetrics metrics = scheduler.getRootQueueMetrics();
  int beforeAppsSubmitted = metrics.getAppsSubmitted();

  ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
  scheduler.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent2);

  int afterAppsSubmitted = metrics.getAppsSubmitted();
  Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
  scheduler.stop();
}
 
Example 9
Source Project: hadoop   Source File: TestFifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBlackListNodes() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  String host = "127.0.0.1";
  RMNode node =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
  fs.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  createMockRMApp(appAttemptId, rm.getRMContext());

  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId, "default",
        "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  fs.handle(attemptEvent);

  // Verify the blacklist can be updated independent of requesting containers
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(),
      Collections.singletonList(host), null);
  Assert.assertTrue(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(), null,
      Collections.singletonList(host));
  Assert.assertFalse(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  rm.stop();
}
 
Example 10
Source Project: hadoop   Source File: TestAMRMClientOnRMRestart.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
  // Dispatch inline for test sanity
  return new EventHandler<SchedulerEvent>() {
    @Override
    public void handle(SchedulerEvent event) {
      scheduler.handle(event);
    }
  };
}
 
Example 11
Source Project: hadoop   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
  // Dispatch inline for test sanity
  return new EventHandler<SchedulerEvent>() {
    @Override
    public void handle(SchedulerEvent event) {
      scheduler.handle(event);
    }
  };
}
 
Example 12
Source Project: big-c   Source File: ResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run() {

  SchedulerEvent event;

  while (!stopped && !Thread.currentThread().isInterrupted()) {
    try {
      event = eventQueue.take();
    } catch (InterruptedException e) {
      LOG.error("Returning, interrupted : " + e);
      return; // TODO: Kill RM.
    }

    try {
      scheduler.handle(event);
    } catch (Throwable t) {
      // An error occurred, but we are shutting down anyway.
      // If it was an InterruptedException, the very act of 
      // shutdown could have caused it and is probably harmless.
      if (stopped) {
        LOG.warn("Exception during shutdown: ", t);
        break;
      }
      LOG.fatal("Error in handling event type " + event.getType()
          + " to the scheduler", t);
      if (shouldExitOnError
          && !ShutdownHookManager.get().isShutdownInProgress()) {
        LOG.info("Exiting, bbye..");
        System.exit(-1);
      }
    }
  }
}
 
Example 13
Source Project: big-c   Source File: TestAMRMRPCNodeUpdates.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  dispatcher = new DrainDispatcher();
  this.rm = new MockRM() {
    @Override
    public void init(Configuration conf) {
      conf.set(
        CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
        "1.0");
      super.init(conf);
    }
    @Override
    protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
      return new SchedulerEventDispatcher(this.scheduler) {
        @Override
        public void handle(SchedulerEvent event) {
          scheduler.handle(event);
        }
      };
    }

    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm.start();
  amService = rm.getApplicationMasterService();
}
 
Example 14
Source Project: big-c   Source File: TestFifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=5000)
public void testAppAttemptMetrics() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  
  FifoScheduler scheduler = new FifoScheduler();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext rmContext = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, writer, scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  Configuration conf = new Configuration();
  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);
  QueueMetrics metrics = scheduler.getRootQueueMetrics();
  int beforeAppsSubmitted = metrics.getAppsSubmitted();

  ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
  scheduler.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent2);

  int afterAppsSubmitted = metrics.getAppsSubmitted();
  Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
  scheduler.stop();
}
 
Example 15
Source Project: big-c   Source File: TestFifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBlackListNodes() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  String host = "127.0.0.1";
  RMNode node =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
  fs.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  createMockRMApp(appAttemptId, rm.getRMContext());

  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId, "default",
        "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  fs.handle(attemptEvent);

  // Verify the blacklist can be updated independent of requesting containers
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(),
      Collections.singletonList(host), null);
  Assert.assertTrue(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(), null,
      Collections.singletonList(host));
  Assert.assertFalse(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  rm.stop();
}
 
Example 16
Source Project: big-c   Source File: TestAMRMClientOnRMRestart.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
  // Dispatch inline for test sanity
  return new EventHandler<SchedulerEvent>() {
    @Override
    public void handle(SchedulerEvent event) {
      scheduler.handle(event);
    }
  };
}
 
Example 17
Source Project: big-c   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
  // Dispatch inline for test sanity
  return new EventHandler<SchedulerEvent>() {
    @Override
    public void handle(SchedulerEvent event) {
      scheduler.handle(event);
    }
  };
}
 
Example 18
Source Project: incubator-myriad   Source File: YarnNodeCapacityManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void afterSchedulerEventHandled(SchedulerEvent event) {
  switch (event.getType()) {
    case NODE_ADDED:
      if (!(event instanceof NodeAddedSchedulerEvent)) {
        LOGGER.error("{} not an instance of {}", event.getClass().getName(), NodeAddedSchedulerEvent.class.getName());
        return;
      }

      NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent) event;
      NodeId nodeId = nodeAddedEvent.getAddedRMNode().getNodeID();
      String host = nodeId.getHost();

      SchedulerNode node = yarnScheduler.getSchedulerNode(nodeId);
      nodeStore.add(node);
      LOGGER.info("afterSchedulerEventHandled: NM registration from node {}", host);
      break;

    case NODE_UPDATE:
      if (!(event instanceof NodeUpdateSchedulerEvent)) {
        LOGGER.error("{} not an instance of {}", event.getClass().getName(), NodeUpdateSchedulerEvent.class.getName());
        return;
      }

      RMNode rmNode = ((NodeUpdateSchedulerEvent) event).getRMNode();
      handleContainerAllocation(rmNode);

      break;

    default:
      break;
  }
}
 
Example 19
Source Project: incubator-myriad   Source File: CompositeInterceptor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void beforeSchedulerEventHandled(SchedulerEvent event) {
  for (YarnSchedulerInterceptor interceptor : interceptors.values()) {
    final NodeId nodeId = getNodeIdForSchedulerEvent(event);
    if (nodeId != null && interceptor.getCallBackFilter().allowCallBacksForNode(nodeId)) {
      interceptor.beforeSchedulerEventHandled(event);
    }
  }
}
 
Example 20
Source Project: incubator-myriad   Source File: CompositeInterceptor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void afterSchedulerEventHandled(SchedulerEvent event) {
  for (YarnSchedulerInterceptor interceptor : interceptors.values()) {
    NodeId nodeId = getNodeIdForSchedulerEvent(event);
    if (nodeId != null && interceptor.getCallBackFilter().allowCallBacksForNode(nodeId)) {
      interceptor.afterSchedulerEventHandled(event);
    }
  }
}
 
Example 21
Source Project: incubator-myriad   Source File: CompositeInterceptor.java    License: Apache License 2.0 5 votes vote down vote up
private NodeId getNodeIdForSchedulerEvent(SchedulerEvent event) {
  switch (event.getType()) {
    case NODE_ADDED:
      return ((NodeAddedSchedulerEvent) event).getAddedRMNode().getNodeID();
    case NODE_REMOVED:
      return ((NodeRemovedSchedulerEvent) event).getRemovedRMNode().getNodeID();
    case NODE_UPDATE:
      return ((NodeUpdateSchedulerEvent) event).getRMNode().getNodeID();
    case NODE_RESOURCE_UPDATE:
      return ((NodeResourceUpdateSchedulerEvent) event).getRMNode().getNodeID();
  }
  return null;
}
 
Example 22
Source Project: hadoop   Source File: ResourceManager.java    License: Apache License 2.0 4 votes vote down vote up
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
  return new SchedulerEventDispatcher(this.scheduler);
}
 
Example 23
Source Project: hadoop   Source File: FifoScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void handle(SchedulerEvent event) {
  switch(event.getType()) {
  case NODE_ADDED:
  {
    NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)event;
    addNode(nodeAddedEvent.getAddedRMNode());
    recoverContainersOnNode(nodeAddedEvent.getContainerReports(),
      nodeAddedEvent.getAddedRMNode());

  }
  break;
  case NODE_REMOVED:
  {
    NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent)event;
    removeNode(nodeRemovedEvent.getRemovedRMNode());
  }
  break;
  case NODE_RESOURCE_UPDATE:
  {
    NodeResourceUpdateSchedulerEvent nodeResourceUpdatedEvent = 
        (NodeResourceUpdateSchedulerEvent)event;
    updateNodeResource(nodeResourceUpdatedEvent.getRMNode(),
      nodeResourceUpdatedEvent.getResourceOption());
  }
  break;
  case NODE_UPDATE:
  {
    NodeUpdateSchedulerEvent nodeUpdatedEvent = 
    (NodeUpdateSchedulerEvent)event;
    nodeUpdate(nodeUpdatedEvent.getRMNode());
  }
  break;
  case APP_ADDED:
  {
    AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event;
    addApplication(appAddedEvent.getApplicationId(),
      appAddedEvent.getQueue(), appAddedEvent.getUser(),
      appAddedEvent.getIsAppRecovering());
  }
  break;
  case APP_REMOVED:
  {
    AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event;
    doneApplication(appRemovedEvent.getApplicationID(),
      appRemovedEvent.getFinalState());
  }
  break;
  case APP_ATTEMPT_ADDED:
  {
    AppAttemptAddedSchedulerEvent appAttemptAddedEvent =
        (AppAttemptAddedSchedulerEvent) event;
    addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(),
      appAttemptAddedEvent.getTransferStateFromPreviousAttempt(),
      appAttemptAddedEvent.getIsAttemptRecovering());
  }
  break;
  case APP_ATTEMPT_REMOVED:
  {
    AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent =
        (AppAttemptRemovedSchedulerEvent) event;
    try {
      doneApplicationAttempt(
        appAttemptRemovedEvent.getApplicationAttemptID(),
        appAttemptRemovedEvent.getFinalAttemptState(),
        appAttemptRemovedEvent.getKeepContainersAcrossAppAttempts());
    } catch(IOException ie) {
      LOG.error("Unable to remove application "
          + appAttemptRemovedEvent.getApplicationAttemptID(), ie);
    }
  }
  break;
  case CONTAINER_EXPIRED:
  {
    ContainerExpiredSchedulerEvent containerExpiredEvent = 
        (ContainerExpiredSchedulerEvent) event;
    ContainerId containerid = containerExpiredEvent.getContainerId();
    completedContainer(getRMContainer(containerid), 
        SchedulerUtils.createAbnormalContainerStatus(
            containerid, 
            SchedulerUtils.EXPIRED_CONTAINER),
        RMContainerEventType.EXPIRE);
  }
  break;
  default:
    LOG.error("Invalid eventtype " + event.getType() + ". Ignoring!");
  }
}
 
Example 24
Source Project: hadoop   Source File: FairScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void handle(SchedulerEvent event) {
  switch (event.getType()) {
  case NODE_ADDED:
    if (!(event instanceof NodeAddedSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)event;
    addNode(nodeAddedEvent.getAddedRMNode());
    recoverContainersOnNode(nodeAddedEvent.getContainerReports(),
        nodeAddedEvent.getAddedRMNode());
    break;
  case NODE_REMOVED:
    if (!(event instanceof NodeRemovedSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent)event;
    removeNode(nodeRemovedEvent.getRemovedRMNode());
    break;
  case NODE_UPDATE:
    if (!(event instanceof NodeUpdateSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event;
    nodeUpdate(nodeUpdatedEvent.getRMNode());
    break;
  case APP_ADDED:
    if (!(event instanceof AppAddedSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event;
    String queueName =
        resolveReservationQueueName(appAddedEvent.getQueue(),
            appAddedEvent.getApplicationId(),
            appAddedEvent.getReservationID());
    if (queueName != null) {
      addApplication(appAddedEvent.getApplicationId(),
          queueName, appAddedEvent.getUser(),
          appAddedEvent.getIsAppRecovering());
    }
    break;
  case APP_REMOVED:
    if (!(event instanceof AppRemovedSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event;
    removeApplication(appRemovedEvent.getApplicationID(),
      appRemovedEvent.getFinalState());
    break;
  case NODE_RESOURCE_UPDATE:
    if (!(event instanceof NodeResourceUpdateSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    NodeResourceUpdateSchedulerEvent nodeResourceUpdatedEvent = 
        (NodeResourceUpdateSchedulerEvent)event;
    updateNodeResource(nodeResourceUpdatedEvent.getRMNode(),
          nodeResourceUpdatedEvent.getResourceOption());
    break;
  case APP_ATTEMPT_ADDED:
    if (!(event instanceof AppAttemptAddedSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    AppAttemptAddedSchedulerEvent appAttemptAddedEvent =
        (AppAttemptAddedSchedulerEvent) event;
    addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(),
      appAttemptAddedEvent.getTransferStateFromPreviousAttempt(),
      appAttemptAddedEvent.getIsAttemptRecovering());
    break;
  case APP_ATTEMPT_REMOVED:
    if (!(event instanceof AppAttemptRemovedSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent =
        (AppAttemptRemovedSchedulerEvent) event;
    removeApplicationAttempt(
        appAttemptRemovedEvent.getApplicationAttemptID(),
        appAttemptRemovedEvent.getFinalAttemptState(),
        appAttemptRemovedEvent.getKeepContainersAcrossAppAttempts());
    break;
  case CONTAINER_EXPIRED:
    if (!(event instanceof ContainerExpiredSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    ContainerExpiredSchedulerEvent containerExpiredEvent =
        (ContainerExpiredSchedulerEvent)event;
    ContainerId containerId = containerExpiredEvent.getContainerId();
    completedContainer(getRMContainer(containerId),
        SchedulerUtils.createAbnormalContainerStatus(
            containerId,
            SchedulerUtils.EXPIRED_CONTAINER),
        RMContainerEventType.EXPIRE);
    break;
  default:
    LOG.error("Unknown event arrived at FairScheduler: " + event.toString());
  }
}
 
Example 25
Source Project: hadoop   Source File: TestFifoScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout = 50000)
public void testHeadroom() throws Exception {
  
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  // Add a node
  RMNode n1 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2");
  fs.handle(new NodeAddedSchedulerEvent(n1));
  
  // Add two applications
  ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(
      appId1, 1);
  createMockRMApp(appAttemptId1, rm.getRMContext());
  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId1, "queue", "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
  fs.handle(attemptEvent);

  ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2);
  ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(
      appId2, 1);
  createMockRMApp(appAttemptId2, rm.getRMContext());
  SchedulerEvent appEvent2 =
      new AppAddedSchedulerEvent(appId2, "queue", "user");
  fs.handle(appEvent2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId2, false);
  fs.handle(attemptEvent2);

  List<ContainerId> emptyId = new ArrayList<ContainerId>();
  List<ResourceRequest> emptyAsk = new ArrayList<ResourceRequest>();

  // Set up resource requests
  
  // Ask for a 1 GB container for app 1
  List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
  ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
      ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
  fs.allocate(appAttemptId1, ask1, emptyId, null, null);

  // Ask for a 2 GB container for app 2
  List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
  ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
      ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1));
  fs.allocate(appAttemptId2, ask2, emptyId, null, null);
  
  // Trigger container assignment
  fs.handle(new NodeUpdateSchedulerEvent(n1));
  
  // Get the allocation for the applications and verify headroom
  Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
  Assert.assertEquals("Allocation headroom", 1 * GB,
      allocation1.getResourceLimit().getMemory());

  Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null);
  Assert.assertEquals("Allocation headroom", 1 * GB,
      allocation2.getResourceLimit().getMemory());

  rm.stop();
}
 
Example 26
Source Project: hadoop   Source File: TestRMAppTransitions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void handle(SchedulerEvent event) {
  lastSchedulerEvent = event;
}
 
Example 27
Source Project: hadoop   Source File: TestRMAppAttemptTransitions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void handle(SchedulerEvent event) {
  scheduler.handle(event);
}
 
Example 28
Source Project: hadoop   Source File: TestApplicationCleanup.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("resource")
@Test
public void testContainerCleanup() throws Exception {

  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);
  final DrainDispatcher dispatcher = new DrainDispatcher();
  MockRM rm = new MockRM() {
    @Override
    protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
      return new SchedulerEventDispatcher(this.scheduler) {
        @Override
        public void handle(SchedulerEvent event) {
          scheduler.handle(event);
        }
      };
    }

    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm.start();

  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5000);

  RMApp app = rm.submitApp(2000);

  //kick the scheduling
  nm1.nodeHeartbeat(true);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
  am.registerAppAttempt();
  
  //request for containers
  int request = 2;
  am.allocate("127.0.0.1" , 1000, request, 
      new ArrayList<ContainerId>());
  dispatcher.await();
  
  //kick the scheduler
  nm1.nodeHeartbeat(true);
  List<Container> conts = am.allocate(new ArrayList<ResourceRequest>(),
      new ArrayList<ContainerId>()).getAllocatedContainers();
  int contReceived = conts.size();
  int waitCount = 0;
  while (contReceived < request && waitCount++ < 200) {
    LOG.info("Got " + contReceived + " containers. Waiting to get "
             + request);
    Thread.sleep(100);
    conts = am.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers();
    dispatcher.await();
    contReceived += conts.size();
    nm1.nodeHeartbeat(true);
  }
  Assert.assertEquals(request, contReceived);

  // Release a container.
  ArrayList<ContainerId> release = new ArrayList<ContainerId>();
  release.add(conts.get(0).getId());
  am.allocate(new ArrayList<ResourceRequest>(), release);
  dispatcher.await();

  // Send one more heartbeat with a fake running container. This is to
  // simulate the situation that can happen if the NM reports that container
  // is running in the same heartbeat when the RM asks it to clean it up.
  Map<ApplicationId, List<ContainerStatus>> containerStatuses =
      new HashMap<ApplicationId, List<ContainerStatus>>();
  ArrayList<ContainerStatus> containerStatusList =
      new ArrayList<ContainerStatus>();
  containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(0)
    .getId(), ContainerState.RUNNING, "nothing", 0));
  containerStatuses.put(app.getApplicationId(), containerStatusList);

  NodeHeartbeatResponse resp = nm1.nodeHeartbeat(containerStatuses, true);
  waitForContainerCleanup(dispatcher, nm1, resp);

  // Now to test the case when RM already gave cleanup, and NM suddenly
  // realizes that the container is running.
  LOG.info("Testing container launch much after release and "
      + "NM getting cleanup");
  containerStatuses.clear();
  containerStatusList.clear();
  containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(0)
    .getId(), ContainerState.RUNNING, "nothing", 0));
  containerStatuses.put(app.getApplicationId(), containerStatusList);

  resp = nm1.nodeHeartbeat(containerStatuses, true);
  // The cleanup list won't be instantaneous as it is given out by scheduler
  // and not RMNodeImpl.
  waitForContainerCleanup(dispatcher, nm1, resp);

  rm.stop();
}
 
Example 29
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testBlackListNodes() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();

  String host = "127.0.0.1";
  RMNode node =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
  cs.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  RMAppAttemptMetrics attemptMetric =
      new RMAppAttemptMetrics(appAttemptId, rm.getRMContext());
  RMAppImpl app = mock(RMAppImpl.class);
  when(app.getApplicationId()).thenReturn(appId);
  RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
  when(attempt.getAppAttemptId()).thenReturn(appAttemptId);
  when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
  when(app.getCurrentAppAttempt()).thenReturn(attempt);

  rm.getRMContext().getRMApps().put(appId, app);

  SchedulerEvent addAppEvent =
      new AppAddedSchedulerEvent(appId, "default", "user");
  cs.handle(addAppEvent);
  SchedulerEvent addAttemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  cs.handle(addAttemptEvent);

  // Verify the blacklist can be updated independent of requesting containers
  cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(),
      Collections.singletonList(host), null);
  Assert.assertTrue(cs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(), null,
      Collections.singletonList(host));
  Assert.assertFalse(cs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  rm.stop();
}
 
Example 30
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testApplicationHeadRoom() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();

  ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(appId, 1);

  RMAppAttemptMetrics attemptMetric =
      new RMAppAttemptMetrics(appAttemptId, rm.getRMContext());
  RMAppImpl app = mock(RMAppImpl.class);
  when(app.getApplicationId()).thenReturn(appId);
  RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
  when(attempt.getAppAttemptId()).thenReturn(appAttemptId);
  when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
  when(app.getCurrentAppAttempt()).thenReturn(attempt);

  rm.getRMContext().getRMApps().put(appId, app);

  SchedulerEvent addAppEvent =
      new AppAddedSchedulerEvent(appId, "default", "user");
  cs.handle(addAppEvent);
  SchedulerEvent addAttemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  cs.handle(addAttemptEvent);

  Allocation allocate =
      cs.allocate(appAttemptId, Collections.<ResourceRequest> emptyList(),
          Collections.<ContainerId> emptyList(), null, null);

  Assert.assertNotNull(attempt);

  Assert
      .assertEquals(Resource.newInstance(0, 0), allocate.getResourceLimit());
  Assert.assertEquals(Resource.newInstance(0, 0),
      attemptMetric.getApplicationAttemptHeadroom());

  // Add a node to cluster
  Resource newResource = Resource.newInstance(4 * GB, 1);
  RMNode node = MockNodes.newNodeInfo(0, newResource, 1, "127.0.0.1");
  cs.handle(new NodeAddedSchedulerEvent(node));

  allocate =
      cs.allocate(appAttemptId, Collections.<ResourceRequest> emptyList(),
          Collections.<ContainerId> emptyList(), null, null);

  // All resources should be sent as headroom
  Assert.assertEquals(newResource, allocate.getResourceLimit());
  Assert.assertEquals(newResource,
      attemptMetric.getApplicationAttemptHeadroom());

  rm.stop();
}