org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ResourceManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void handle(SchedulerEvent event) {
  try {
    int qSize = eventQueue.size();
    if (qSize !=0 && qSize %1000 == 0) {
      LOG.info("Size of scheduler event-queue is " + qSize);
    }
    int remCapacity = eventQueue.remainingCapacity();
    if (remCapacity < 1000) {
      LOG.info("Very low remaining capacity on scheduler event queue: "
          + remCapacity);
    }
    this.eventQueue.put(event);
  } catch (InterruptedException e) {
    LOG.info("Interrupted. Trying to exit gracefully.");
  }
}
 
Example #2
Source File: ResourceManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void handle(SchedulerEvent event) {
  try {
    int qSize = eventQueue.size();
    if (qSize !=0 && qSize %1000 == 0) {
      LOG.info("Size of scheduler event-queue is " + qSize);
    }
    int remCapacity = eventQueue.remainingCapacity();
    if (remCapacity < 1000) {
      LOG.info("Very low remaining capacity on scheduler event queue: "
          + remCapacity);
    }
    this.eventQueue.put(event);
  } catch (InterruptedException e) {
    LOG.info("Interrupted. Trying to exit gracefully.");
  }
}
 
Example #3
Source File: LeastAMNodesFirstPolicy.java    From incubator-myriad with Apache License 2.0 6 votes vote down vote up
@Override
public void afterSchedulerEventHandled(SchedulerEvent event) {

  try {
    switch (event.getType()) {
      case NODE_UPDATE:
        onNodeUpdated((NodeUpdateSchedulerEvent) event);
        break;

      case NODE_REMOVED:
        onNodeRemoved((NodeRemovedSchedulerEvent) event);
        break;

      default:
        break;
    }
  } catch (ClassCastException e) {
    LOGGER.error("incorrect event object", e);
  }
}
 
Example #4
Source File: TestSchedulerUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static SchedulerApplication<SchedulerApplicationAttempt>
    verifyAppAddedAndRemovedFromScheduler(
        Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>> applications,
        EventHandler<SchedulerEvent> handler, String queueName)
        throws Exception {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appId, queueName, "user");
  handler.handle(appAddedEvent);
  SchedulerApplication<SchedulerApplicationAttempt> app =
      applications.get(appId);
  // verify application is added.
  Assert.assertNotNull(app);
  Assert.assertEquals("user", app.getUser());

  AppRemovedSchedulerEvent appRemoveEvent =
      new AppRemovedSchedulerEvent(appId, RMAppState.FINISHED);
  handler.handle(appRemoveEvent);
  Assert.assertNull(applications.get(appId));
  return app;
}
 
Example #5
Source File: TestSchedulerUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static SchedulerApplication<SchedulerApplicationAttempt>
    verifyAppAddedAndRemovedFromScheduler(
        Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>> applications,
        EventHandler<SchedulerEvent> handler, String queueName)
        throws Exception {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appId, queueName, "user");
  handler.handle(appAddedEvent);
  SchedulerApplication<SchedulerApplicationAttempt> app =
      applications.get(appId);
  // verify application is added.
  Assert.assertNotNull(app);
  Assert.assertEquals("user", app.getUser());

  AppRemovedSchedulerEvent appRemoveEvent =
      new AppRemovedSchedulerEvent(appId, RMAppState.FINISHED);
  handler.handle(appRemoveEvent);
  Assert.assertNull(applications.get(appId));
  return app;
}
 
Example #6
Source File: TestAMRMClientOnRMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
  // Dispatch inline for test sanity
  return new EventHandler<SchedulerEvent>() {
    @Override
    public void handle(SchedulerEvent event) {
      scheduler.handle(event);
    }
  };
}
 
Example #7
Source File: ResourceManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void run() {

  SchedulerEvent event;

  while (!stopped && !Thread.currentThread().isInterrupted()) {
    try {
      event = eventQueue.take();
    } catch (InterruptedException e) {
      LOG.error("Returning, interrupted : " + e);
      return; // TODO: Kill RM.
    }

    try {
      scheduler.handle(event);
    } catch (Throwable t) {
      // An error occurred, but we are shutting down anyway.
      // If it was an InterruptedException, the very act of 
      // shutdown could have caused it and is probably harmless.
      if (stopped) {
        LOG.warn("Exception during shutdown: ", t);
        break;
      }
      LOG.fatal("Error in handling event type " + event.getType()
          + " to the scheduler", t);
      if (shouldExitOnError
          && !ShutdownHookManager.get().isShutdownInProgress()) {
        LOG.info("Exiting, bbye..");
        System.exit(-1);
      }
    }
  }
}
 
Example #8
Source File: TestRMContainerAllocator.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
  // Dispatch inline for test sanity
  return new EventHandler<SchedulerEvent>() {
    @Override
    public void handle(SchedulerEvent event) {
      scheduler.handle(event);
    }
  };
}
 
Example #9
Source File: ResourceManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void run() {

  SchedulerEvent event;

  while (!stopped && !Thread.currentThread().isInterrupted()) {
    try {
      event = eventQueue.take();
    } catch (InterruptedException e) {
      LOG.error("Returning, interrupted : " + e);
      return; // TODO: Kill RM.
    }

    try {
      scheduler.handle(event);
    } catch (Throwable t) {
      // An error occurred, but we are shutting down anyway.
      // If it was an InterruptedException, the very act of 
      // shutdown could have caused it and is probably harmless.
      if (stopped) {
        LOG.warn("Exception during shutdown: ", t);
        break;
      }
      LOG.fatal("Error in handling event type " + event.getType()
          + " to the scheduler", t);
      if (shouldExitOnError
          && !ShutdownHookManager.get().isShutdownInProgress()) {
        LOG.info("Exiting, bbye..");
        System.exit(-1);
      }
    }
  }
}
 
Example #10
Source File: TestAMRMRPCNodeUpdates.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  dispatcher = new DrainDispatcher();
  this.rm = new MockRM() {
    @Override
    public void init(Configuration conf) {
      conf.set(
        CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
        "1.0");
      super.init(conf);
    }
    @Override
    protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
      return new SchedulerEventDispatcher(this.scheduler) {
        @Override
        public void handle(SchedulerEvent event) {
          scheduler.handle(event);
        }
      };
    }

    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm.start();
  amService = rm.getApplicationMasterService();
}
 
Example #11
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout=5000)
public void testAppAttemptMetrics() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  
  FifoScheduler scheduler = new FifoScheduler();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext rmContext = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, writer, scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  Configuration conf = new Configuration();
  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);
  QueueMetrics metrics = scheduler.getRootQueueMetrics();
  int beforeAppsSubmitted = metrics.getAppsSubmitted();

  ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
  scheduler.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent2);

  int afterAppsSubmitted = metrics.getAppsSubmitted();
  Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
  scheduler.stop();
}
 
Example #12
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testBlackListNodes() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  String host = "127.0.0.1";
  RMNode node =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
  fs.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  createMockRMApp(appAttemptId, rm.getRMContext());

  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId, "default",
        "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  fs.handle(attemptEvent);

  // Verify the blacklist can be updated independent of requesting containers
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(),
      Collections.singletonList(host), null);
  Assert.assertTrue(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(), null,
      Collections.singletonList(host));
  Assert.assertFalse(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  rm.stop();
}
 
Example #13
Source File: TestAMRMClientOnRMRestart.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
  // Dispatch inline for test sanity
  return new EventHandler<SchedulerEvent>() {
    @Override
    public void handle(SchedulerEvent event) {
      scheduler.handle(event);
    }
  };
}
 
Example #14
Source File: TestRMContainerAllocator.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
  // Dispatch inline for test sanity
  return new EventHandler<SchedulerEvent>() {
    @Override
    public void handle(SchedulerEvent event) {
      scheduler.handle(event);
    }
  };
}
 
Example #15
Source File: YarnNodeCapacityManager.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
@Override
public void afterSchedulerEventHandled(SchedulerEvent event) {
  switch (event.getType()) {
    case NODE_ADDED:
      if (!(event instanceof NodeAddedSchedulerEvent)) {
        LOGGER.error("{} not an instance of {}", event.getClass().getName(), NodeAddedSchedulerEvent.class.getName());
        return;
      }

      NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent) event;
      NodeId nodeId = nodeAddedEvent.getAddedRMNode().getNodeID();
      String host = nodeId.getHost();

      SchedulerNode node = yarnScheduler.getSchedulerNode(nodeId);
      nodeStore.add(node);
      LOGGER.info("afterSchedulerEventHandled: NM registration from node {}", host);
      break;

    case NODE_UPDATE:
      if (!(event instanceof NodeUpdateSchedulerEvent)) {
        LOGGER.error("{} not an instance of {}", event.getClass().getName(), NodeUpdateSchedulerEvent.class.getName());
        return;
      }

      RMNode rmNode = ((NodeUpdateSchedulerEvent) event).getRMNode();
      handleContainerAllocation(rmNode);

      break;

    default:
      break;
  }
}
 
Example #16
Source File: CompositeInterceptor.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
@Override
public void beforeSchedulerEventHandled(SchedulerEvent event) {
  for (YarnSchedulerInterceptor interceptor : interceptors.values()) {
    final NodeId nodeId = getNodeIdForSchedulerEvent(event);
    if (nodeId != null && interceptor.getCallBackFilter().allowCallBacksForNode(nodeId)) {
      interceptor.beforeSchedulerEventHandled(event);
    }
  }
}
 
Example #17
Source File: CompositeInterceptor.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
@Override
public void afterSchedulerEventHandled(SchedulerEvent event) {
  for (YarnSchedulerInterceptor interceptor : interceptors.values()) {
    NodeId nodeId = getNodeIdForSchedulerEvent(event);
    if (nodeId != null && interceptor.getCallBackFilter().allowCallBacksForNode(nodeId)) {
      interceptor.afterSchedulerEventHandled(event);
    }
  }
}
 
Example #18
Source File: CompositeInterceptor.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
private NodeId getNodeIdForSchedulerEvent(SchedulerEvent event) {
  switch (event.getType()) {
    case NODE_ADDED:
      return ((NodeAddedSchedulerEvent) event).getAddedRMNode().getNodeID();
    case NODE_REMOVED:
      return ((NodeRemovedSchedulerEvent) event).getRemovedRMNode().getNodeID();
    case NODE_UPDATE:
      return ((NodeUpdateSchedulerEvent) event).getRMNode().getNodeID();
    case NODE_RESOURCE_UPDATE:
      return ((NodeResourceUpdateSchedulerEvent) event).getRMNode().getNodeID();
  }
  return null;
}
 
Example #19
Source File: TestAMRMRPCNodeUpdates.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  dispatcher = new DrainDispatcher();
  this.rm = new MockRM() {
    @Override
    public void init(Configuration conf) {
      conf.set(
        CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
        "1.0");
      super.init(conf);
    }
    @Override
    protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
      return new SchedulerEventDispatcher(this.scheduler) {
        @Override
        public void handle(SchedulerEvent event) {
          scheduler.handle(event);
        }
      };
    }

    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm.start();
  amService = rm.getApplicationMasterService();
}
 
Example #20
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testBlackListNodes() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  String host = "127.0.0.1";
  RMNode node =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
  fs.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  createMockRMApp(appAttemptId, rm.getRMContext());

  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId, "default",
        "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  fs.handle(attemptEvent);

  // Verify the blacklist can be updated independent of requesting containers
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(),
      Collections.singletonList(host), null);
  Assert.assertTrue(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(), null,
      Collections.singletonList(host));
  Assert.assertFalse(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  rm.stop();
}
 
Example #21
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=5000)
public void testAppAttemptMetrics() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  
  FifoScheduler scheduler = new FifoScheduler();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext rmContext = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, writer, scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  Configuration conf = new Configuration();
  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);
  QueueMetrics metrics = scheduler.getRootQueueMetrics();
  int beforeAppsSubmitted = metrics.getAppsSubmitted();

  ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
  scheduler.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent2);

  int afterAppsSubmitted = metrics.getAppsSubmitted();
  Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
  scheduler.stop();
}
 
Example #22
Source File: TestRMNodeTransitions.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public void handle(SchedulerEvent event) {
  scheduler.handle(event);
}
 
Example #23
Source File: MyriadFifoScheduler.java    From incubator-myriad with Apache License 2.0 4 votes vote down vote up
@Override
public synchronized void handle(SchedulerEvent event) {
  this.yarnSchedulerInterceptor.beforeSchedulerEventHandled(event);
  super.handle(event);
  this.yarnSchedulerInterceptor.afterSchedulerEventHandled(event);
}
 
Example #24
Source File: MyriadCapacityScheduler.java    From incubator-myriad with Apache License 2.0 4 votes vote down vote up
@Override
public synchronized void handle(SchedulerEvent event) {
  this.yarnSchedulerInterceptor.beforeSchedulerEventHandled(event);
  super.handle(event);
  this.yarnSchedulerInterceptor.afterSchedulerEventHandled(event);
}
 
Example #25
Source File: MyriadFairScheduler.java    From incubator-myriad with Apache License 2.0 4 votes vote down vote up
@Override
public synchronized void handle(SchedulerEvent event) {
  this.yarnSchedulerInterceptor.beforeSchedulerEventHandled(event);
  super.handle(event);
  this.yarnSchedulerInterceptor.afterSchedulerEventHandled(event);
}
 
Example #26
Source File: ResourceManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
  return new SchedulerEventDispatcher(this.scheduler);
}
 
Example #27
Source File: FifoScheduler.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public void handle(SchedulerEvent event) {
  switch(event.getType()) {
  case NODE_ADDED:
  {
    NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)event;
    addNode(nodeAddedEvent.getAddedRMNode());
    recoverContainersOnNode(nodeAddedEvent.getContainerReports(),
      nodeAddedEvent.getAddedRMNode());

  }
  break;
  case NODE_REMOVED:
  {
    NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent)event;
    removeNode(nodeRemovedEvent.getRemovedRMNode());
  }
  break;
  case NODE_RESOURCE_UPDATE:
  {
    NodeResourceUpdateSchedulerEvent nodeResourceUpdatedEvent = 
        (NodeResourceUpdateSchedulerEvent)event;
    updateNodeResource(nodeResourceUpdatedEvent.getRMNode(),
      nodeResourceUpdatedEvent.getResourceOption());
  }
  break;
  case NODE_UPDATE:
  {
    NodeUpdateSchedulerEvent nodeUpdatedEvent = 
    (NodeUpdateSchedulerEvent)event;
    nodeUpdate(nodeUpdatedEvent.getRMNode());
  }
  break;
  case APP_ADDED:
  {
    AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event;
    addApplication(appAddedEvent.getApplicationId(),
      appAddedEvent.getQueue(), appAddedEvent.getUser(),
      appAddedEvent.getIsAppRecovering());
  }
  break;
  case APP_REMOVED:
  {
    AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event;
    doneApplication(appRemovedEvent.getApplicationID(),
      appRemovedEvent.getFinalState());
  }
  break;
  case APP_ATTEMPT_ADDED:
  {
    AppAttemptAddedSchedulerEvent appAttemptAddedEvent =
        (AppAttemptAddedSchedulerEvent) event;
    addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(),
      appAttemptAddedEvent.getTransferStateFromPreviousAttempt(),
      appAttemptAddedEvent.getIsAttemptRecovering());
  }
  break;
  case APP_ATTEMPT_REMOVED:
  {
    AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent =
        (AppAttemptRemovedSchedulerEvent) event;
    try {
      doneApplicationAttempt(
        appAttemptRemovedEvent.getApplicationAttemptID(),
        appAttemptRemovedEvent.getFinalAttemptState(),
        appAttemptRemovedEvent.getKeepContainersAcrossAppAttempts());
    } catch(IOException ie) {
      LOG.error("Unable to remove application "
          + appAttemptRemovedEvent.getApplicationAttemptID(), ie);
    }
  }
  break;
  case CONTAINER_EXPIRED:
  {
    ContainerExpiredSchedulerEvent containerExpiredEvent = 
        (ContainerExpiredSchedulerEvent) event;
    ContainerId containerid = containerExpiredEvent.getContainerId();
    completedContainer(getRMContainer(containerid), 
        SchedulerUtils.createAbnormalContainerStatus(
            containerid, 
            SchedulerUtils.EXPIRED_CONTAINER),
        RMContainerEventType.EXPIRE);
  }
  break;
  default:
    LOG.error("Invalid eventtype " + event.getType() + ". Ignoring!");
  }
}
 
Example #28
Source File: FairScheduler.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public void handle(SchedulerEvent event) {
  switch (event.getType()) {
  case NODE_ADDED:
    if (!(event instanceof NodeAddedSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)event;
    addNode(nodeAddedEvent.getAddedRMNode());
    recoverContainersOnNode(nodeAddedEvent.getContainerReports(),
        nodeAddedEvent.getAddedRMNode());
    break;
  case NODE_REMOVED:
    if (!(event instanceof NodeRemovedSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent)event;
    removeNode(nodeRemovedEvent.getRemovedRMNode());
    break;
  case NODE_UPDATE:
    if (!(event instanceof NodeUpdateSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event;
    nodeUpdate(nodeUpdatedEvent.getRMNode());
    break;
  case APP_ADDED:
    if (!(event instanceof AppAddedSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent) event;
    String queueName =
        resolveReservationQueueName(appAddedEvent.getQueue(),
            appAddedEvent.getApplicationId(),
            appAddedEvent.getReservationID());
    if (queueName != null) {
      addApplication(appAddedEvent.getApplicationId(),
          queueName, appAddedEvent.getUser(),
          appAddedEvent.getIsAppRecovering());
    }
    break;
  case APP_REMOVED:
    if (!(event instanceof AppRemovedSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event;
    removeApplication(appRemovedEvent.getApplicationID(),
      appRemovedEvent.getFinalState());
    break;
  case NODE_RESOURCE_UPDATE:
    if (!(event instanceof NodeResourceUpdateSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    NodeResourceUpdateSchedulerEvent nodeResourceUpdatedEvent = 
        (NodeResourceUpdateSchedulerEvent)event;
    updateNodeResource(nodeResourceUpdatedEvent.getRMNode(),
          nodeResourceUpdatedEvent.getResourceOption());
    break;
  case APP_ATTEMPT_ADDED:
    if (!(event instanceof AppAttemptAddedSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    AppAttemptAddedSchedulerEvent appAttemptAddedEvent =
        (AppAttemptAddedSchedulerEvent) event;
    addApplicationAttempt(appAttemptAddedEvent.getApplicationAttemptId(),
      appAttemptAddedEvent.getTransferStateFromPreviousAttempt(),
      appAttemptAddedEvent.getIsAttemptRecovering());
    break;
  case APP_ATTEMPT_REMOVED:
    if (!(event instanceof AppAttemptRemovedSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent =
        (AppAttemptRemovedSchedulerEvent) event;
    removeApplicationAttempt(
        appAttemptRemovedEvent.getApplicationAttemptID(),
        appAttemptRemovedEvent.getFinalAttemptState(),
        appAttemptRemovedEvent.getKeepContainersAcrossAppAttempts());
    break;
  case CONTAINER_EXPIRED:
    if (!(event instanceof ContainerExpiredSchedulerEvent)) {
      throw new RuntimeException("Unexpected event type: " + event);
    }
    ContainerExpiredSchedulerEvent containerExpiredEvent =
        (ContainerExpiredSchedulerEvent)event;
    ContainerId containerId = containerExpiredEvent.getContainerId();
    completedContainer(getRMContainer(containerId),
        SchedulerUtils.createAbnormalContainerStatus(
            containerId,
            SchedulerUtils.EXPIRED_CONTAINER),
        RMContainerEventType.EXPIRE);
    break;
  default:
    LOG.error("Unknown event arrived at FairScheduler: " + event.toString());
  }
}
 
Example #29
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test (timeout = 50000)
public void testHeadroom() throws Exception {
  
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  // Add a node
  RMNode n1 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2");
  fs.handle(new NodeAddedSchedulerEvent(n1));
  
  // Add two applications
  ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(
      appId1, 1);
  createMockRMApp(appAttemptId1, rm.getRMContext());
  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId1, "queue", "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
  fs.handle(attemptEvent);

  ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2);
  ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(
      appId2, 1);
  createMockRMApp(appAttemptId2, rm.getRMContext());
  SchedulerEvent appEvent2 =
      new AppAddedSchedulerEvent(appId2, "queue", "user");
  fs.handle(appEvent2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId2, false);
  fs.handle(attemptEvent2);

  List<ContainerId> emptyId = new ArrayList<ContainerId>();
  List<ResourceRequest> emptyAsk = new ArrayList<ResourceRequest>();

  // Set up resource requests
  
  // Ask for a 1 GB container for app 1
  List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
  ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
      ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
  fs.allocate(appAttemptId1, ask1, emptyId, null, null);

  // Ask for a 2 GB container for app 2
  List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
  ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
      ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1));
  fs.allocate(appAttemptId2, ask2, emptyId, null, null);
  
  // Trigger container assignment
  fs.handle(new NodeUpdateSchedulerEvent(n1));
  
  // Get the allocation for the applications and verify headroom
  Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
  Assert.assertEquals("Allocation headroom", 1 * GB,
      allocation1.getResourceLimit().getMemory());

  Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null);
  Assert.assertEquals("Allocation headroom", 1 * GB,
      allocation2.getResourceLimit().getMemory());

  rm.stop();
}
 
Example #30
Source File: TestRMAppTransitions.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public void handle(SchedulerEvent event) {
  lastSchedulerEvent = event;
}