Java Code Examples for org.apache.hadoop.yarn.event.DrainDispatcher#start()

@Before
public void setup() {
  dispatcher = new DrainDispatcher();
  dispatcher.init(new Configuration());
  dispatcher.start();
  eventHandler = dispatcher.getEventHandler();
}
 

@Test (timeout=10000)
public void testFailAbortDoesntHang() throws IOException {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  conf.set(MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS, "1000");
  
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = Mockito.mock(OutputCommitter.class);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();
  //Job has only 1 mapper task. No reducers
  conf.setInt(MRJobConfig.NUM_REDUCES, 0);
  conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 1);
  JobImpl job = createRunningStubbedJob(conf, dispatcher, 1, null);

  //Fail / finish all the tasks. This should land the JobImpl directly in the
  //FAIL_ABORT state
  for(Task t: job.tasks.values()) {
    TaskImpl task = (TaskImpl) t;
    task.handle(new TaskEvent(task.getID(), TaskEventType.T_SCHEDULE));
    for(TaskAttempt ta: task.getAttempts().values()) {
      task.handle(new TaskTAttemptEvent(ta.getID(),
        TaskEventType.T_ATTEMPT_FAILED));
    }
  }

  dispatcher.await();
  //Verify abortJob is called once and the job failed
  Mockito.verify(committer, Mockito.timeout(2000).times(1))
    .abortJob((JobContext) Mockito.any(), (State) Mockito.any());
  assertJobState(job, JobStateInternal.FAILED);

  dispatcher.stop();
}
 

@Test (timeout=10000)
public void testFailAbortDoesntHang() throws IOException {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  conf.set(MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS, "1000");
  
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = Mockito.mock(OutputCommitter.class);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();
  //Job has only 1 mapper task. No reducers
  conf.setInt(MRJobConfig.NUM_REDUCES, 0);
  conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, 1);
  JobImpl job = createRunningStubbedJob(conf, dispatcher, 1, null);

  //Fail / finish all the tasks. This should land the JobImpl directly in the
  //FAIL_ABORT state
  for(Task t: job.tasks.values()) {
    TaskImpl task = (TaskImpl) t;
    task.handle(new TaskEvent(task.getID(), TaskEventType.T_SCHEDULE));
    for(TaskAttempt ta: task.getAttempts().values()) {
      task.handle(new TaskTAttemptEvent(ta.getID(),
        TaskEventType.T_ATTEMPT_FAILED));
    }
  }

  dispatcher.await();
  //Verify abortJob is called once and the job failed
  Mockito.verify(committer, Mockito.timeout(2000).times(1))
    .abortJob((JobContext) Mockito.any(), (State) Mockito.any());
  assertJobState(job, JobStateInternal.FAILED);

  dispatcher.stop();
}
 

@Test( timeout = 10000)
@SuppressWarnings("unchecked") // mocked generics
public void testLocalizerRunnerException() throws Exception {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
  dispatcher.register(ApplicationEventType.class, applicationBus);
  EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
  dispatcher.register(ContainerEventType.class, containerBus);

  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
  dirsHandlerSpy.init(conf);

  DeletionService delServiceReal = new DeletionService(exec);
  DeletionService delService = spy(delServiceReal);
  delService.init(new Configuration());
  delService.start();

  ResourceLocalizationService rawService =
      new ResourceLocalizationService(dispatcher, exec, delService,
      dirsHandlerSpy, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  try {
    spyService.init(conf);
    spyService.start();

    // init application
    final Application app = mock(Application.class);
    final ApplicationId appId =
        BuilderUtils.newApplicationId(314159265358979L, 3);
    when(app.getUser()).thenReturn("user0");
    when(app.getAppId()).thenReturn(appId);
    spyService.handle(new ApplicationLocalizationEvent(
        LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
    dispatcher.await();

    Random r = new Random();
    long seed = r.nextLong();
    System.out.println("SEED: " + seed);
    r.setSeed(seed);
    final Container c = getMockContainer(appId, 42, "user0");
    final LocalResource resource1 = getPrivateMockedResource(r);
    System.out.println("Here 4");
    
    final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
      new HashMap<LocalResourceVisibility, 
                  Collection<LocalResourceRequest>>();
    List<LocalResourceRequest> privateResourceList =
        new ArrayList<LocalResourceRequest>();
    privateResourceList.add(req1);
    rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);

    final Constructor<?>[] constructors =
        FSError.class.getDeclaredConstructors();
    constructors[0].setAccessible(true);
    FSError fsError =
        (FSError) constructors[0].newInstance(new IOException("Disk Error"));

    Mockito
      .doThrow(fsError)
      .when(dirsHandlerSpy)
      .getLocalPathForWrite(isA(String.class));
    spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
    Thread.sleep(1000);
    dispatcher.await();
    // Verify if ContainerResourceFailedEvent is invoked on FSError
    verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
  } finally {
    spyService.stop();
    dispatcher.stop();
    delService.stop();
  }
}
 

@Before
public void setup() {
  conf = new Configuration();
  conf.setBoolean(TezConfiguration.TEZ_AM_CONTAINER_REUSE_ENABLED, false);
  appAttemptId = ApplicationAttemptId.newInstance(
      ApplicationId.newInstance(100, 1), 1);
  dagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 1);
  Assert.assertNotNull(dagId);
  dagPlan = createTestDAGPlan();
  dispatcher = new DrainDispatcher();
  fsTokens = new Credentials();
  appContext = mock(AppContext.class);
  historyEventHandler = mock(HistoryEventHandler.class);
  doReturn(conf).when(appContext).getAMConf();
  doReturn(appAttemptId).when(appContext).getApplicationAttemptId();
  doReturn(appAttemptId.getApplicationId()).when(appContext).getApplicationID();
  doReturn(dagId).when(appContext).getCurrentDAGID();
  doReturn(historyEventHandler).when(appContext).getHistoryHandler();
  dag = new DAGImpl(dagId, conf, dagPlan,
      dispatcher.getEventHandler(),  taskAttemptListener,
      fsTokens, clock, "user", thh, appContext);
  doReturn(dag).when(appContext).getCurrentDAG();
  mrrAppContext = mock(AppContext.class);
  mrrDagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 2);
  mrrDagPlan = createTestMRRDAGPlan();
  mrrDag = new DAGImpl(mrrDagId, conf, mrrDagPlan,
      dispatcher.getEventHandler(),  taskAttemptListener,
      fsTokens, clock, "user", thh,
      mrrAppContext);
  doReturn(conf).when(mrrAppContext).getAMConf();
  doReturn(mrrDag).when(mrrAppContext).getCurrentDAG();
  doReturn(appAttemptId).when(mrrAppContext).getApplicationAttemptId();
  doReturn(appAttemptId.getApplicationId()).when(mrrAppContext).getApplicationID();
  doReturn(historyEventHandler).when(mrrAppContext).getHistoryHandler();
  groupAppContext = mock(AppContext.class);
  groupDagId = TezDAGID.getInstance(appAttemptId.getApplicationId(), 3);
  groupDagPlan = createGroupDAGPlan();
  groupDag = new DAGImpl(groupDagId, conf, groupDagPlan,
      dispatcher.getEventHandler(),  taskAttemptListener,
      fsTokens, clock, "user", thh,
      groupAppContext);
  doReturn(conf).when(groupAppContext).getAMConf();
  doReturn(groupDag).when(groupAppContext).getCurrentDAG();
  doReturn(appAttemptId).when(groupAppContext).getApplicationAttemptId();
  doReturn(appAttemptId.getApplicationId())
      .when(groupAppContext).getApplicationID();
  doReturn(historyEventHandler).when(groupAppContext).getHistoryHandler();
  taskEventDispatcher = new TaskEventDispatcher();
  dispatcher.register(TaskEventType.class, taskEventDispatcher);
  taskAttemptEventDispatcher = new TaskAttemptEventDispatcher();
  dispatcher.register(TaskAttemptEventType.class, taskAttemptEventDispatcher);
  vertexEventDispatcher = new VertexEventDispatcher();
  dispatcher.register(VertexEventType.class, vertexEventDispatcher);
  dagEventDispatcher = new DagEventDispatcher();
  dispatcher.register(DAGEventType.class, dagEventDispatcher);
  dagFinishEventHandler = new DAGFinishEventHandler();
  dispatcher.register(DAGAppMasterEventType.class, dagFinishEventHandler);
  dispatcher.register(TaskEventType.class, new TaskEventHandler());
  dispatcher.init(conf);
  dispatcher.start();
}
 

@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
  if (isSecurityEnabled) {
    authMethod = AuthenticationMethod.KERBEROS;
  }
  SecurityUtil.setAuthenticationMethod(authMethod, conf);
  UserGroupInformation.setConfiguration(conf);

  rmDispatcher = new DrainDispatcher();
  ContainerAllocationExpirer containerAllocationExpirer = 
      mock(ContainerAllocationExpirer.class);
  AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
  AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class);
  store = mock(RMStateStore.class);
  writer = mock(RMApplicationHistoryWriter.class);
  DelegationTokenRenewer renewer = mock(DelegationTokenRenewer.class);
  RMContext realRMContext = 
      new RMContextImpl(rmDispatcher,
        containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
        renewer, new AMRMTokenSecretManager(conf, this.rmContext),
        new RMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInRM(conf),
        new ClientToAMTokenSecretManagerInRM(),
        writer);
  ((RMContextImpl)realRMContext).setStateStore(store);
  publisher = mock(SystemMetricsPublisher.class);
  ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher);

  this.rmContext = spy(realRMContext);

  ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
  doReturn(null).when(resourceScheduler)
            .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
  doReturn(resourceScheduler).when(rmContext).getScheduler();

  rmDispatcher.register(RMAppAttemptEventType.class,
      new TestApplicationAttemptEventDispatcher(this.rmContext));

  rmDispatcher.register(RMAppEventType.class,
      new TestApplicationEventDispatcher(rmContext));
  
  rmDispatcher.register(RMAppManagerEventType.class,
      new TestApplicationManagerEventDispatcher());
  
  schedulerDispatcher = new TestSchedulerEventDispatcher();
  rmDispatcher.register(SchedulerEventType.class,
      schedulerDispatcher);
  
  rmDispatcher.init(conf);
  rmDispatcher.start();
}
 

@Test
public void testExpireWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify EXPIRE and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.EXPIRED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.EXPIRE));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  verify(writer, never()).containerFinished(any(RMContainer.class));
  verify(publisher, never()).containerFinished(any(RMContainer.class),
      anyLong());
}
 

@Test
public void testReleaseWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);
  ConcurrentMap<ApplicationId, RMApp> rmApps =
      spy(new ConcurrentHashMap<ApplicationId, RMApp>());
  RMApp rmApp = mock(RMApp.class);
  when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
  Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any());

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getRMApps()).thenReturn(rmApps);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify RELEASED and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.RELEASED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.RELEASED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
  assertEquals(SchedulerUtils.RELEASED_CONTAINER,
      rmContainer.getDiagnosticsInfo());
  assertEquals(ContainerExitStatus.ABORTED,
      rmContainer.getContainerExitStatus());
  assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState());
  verify(writer).containerFinished(any(RMContainer.class));
  verify(publisher).containerFinished(any(RMContainer.class), anyLong());

  ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor
      .forClass(RMAppAttemptContainerFinishedEvent.class);
  verify(appAttemptEventHandler).handle(captor.capture());
  RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue();
  assertEquals(appAttemptId, cfEvent.getApplicationAttemptId());
  assertEquals(containerStatus, cfEvent.getContainerStatus());
  assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType());
  
  // In RELEASED state. A FINIHSED event may come in.
  rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils
      .createAbnormalContainerStatus(containerId, "FinishedContainer"),
      RMContainerEventType.FINISHED));
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
}
 

private DrainDispatcher createDispatcher() {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(this.conf);
  dispatcher.start();
  return dispatcher;
}
 

private DrainDispatcher createDispatcher(Configuration conf) {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  return dispatcher;
}
 

WrappedApplication(int id, long timestamp, String user, int numContainers) {
  Configuration conf = new Configuration();
  
  dispatcher = new DrainDispatcher();
  containerTokenIdentifierMap =
      new HashMap<ContainerId, ContainerTokenIdentifier>();
  dispatcher.init(conf);

  localizerBus = mock(EventHandler.class);
  launcherBus = mock(EventHandler.class);
  monitorBus = mock(EventHandler.class);
  auxBus = mock(EventHandler.class);
  containerBus = mock(EventHandler.class);
  logAggregationBus = mock(EventHandler.class);

  dispatcher.register(LocalizationEventType.class, localizerBus);
  dispatcher.register(ContainersLauncherEventType.class, launcherBus);
  dispatcher.register(ContainersMonitorEventType.class, monitorBus);
  dispatcher.register(AuxServicesEventType.class, auxBus);
  dispatcher.register(ContainerEventType.class, containerBus);
  dispatcher.register(LogHandlerEventType.class, logAggregationBus);

  nmTokenSecretMgr = mock(NMTokenSecretManagerInNM.class);

  context = mock(Context.class);
  
  when(context.getContainerTokenSecretManager()).thenReturn(
    new NMContainerTokenSecretManager(conf));
  when(context.getApplicationACLsManager()).thenReturn(
    new ApplicationACLsManager(conf));
  when(context.getNMTokenSecretManager()).thenReturn(nmTokenSecretMgr);
  
  // Setting master key
  MasterKey masterKey = new MasterKeyPBImpl();
  masterKey.setKeyId(123);
  masterKey.setBytes(ByteBuffer.wrap(new byte[] { (new Integer(123)
    .byteValue()) }));
  context.getContainerTokenSecretManager().setMasterKey(masterKey);
  
  this.user = user;
  this.appId = BuilderUtils.newApplicationId(timestamp, id);

  app = new ApplicationImpl(dispatcher, this.user, appId, null, context);
  containers = new ArrayList<Container>();
  for (int i = 0; i < numContainers; i++) {
    Container container = createMockedContainer(this.appId, i);
    containers.add(container);
    long currentTime = System.currentTimeMillis();
    ContainerTokenIdentifier identifier =
        new ContainerTokenIdentifier(container.getContainerId(), "", "",
          null, currentTime + 2000, masterKey.getKeyId(), currentTime,
          Priority.newInstance(0), 0);
    containerTokenIdentifierMap
      .put(identifier.getContainerID(), identifier);
    context.getContainerTokenSecretManager().startContainerSuccessful(
      identifier);
    Assert.assertFalse(context.getContainerTokenSecretManager()
      .isValidStartContainerRequest(identifier));
  }

  dispatcher.start();
}
 

private DrainDispatcher createDispatcher(Configuration conf) {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  return dispatcher;
}
 

private DrainDispatcher createDispatcher(Configuration conf) {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  return dispatcher;
}
 

@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
  if (isSecurityEnabled) {
    authMethod = AuthenticationMethod.KERBEROS;
  }
  SecurityUtil.setAuthenticationMethod(authMethod, conf);
  UserGroupInformation.setConfiguration(conf);

  rmDispatcher = new DrainDispatcher();
  ContainerAllocationExpirer containerAllocationExpirer = 
      mock(ContainerAllocationExpirer.class);
  AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
  AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class);
  store = mock(RMStateStore.class);
  writer = mock(RMApplicationHistoryWriter.class);
  DelegationTokenRenewer renewer = mock(DelegationTokenRenewer.class);
  RMContext realRMContext = 
      new RMContextImpl(rmDispatcher,
        containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
        renewer, new AMRMTokenSecretManager(conf, this.rmContext),
        new RMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInRM(conf),
        new ClientToAMTokenSecretManagerInRM(),
        writer);
  ((RMContextImpl)realRMContext).setStateStore(store);
  publisher = mock(SystemMetricsPublisher.class);
  ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher);

  this.rmContext = spy(realRMContext);

  ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
  doReturn(null).when(resourceScheduler)
            .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
  doReturn(resourceScheduler).when(rmContext).getScheduler();

  rmDispatcher.register(RMAppAttemptEventType.class,
      new TestApplicationAttemptEventDispatcher(this.rmContext));

  rmDispatcher.register(RMAppEventType.class,
      new TestApplicationEventDispatcher(rmContext));
  
  rmDispatcher.register(RMAppManagerEventType.class,
      new TestApplicationManagerEventDispatcher());
  
  schedulerDispatcher = new TestSchedulerEventDispatcher();
  rmDispatcher.register(SchedulerEventType.class,
      schedulerDispatcher);
  
  rmDispatcher.init(conf);
  rmDispatcher.start();
}
 

@Test
public void testExpireWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify EXPIRE and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.EXPIRED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.EXPIRE));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  verify(writer, never()).containerFinished(any(RMContainer.class));
  verify(publisher, never()).containerFinished(any(RMContainer.class),
      anyLong());
}
 

@Test
public void testReleaseWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);
  ConcurrentMap<ApplicationId, RMApp> rmApps =
      spy(new ConcurrentHashMap<ApplicationId, RMApp>());
  RMApp rmApp = mock(RMApp.class);
  when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
  Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any());

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getRMApps()).thenReturn(rmApps);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify RELEASED and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.RELEASED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.RELEASED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
  assertEquals(SchedulerUtils.RELEASED_CONTAINER,
      rmContainer.getDiagnosticsInfo());
  assertEquals(ContainerExitStatus.ABORTED,
      rmContainer.getContainerExitStatus());
  assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState());
  verify(writer).containerFinished(any(RMContainer.class));
  verify(publisher).containerFinished(any(RMContainer.class), anyLong());

  ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor
      .forClass(RMAppAttemptContainerFinishedEvent.class);
  verify(appAttemptEventHandler).handle(captor.capture());
  RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue();
  assertEquals(appAttemptId, cfEvent.getApplicationAttemptId());
  assertEquals(containerStatus, cfEvent.getContainerStatus());
  assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType());
  
  // In RELEASED state. A FINIHSED event may come in.
  rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils
      .createAbnormalContainerStatus(containerId, "FinishedContainer"),
      RMContainerEventType.FINISHED));
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
}
 

private DrainDispatcher createDispatcher() {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(this.conf);
  dispatcher.start();
  return dispatcher;
}
 

private DrainDispatcher createDispatcher(Configuration conf) {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  return dispatcher;
}
 

WrappedApplication(int id, long timestamp, String user, int numContainers) {
  Configuration conf = new Configuration();
  
  dispatcher = new DrainDispatcher();
  containerTokenIdentifierMap =
      new HashMap<ContainerId, ContainerTokenIdentifier>();
  dispatcher.init(conf);

  localizerBus = mock(EventHandler.class);
  launcherBus = mock(EventHandler.class);
  monitorBus = mock(EventHandler.class);
  auxBus = mock(EventHandler.class);
  containerBus = mock(EventHandler.class);
  logAggregationBus = mock(EventHandler.class);

  dispatcher.register(LocalizationEventType.class, localizerBus);
  dispatcher.register(ContainersLauncherEventType.class, launcherBus);
  dispatcher.register(ContainersMonitorEventType.class, monitorBus);
  dispatcher.register(AuxServicesEventType.class, auxBus);
  dispatcher.register(ContainerEventType.class, containerBus);
  dispatcher.register(LogHandlerEventType.class, logAggregationBus);

  nmTokenSecretMgr = mock(NMTokenSecretManagerInNM.class);

  context = mock(Context.class);
  
  when(context.getContainerTokenSecretManager()).thenReturn(
    new NMContainerTokenSecretManager(conf));
  when(context.getApplicationACLsManager()).thenReturn(
    new ApplicationACLsManager(conf));
  when(context.getNMTokenSecretManager()).thenReturn(nmTokenSecretMgr);
  
  // Setting master key
  MasterKey masterKey = new MasterKeyPBImpl();
  masterKey.setKeyId(123);
  masterKey.setBytes(ByteBuffer.wrap(new byte[] { (new Integer(123)
    .byteValue()) }));
  context.getContainerTokenSecretManager().setMasterKey(masterKey);
  
  this.user = user;
  this.appId = BuilderUtils.newApplicationId(timestamp, id);

  app = new ApplicationImpl(dispatcher, this.user, appId, null, context);
  containers = new ArrayList<Container>();
  for (int i = 0; i < numContainers; i++) {
    Container container = createMockedContainer(this.appId, i);
    containers.add(container);
    long currentTime = System.currentTimeMillis();
    ContainerTokenIdentifier identifier =
        new ContainerTokenIdentifier(container.getContainerId(), "", "",
          null, currentTime + 2000, masterKey.getKeyId(), currentTime,
          Priority.newInstance(0), 0);
    containerTokenIdentifierMap
      .put(identifier.getContainerID(), identifier);
    context.getContainerTokenSecretManager().startContainerSuccessful(
      identifier);
    Assert.assertFalse(context.getContainerTokenSecretManager()
      .isValidStartContainerRequest(identifier));
  }

  dispatcher.start();
}
 

@Test( timeout = 10000)
@SuppressWarnings("unchecked") // mocked generics
public void testLocalizerRunnerException() throws Exception {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
  dispatcher.register(ApplicationEventType.class, applicationBus);
  EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
  dispatcher.register(ContainerEventType.class, containerBus);

  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
  dirsHandlerSpy.init(conf);

  DeletionService delServiceReal = new DeletionService(exec);
  DeletionService delService = spy(delServiceReal);
  delService.init(new Configuration());
  delService.start();

  ResourceLocalizationService rawService =
      new ResourceLocalizationService(dispatcher, exec, delService,
      dirsHandlerSpy, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  try {
    spyService.init(conf);
    spyService.start();

    // init application
    final Application app = mock(Application.class);
    final ApplicationId appId =
        BuilderUtils.newApplicationId(314159265358979L, 3);
    when(app.getUser()).thenReturn("user0");
    when(app.getAppId()).thenReturn(appId);
    spyService.handle(new ApplicationLocalizationEvent(
        LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
    dispatcher.await();

    Random r = new Random();
    long seed = r.nextLong();
    System.out.println("SEED: " + seed);
    r.setSeed(seed);
    final Container c = getMockContainer(appId, 42, "user0");
    final LocalResource resource1 = getPrivateMockedResource(r);
    System.out.println("Here 4");
    
    final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
      new HashMap<LocalResourceVisibility, 
                  Collection<LocalResourceRequest>>();
    List<LocalResourceRequest> privateResourceList =
        new ArrayList<LocalResourceRequest>();
    privateResourceList.add(req1);
    rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);

    final Constructor<?>[] constructors =
        FSError.class.getDeclaredConstructors();
    constructors[0].setAccessible(true);
    FSError fsError =
        (FSError) constructors[0].newInstance(new IOException("Disk Error"));

    Mockito
      .doThrow(fsError)
      .when(dirsHandlerSpy)
      .getLocalPathForWrite(isA(String.class));
    spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
    Thread.sleep(1000);
    dispatcher.await();
    // Verify if ContainerResourceFailedEvent is invoked on FSError
    verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
  } finally {
    spyService.stop();
    dispatcher.stop();
    delService.stop();
  }
}