org.apache.hadoop.yarn.event.Event Java Examples

The following examples show how to use org.apache.hadoop.yarn.event.Event. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestTaskCommunicatorManager1.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 5000)
public void testTaskEventRoutingTaskAttemptOnly() throws Exception {
  List<TezEvent> events = Arrays.asList(
    new TezEvent(new TaskAttemptCompletedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM,
        "v1", "v2", taskAttemptID))
  );
  generateHeartbeat(events, 0, 1, 0, new ArrayList<TezEvent>());

  ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(1)).handle(arg.capture());
  final List<Event> argAllValues = arg.getAllValues();

  final Event event = argAllValues.get(0);
  // Route to TaskAttempt directly rather than through Vertex
  assertEquals("only event should be route event", TaskAttemptEventType.TA_DONE,
      event.getType());
}
 
Example #2
Source File: TestRMContainerAllocator.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static AppContext createAppContext(
    ApplicationAttemptId appAttemptId, Job job) {
  AppContext context = mock(AppContext.class);
  ApplicationId appId = appAttemptId.getApplicationId();
  when(context.getApplicationID()).thenReturn(appId);
  when(context.getApplicationAttemptId()).thenReturn(appAttemptId);
  when(context.getJob(isA(JobId.class))).thenReturn(job);
  when(context.getClusterInfo()).thenReturn(
    new ClusterInfo(Resource.newInstance(10240, 1)));
  when(context.getEventHandler()).thenReturn(new EventHandler() {
    @Override
    public void handle(Event event) {
      // Only capture interesting events.
      if (event instanceof TaskAttemptContainerAssignedEvent) {
        events.add((TaskAttemptContainerAssignedEvent) event);
      } else if (event instanceof TaskAttemptKillEvent) {
        taskAttemptKillEvents.add((TaskAttemptKillEvent)event);
      } else if (event instanceof JobUpdatedNodesEvent) {
        jobUpdatedNodeEvents.add((JobUpdatedNodesEvent)event);
      } else if (event instanceof JobEvent) {
        jobEvents.add((JobEvent)event);
      }
    }
  });
  return context;
}
 
Example #3
Source File: TestContainerLaunch.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout = 10000)
public void testCallFailureWithNullLocalizedResources() {
  Container container = mock(Container.class);
  when(container.getContainerId()).thenReturn(ContainerId.newContainerId(
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(
          System.currentTimeMillis(), 1), 1), 1));
  ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
  when(clc.getCommands()).thenReturn(Collections.<String>emptyList());
  when(container.getLaunchContext()).thenReturn(clc);
  when(container.getLocalizedResources()).thenReturn(null);
  Dispatcher dispatcher = mock(Dispatcher.class);
  EventHandler eventHandler = new EventHandler() {
    public void handle(Event event) {
      Assert.assertTrue(event instanceof ContainerExitEvent);
      ContainerExitEvent exitEvent = (ContainerExitEvent) event;
      Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
          exitEvent.getType());
    }
  };
  when(dispatcher.getEventHandler()).thenReturn(eventHandler);
  ContainerLaunch launch = new ContainerLaunch(context, new Configuration(),
      dispatcher, exec, null, container, dirsHandler, containerManager);
  launch.call();
}
 
Example #4
Source File: TestDelegationTokenRenewer.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  counter = new AtomicInteger(0);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  eventQueue = new LinkedBlockingQueue<Event>();
  dispatcher = new AsyncDispatcher(eventQueue);
  Renewer.reset();
  delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext =  mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      delegationTokenRenewer);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.start();
}
 
Example #5
Source File: TajoAsyncDispatcher.java    From incubator-tajo with Apache License 2.0 6 votes vote down vote up
public void handle(Event event) {
  /* all this method does is enqueue all the events onto the queue */
  int qSize = eventQueue.size();
  if (qSize !=0 && qSize %1000 == 0) {
    LOG.info("Size of event-queue is " + qSize);
  }
  int remCapacity = eventQueue.remainingCapacity();
  if (remCapacity < 1000) {
    LOG.warn("Very low remaining capacity in the event-queue: "
        + remCapacity);
  }
  try {
    if(LOG.isDebugEnabled()) {
      LOG.debug(id + ",add event:" +
          event.getType() + "," + event + "," +
          (eventHandlingThread == null ? "null" : eventHandlingThread.isAlive()));
    }
    eventQueue.put(event);
  } catch (InterruptedException e) {
    if (!stopped) {
      LOG.warn("AsyncDispatcher thread interrupted", e);
    }
    throw new YarnRuntimeException(e);
  }
}
 
Example #6
Source File: TestTaskImpl.java    From tez with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test(timeout = 5000)
public void testTaskSucceedAndRetroActiveKilled() {
  TezTaskID taskId = getNewTaskID();
  scheduleTaskAttempt(taskId);
  launchTaskAttempt(mockTask.getLastAttempt().getID());
  updateAttemptState(mockTask.getLastAttempt(), TaskAttemptState.RUNNING);

  mockTask.handle(createTaskTASucceededEvent(mockTask.getLastAttempt().getID()));

  // The task should now have succeeded
  assertTaskSucceededState();
  verify(mockTask.stateChangeNotifier).taskSucceeded(any(String.class), eq(taskId),
      eq(mockTask.getLastAttempt().getID().getId()));

  eventHandler.events.clear();
  // Now kill the attempt after it has succeeded
  mockTask.handle(createTaskTAKilledEvent(mockTask.getLastAttempt().getID()));

  // The task should still be in the scheduled state
  assertTaskScheduledState();
  Event event = eventHandler.events.get(0);
  Assert.assertEquals(VertexEventType.V_TASK_RESCHEDULED, event.getType());
}
 
Example #7
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public ClientRMService createRMService() throws IOException {
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
      yarnScheduler);
  when(rmContext.getRMApps()).thenReturn(apps);
  when(rmContext.getYarnConfiguration()).thenReturn(new Configuration());
  RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
      mock(ApplicationACLsManager.class), new Configuration());
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
  when(
      mockQueueACLsManager.checkAccess(any(UserGroupInformation.class),
          any(QueueACL.class), anyString())).thenReturn(true);
  return new ClientRMService(rmContext, yarnScheduler, appManager,
      mockAclsManager, mockQueueACLsManager, null);
}
 
Example #8
Source File: TestKill.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected void dispatch(Event event) {
  if (event instanceof TaskAttemptEvent) {
    TaskAttemptEvent attemptEvent = (TaskAttemptEvent) event;
    TaskAttemptId attemptID = ((TaskAttemptEvent) event).getTaskAttemptID();
    if (attemptEvent.getType() == this.attemptEventTypeToWait
        && attemptID.getTaskId().getId() == 0 && attemptID.getId() == 0 ) {
      try {
        latch.await();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
  super.dispatch(event);
}
 
Example #9
Source File: TestRMContainerAllocator.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static AppContext createAppContext(
    ApplicationAttemptId appAttemptId, Job job) {
  AppContext context = mock(AppContext.class);
  ApplicationId appId = appAttemptId.getApplicationId();
  when(context.getApplicationID()).thenReturn(appId);
  when(context.getApplicationAttemptId()).thenReturn(appAttemptId);
  when(context.getJob(isA(JobId.class))).thenReturn(job);
  when(context.getClusterInfo()).thenReturn(
    new ClusterInfo(Resource.newInstance(10240, 1)));
  when(context.getEventHandler()).thenReturn(new EventHandler() {
    @Override
    public void handle(Event event) {
      // Only capture interesting events.
      if (event instanceof TaskAttemptContainerAssignedEvent) {
        events.add((TaskAttemptContainerAssignedEvent) event);
      } else if (event instanceof TaskAttemptKillEvent) {
        taskAttemptKillEvents.add((TaskAttemptKillEvent)event);
      } else if (event instanceof JobUpdatedNodesEvent) {
        jobUpdatedNodeEvents.add((JobUpdatedNodesEvent)event);
      } else if (event instanceof JobEvent) {
        jobEvents.add((JobEvent)event);
      }
    }
  });
  return context;
}
 
Example #10
Source File: TajoAsyncDispatcher.java    From incubator-tajo with Apache License 2.0 6 votes vote down vote up
Runnable createThread() {
  return new Runnable() {
    @Override
    public void run() {
      while (!stopped && !Thread.currentThread().isInterrupted()) {
        Event event;
        try {
          event = eventQueue.take();
          if(LOG.isDebugEnabled()) {
            LOG.debug(id + ",event take:" + event.getType() + "," + event);
          }
        } catch(InterruptedException ie) {
          if (!stopped) {
            LOG.warn("AsyncDispatcher thread interrupted");
          }
          return;
        }
        dispatch(event);
      }
    }
  };
}
 
Example #11
Source File: TestAMContainer.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test
public void testContainerPreemptedAtRunning() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();

  wc.assignTaskAttempt(wc.taskAttemptID);
  wc.containerLaunched();
  wc.pullTaskToRun();
  wc.verifyState(AMContainerState.RUNNING);

  wc.containerCompleted(ContainerExitStatus.PREEMPTED);
  wc.verifyState(AMContainerState.COMPLETED);
  verify(wc.tal).registerRunningContainer(wc.containerID);
  verify(wc.tal).unregisterRunningContainer(wc.containerID);
  verify(wc.chh).register(wc.containerID);
  verify(wc.chh).unregister(wc.containerID);

  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM);

  assertFalse(wc.amContainer.isInErrorState());

  // Pending task complete. (Ideally, container should be dead at this point
  // and this event should not be generated. Network timeout on NM-RM heartbeat
  // can cause it to be genreated)
  wc.taskAttemptSucceeded(wc.taskAttemptID);
  wc.verifyNoOutgoingEvents();
  wc.verifyHistoryStopEvent();

  assertFalse(wc.amContainer.isInErrorState());
}
 
Example #12
Source File: RMStateStoreTestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void handle(Event event) {
  if (event instanceof RMAppAttemptEvent) {
    RMAppAttemptEvent rmAppAttemptEvent = (RMAppAttemptEvent) event;
    assertEquals(attemptId, rmAppAttemptEvent.getApplicationAttemptId());
  }
  notified = true;
  synchronized (this) {
    notifyAll();
  }
}
 
Example #13
Source File: TestTaskAttempt.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void handle(Event event) {
  if (event instanceof JobEvent) {
    JobEvent je = ((JobEvent) event);
    if (JobEventType.INTERNAL_ERROR == je.getType()) {
      internalError = true;
    }
  }
}
 
Example #14
Source File: TestDelegationTokenRenewer.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testAppRejectionWithCancelledDelegationToken() throws Exception {
  MyFS dfs = (MyFS)FileSystem.get(conf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+conf.hashCode());

  MyToken token = dfs.getDelegationToken("user1");
  token.cancelToken();

  Credentials ts = new Credentials();
  ts.addToken(token.getKind(), token);
  
  // register the tokens for renewal
  ApplicationId appId =  BuilderUtils.newApplicationId(0, 0);
  delegationTokenRenewer.addApplicationAsync(appId, ts, true, "user");
  int waitCnt = 20;
  while (waitCnt-- >0) {
    if (!eventQueue.isEmpty()) {
      Event evt = eventQueue.take();
      if (evt.getType() == RMAppEventType.APP_REJECTED) {
        Assert.assertTrue(
            ((RMAppEvent) evt).getApplicationId().equals(appId));
        return;
      }
    } else {
      Thread.sleep(500);
    }
  }
  fail("App submission with a cancelled token should have failed");
}
 
Example #15
Source File: TestTaskSchedulerHelpers.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public Event verifyInvocation(Class<? extends Event> eventClass) {
  for (Event e : events) {
    if (e.getClass().getName().equals(eventClass.getName())) {
      return e;
    }
  }
  fail("Expected Event: " + eventClass.getName() + " not sent");
  return null;
}
 
Example #16
Source File: TestTaskAttempt.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleRackRequest() throws Exception {
  TaskAttemptImpl.RequestContainerTransition rct =
      new TaskAttemptImpl.RequestContainerTransition(false);

  EventHandler eventHandler = mock(EventHandler.class);
  String[] hosts = new String[3];
  hosts[0] = "host1";
  hosts[1] = "host2";
  hosts[2] = "host3";
  TaskSplitMetaInfo splitInfo =
      new TaskSplitMetaInfo(hosts, 0, 128 * 1024 * 1024l);

  TaskAttemptImpl mockTaskAttempt =
      createMapTaskAttemptImplForTest(eventHandler, splitInfo);
  TaskAttemptEvent mockTAEvent = mock(TaskAttemptEvent.class);

  rct.transition(mockTaskAttempt, mockTAEvent);

  ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(2)).handle(arg.capture());
  if (!(arg.getAllValues().get(1) instanceof ContainerRequestEvent)) {
    Assert.fail("Second Event not of type ContainerRequestEvent");
  }
  ContainerRequestEvent cre =
      (ContainerRequestEvent) arg.getAllValues().get(1);
  String[] requestedRacks = cre.getRacks();
  //Only a single occurrence of /DefaultRack
  assertEquals(1, requestedRacks.length);
}
 
Example #17
Source File: TestRMApplicationHistoryWriter.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Override
protected void dispatch(Event event) {
  if (event instanceof WritingApplicationHistoryEvent) {
    WritingApplicationHistoryEvent ashEvent =
        (WritingApplicationHistoryEvent) event;
    switch (ashEvent.getType()) {
      case APP_START:
        incrementCounts(((WritingApplicationStartEvent) event)
          .getApplicationId());
        break;
      case APP_FINISH:
        incrementCounts(((WritingApplicationFinishEvent) event)
          .getApplicationId());
        break;
      case APP_ATTEMPT_START:
        incrementCounts(((WritingApplicationAttemptStartEvent) event)
          .getApplicationAttemptId().getApplicationId());
        break;
      case APP_ATTEMPT_FINISH:
        incrementCounts(((WritingApplicationAttemptFinishEvent) event)
          .getApplicationAttemptId().getApplicationId());
        break;
      case CONTAINER_START:
        incrementCounts(((WritingContainerStartEvent) event)
          .getContainerId().getApplicationAttemptId().getApplicationId());
        break;
      case CONTAINER_FINISH:
        incrementCounts(((WritingContainerFinishEvent) event)
          .getContainerId().getApplicationAttemptId().getApplicationId());
        break;
    }
  }
  super.dispatch(event);
}
 
Example #18
Source File: TestTaskImpl.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 20000)
public void testKilledAttemptUpdatesDAGScheduler() {
  TezTaskID taskId = getNewTaskID();
  scheduleTaskAttempt(taskId);
  MockTaskAttemptImpl firstAttempt = mockTask.getLastAttempt();
  launchTaskAttempt(firstAttempt.getID());
  updateAttemptState(firstAttempt, TaskAttemptState.RUNNING);

  // Add a speculative task attempt
  mockTask.handle(createTaskTAAddSpecAttempt(firstAttempt.getID()));
  MockTaskAttemptImpl specAttempt = mockTask.getLastAttempt();
  launchTaskAttempt(specAttempt.getID());
  updateAttemptState(specAttempt, TaskAttemptState.RUNNING);
  assertEquals(2, mockTask.getAttemptList().size());

  // Have the first task succeed
  eventHandler.events.clear();
  mockTask.handle(createTaskTASucceededEvent(firstAttempt.getID()));
  verifyOutgoingEvents(eventHandler.events, DAGEventType.DAG_SCHEDULER_UPDATE,
      VertexEventType.V_TASK_COMPLETED, VertexEventType.V_TASK_ATTEMPT_COMPLETED);

  // The task should now have succeeded and sent kill to other attempt
  assertTaskSucceededState();
  verify(mockTask.stateChangeNotifier).taskSucceeded(any(String.class), eq(taskId),
      eq(firstAttempt.getID().getId()));
  @SuppressWarnings("rawtypes")
  Event event = eventHandler.events.get(eventHandler.events.size()-1);
  assertEquals(TaskAttemptEventType.TA_KILL_REQUEST, event.getType());
  assertEquals(specAttempt.getID(),
      ((TaskAttemptEventKillRequest) event).getTaskAttemptID());

  eventHandler.events.clear();
  // Emulate the spec attempt being killed
  mockTask.handle(createTaskTAKilledEvent(specAttempt.getID()));
  assertTaskSucceededState();
  verifyOutgoingEvents(eventHandler.events, DAGEventType.DAG_SCHEDULER_UPDATE,
      VertexEventType.V_TASK_ATTEMPT_COMPLETED);
}
 
Example #19
Source File: TestAMContainer.java    From tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout=5000)
public void testContainerCompletedAtLaunchingSpecificClusterError() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();

  wc.assignTaskAttempt(wc.taskAttemptID);

  wc.containerCompleted(ContainerExitStatus.DISKS_FAILED, TaskAttemptTerminationCause.NODE_DISK_ERROR, "DiskFailed");
  wc.verifyState(AMContainerState.COMPLETED);
  verify(wc.tal).registerRunningContainer(wc.containerID, 0);
  verifyUnregisterRunningContainer(wc.tal, wc.containerID, 0, ContainerEndReason.OTHER, "DiskFailed");

  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM,
      AMNodeEventType.N_CONTAINER_COMPLETED);
  Assert.assertEquals(TaskAttemptTerminationCause.NODE_DISK_ERROR,
      ((TaskAttemptEventContainerTerminatedBySystem)outgoingEvents.get(0)).getTerminationCause());

  assertFalse(wc.amContainer.isInErrorState());

  // Container launched generated by NM call.
  wc.containerLaunched();
  wc.verifyNoOutgoingEvents();

  assertFalse(wc.amContainer.isInErrorState());
}
 
Example #20
Source File: TestAMContainer.java    From tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout=5000)
public void testLaunchFailure() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();
  wc.assignTaskAttempt(wc.taskAttemptID);
  wc.verifyState(AMContainerState.LAUNCHING);
  verify(wc.tal).registerRunningContainer(wc.containerID, 0);
  wc.launchFailed();
  wc.verifyState(AMContainerState.STOPPING);
  verify(wc.tal).registerRunningContainer(wc.containerID, 0);
  verifyUnregisterRunningContainer(wc.tal, wc.containerID, 0, ContainerEndReason.LAUNCH_FAILED,
      "launchFailed");

  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATING,
      AMSchedulerEventType.S_CONTAINER_DEALLOCATE,
      AMNodeEventType.N_CONTAINER_COMPLETED);
  for (Event e : outgoingEvents) {
    if (e.getType() == TaskAttemptEventType.TA_CONTAINER_TERMINATING) {
      Assert.assertEquals(TaskAttemptTerminationCause.CONTAINER_LAUNCH_FAILED,
          ((TaskAttemptEventContainerTerminating)e).getTerminationCause());
    }
  }

  wc.containerCompleted();
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED);

  // Valid transition. Container complete, but not with an error.
  assertFalse(wc.amContainer.isInErrorState());
}
 
Example #21
Source File: TestAMContainer.java    From tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout=5000)
public void testNodeFailedAtRunning() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();
  wc.containerLaunched();
  wc.assignTaskAttempt(wc.taskAttemptID);
  wc.verifyState(AMContainerState.RUNNING);

  wc.nodeFailed();
  // Expecting a complete event from the RM
  wc.verifyState(AMContainerState.STOPPING);
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(4);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_NODE_FAILED,
      TaskAttemptEventType.TA_CONTAINER_TERMINATING,
      AMSchedulerEventType.S_CONTAINER_DEALLOCATE,
      AMNodeEventType.N_CONTAINER_COMPLETED);

  for (Event event : outgoingEvents) {
    if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
      TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
      assertTrue(nfEvent.getDiagnosticInfo().contains("nodeFailed"));
    }
  }

  wc.containerCompleted();
  wc.verifyHistoryStopEvent();
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED);

  assertFalse(wc.amContainer.isInErrorState());
}
 
Example #22
Source File: TestAMContainer.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test
public void testAllocationAtRunning() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();
  wc.containerLaunched();
  wc.assignTaskAttempt(wc.taskAttemptID);
  wc.pullTaskToRun();
  wc.verifyState(AMContainerState.RUNNING);

  TezTaskAttemptID taID2 = TezTaskAttemptID.getInstance(wc.taskID, 2);
  wc.assignTaskAttempt(taID2);

  wc.verifyState(AMContainerState.STOP_REQUESTED);
  verify(wc.tal).unregisterRunningContainer(wc.containerID);
  verify(wc.chh).unregister(wc.containerID);
  // 1 for NM stop request. 2 TERMINATING to TaskAttempt.
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
      TaskAttemptEventType.TA_CONTAINER_TERMINATING,
      TaskAttemptEventType.TA_CONTAINER_TERMINATING);
  assertTrue(wc.amContainer.isInErrorState());

  wc.nmStopSent();
  wc.containerCompleted(false);
  wc.verifyHistoryStopEvent();
  // 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED);

  assertNull(wc.amContainer.getRunningTaskAttempt());
  assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
  assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
}
 
Example #23
Source File: TestAMContainer.java    From tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout=5000)
public void testContainerCompletedAtLaunchingSpecificError() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();


  wc.assignTaskAttempt(wc.taskAttemptID);

  wc.containerCompleted(ContainerExitStatus.ABORTED, TaskAttemptTerminationCause.NODE_FAILED, "NodeFailed");
  wc.verifyState(AMContainerState.COMPLETED);
  verify(wc.tal).registerRunningContainer(wc.containerID, 0);
  verifyUnregisterRunningContainer(wc.tal, wc.containerID, 0, ContainerEndReason.NODE_FAILED,
      "NodeFailed");

  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED,
      AMNodeEventType.N_CONTAINER_COMPLETED);
  Assert.assertEquals(TaskAttemptTerminationCause.NODE_FAILED,
      ((TaskAttemptEventContainerTerminated)outgoingEvents.get(0)).getTerminationCause());

  assertFalse(wc.amContainer.isInErrorState());

  // Container launched generated by NM call.
  wc.containerLaunched();
  wc.verifyNoOutgoingEvents();

  assertFalse(wc.amContainer.isInErrorState());
}
 
Example #24
Source File: TestAMContainer.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test
public void testNodeFailedAtIdle() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();
  wc.containerLaunched();
  wc.assignTaskAttempt(wc.taskAttemptID);
  wc.verifyState(AMContainerState.IDLE);

  wc.nodeFailed();
  // Expecting a complete event from the RM
  wc.verifyState(AMContainerState.STOPPING);
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_NODE_FAILED,
      TaskAttemptEventType.TA_CONTAINER_TERMINATING,
      AMSchedulerEventType.S_CONTAINER_DEALLOCATE);

  for (Event event : outgoingEvents) {
    if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
      TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
      assertEquals("nodeFailed", nfEvent.getDiagnosticInfo());
    }
  }

  wc.containerCompleted(false);
  wc.verifyHistoryStopEvent();
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED);

  assertFalse(wc.amContainer.isInErrorState());
}
 
Example #25
Source File: TestTaskSchedulerManager.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testContainerInternalPreempted() throws IOException, ServicePluginException {
  Configuration conf = new Configuration(false);
  schedulerHandler.init(conf);
  schedulerHandler.start();

  AMContainer mockAmContainer = mock(AMContainer.class);
  when(mockAmContainer.getTaskSchedulerIdentifier()).thenReturn(0);
  when(mockAmContainer.getContainerLauncherIdentifier()).thenReturn(0);
  when(mockAmContainer.getTaskCommunicatorIdentifier()).thenReturn(0);
  ContainerId mockCId = mock(ContainerId.class);
  verify(mockTaskScheduler, times(0)).deallocateContainer((ContainerId) any());
  when(mockAMContainerMap.get(mockCId)).thenReturn(mockAmContainer);
  schedulerHandler.preemptContainer(0, mockCId);
  verify(mockTaskScheduler, times(1)).deallocateContainer(mockCId);
  assertEquals(1, mockEventHandler.events.size());
  Event event = mockEventHandler.events.get(0);
  assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
  AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
  assertEquals(mockCId, completedEvent.getContainerId());
  assertEquals("Container preempted internally", completedEvent.getDiagnostics());
  assertTrue(completedEvent.isPreempted());
  Assert.assertFalse(completedEvent.isDiskFailed());
  assertEquals(TaskAttemptTerminationCause.INTERNAL_PREEMPTION,
      completedEvent.getTerminationCause());

  schedulerHandler.stop();
  schedulerHandler.close();
}
 
Example #26
Source File: TestKillQuery.java    From tajo with Apache License 2.0 5 votes vote down vote up
@Override
protected void dispatch(Event event) {
  if (event.getType() == eventType) {
    latch.countDown();
  }
  super.dispatch(event);
}
 
Example #27
Source File: TestAMContainer.java    From tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout=5000)
public void testStopRequestedAtRunning() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();
  wc.containerLaunched();
  wc.assignTaskAttempt(wc.taskAttemptID);
  wc.verifyState(AMContainerState.RUNNING);

  wc.stopRequest();
  wc.verifyState(AMContainerState.STOP_REQUESTED);
  verifyUnregisterRunningContainer(wc.tal, wc.containerID, 0, ContainerEndReason.OTHER,
      "received a STOP_REQUEST");
  verify(wc.chh).unregister(wc.containerID);
  // 1 to TA, 1 for RM de-allocate.
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATING,
      ContainerLauncherEventType.CONTAINER_STOP_REQUEST,
      AMNodeEventType.N_CONTAINER_COMPLETED);
  // TODO Should this be an RM DE-ALLOCATE instead ?

  wc.containerCompleted();
  wc.verifyHistoryStopEvent();
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED);

  assertFalse(wc.amContainer.isInErrorState());

  assertNull(wc.amContainer.getCurrentTaskAttempt());
  assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
}
 
Example #28
Source File: TestTaskAttempt.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public void handle(Event event) {
  if (event instanceof DAGEvent) {
    DAGEvent je = ((DAGEvent) event);
    if (DAGEventType.INTERNAL_ERROR == je.getType()) {
      internalError = true;
    }
  }
}
 
Example #29
Source File: TestAMContainer.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test (timeout=5000)
public void testSingleSuccessfulTaskFlowStopRequest() {
  WrappedContainer wc = new WrappedContainer();

  wc.verifyState(AMContainerState.ALLOCATED);

  wc.launchContainer();
  wc.assignTaskAttempt(wc.taskAttemptID);
  wc.containerLaunched();
  wc.taskAttemptSucceeded(wc.taskAttemptID);

  wc.stopRequest();
  wc.verifyState(AMContainerState.STOP_REQUESTED);
  // Event to NM to stop the container.

  List<Event> outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      ContainerLauncherEventType.CONTAINER_STOP_REQUEST,
      AMNodeEventType.N_CONTAINER_COMPLETED);

  wc.nmStopSent();
  wc.verifyState(AMContainerState.STOPPING);
  wc.verifyNoOutgoingEvents();

  wc.containerCompleted();
  wc.verifyHistoryStopEvent();
  wc.verifyState(AMContainerState.COMPLETED);
  wc.verifyNoOutgoingEvents();
  verifyUnregisterRunningContainer(wc.tal, wc.containerID, 0, ContainerEndReason.OTHER,
      "received a STOP_REQUEST");
  verify(wc.chh).unregister(wc.containerID);

  assertNull(wc.amContainer.getCurrentTaskAttempt());
  assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
  assertFalse(wc.amContainer.isInErrorState());
}
 
Example #30
Source File: TestRMApplicationHistoryWriter.java    From big-c with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Override
protected void dispatch(Event event) {
  if (event instanceof WritingApplicationHistoryEvent) {
    WritingApplicationHistoryEvent ashEvent =
        (WritingApplicationHistoryEvent) event;
    switch (ashEvent.getType()) {
      case APP_START:
        incrementCounts(((WritingApplicationStartEvent) event)
          .getApplicationId());
        break;
      case APP_FINISH:
        incrementCounts(((WritingApplicationFinishEvent) event)
          .getApplicationId());
        break;
      case APP_ATTEMPT_START:
        incrementCounts(((WritingApplicationAttemptStartEvent) event)
          .getApplicationAttemptId().getApplicationId());
        break;
      case APP_ATTEMPT_FINISH:
        incrementCounts(((WritingApplicationAttemptFinishEvent) event)
          .getApplicationAttemptId().getApplicationId());
        break;
      case CONTAINER_START:
        incrementCounts(((WritingContainerStartEvent) event)
          .getContainerId().getApplicationAttemptId().getApplicationId());
        break;
      case CONTAINER_FINISH:
        incrementCounts(((WritingContainerFinishEvent) event)
          .getContainerId().getApplicationAttemptId().getApplicationId());
        break;
    }
  }
  super.dispatch(event);
}