Java Code Examples for org.apache.hadoop.yarn.event.Event

The following examples show how to use org.apache.hadoop.yarn.event.Event. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestContainerLaunch.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout = 10000)
public void testCallFailureWithNullLocalizedResources() {
  Container container = mock(Container.class);
  when(container.getContainerId()).thenReturn(ContainerId.newContainerId(
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(
          System.currentTimeMillis(), 1), 1), 1));
  ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
  when(clc.getCommands()).thenReturn(Collections.<String>emptyList());
  when(container.getLaunchContext()).thenReturn(clc);
  when(container.getLocalizedResources()).thenReturn(null);
  Dispatcher dispatcher = mock(Dispatcher.class);
  EventHandler eventHandler = new EventHandler() {
    public void handle(Event event) {
      Assert.assertTrue(event instanceof ContainerExitEvent);
      ContainerExitEvent exitEvent = (ContainerExitEvent) event;
      Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
          exitEvent.getType());
    }
  };
  when(dispatcher.getEventHandler()).thenReturn(eventHandler);
  ContainerLaunch launch = new ContainerLaunch(context, new Configuration(),
      dispatcher, exec, null, container, dirsHandler, containerManager);
  launch.call();
}
 
Example 2
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
public ClientRMService createRMService() throws IOException {
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
      yarnScheduler);
  when(rmContext.getRMApps()).thenReturn(apps);
  when(rmContext.getYarnConfiguration()).thenReturn(new Configuration());
  RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
      mock(ApplicationACLsManager.class), new Configuration());
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
  when(
      mockQueueACLsManager.checkAccess(any(UserGroupInformation.class),
          any(QueueACL.class), anyString())).thenReturn(true);
  return new ClientRMService(rmContext, yarnScheduler, appManager,
      mockAclsManager, mockQueueACLsManager, null);
}
 
Example 3
Source Project: hadoop   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 6 votes vote down vote up
private static AppContext createAppContext(
    ApplicationAttemptId appAttemptId, Job job) {
  AppContext context = mock(AppContext.class);
  ApplicationId appId = appAttemptId.getApplicationId();
  when(context.getApplicationID()).thenReturn(appId);
  when(context.getApplicationAttemptId()).thenReturn(appAttemptId);
  when(context.getJob(isA(JobId.class))).thenReturn(job);
  when(context.getClusterInfo()).thenReturn(
    new ClusterInfo(Resource.newInstance(10240, 1)));
  when(context.getEventHandler()).thenReturn(new EventHandler() {
    @Override
    public void handle(Event event) {
      // Only capture interesting events.
      if (event instanceof TaskAttemptContainerAssignedEvent) {
        events.add((TaskAttemptContainerAssignedEvent) event);
      } else if (event instanceof TaskAttemptKillEvent) {
        taskAttemptKillEvents.add((TaskAttemptKillEvent)event);
      } else if (event instanceof JobUpdatedNodesEvent) {
        jobUpdatedNodeEvents.add((JobUpdatedNodesEvent)event);
      } else if (event instanceof JobEvent) {
        jobEvents.add((JobEvent)event);
      }
    }
  });
  return context;
}
 
Example 4
Source Project: big-c   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  counter = new AtomicInteger(0);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  eventQueue = new LinkedBlockingQueue<Event>();
  dispatcher = new AsyncDispatcher(eventQueue);
  Renewer.reset();
  delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext =  mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      delegationTokenRenewer);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.start();
}
 
Example 5
Source Project: big-c   Source File: TestKill.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void dispatch(Event event) {
  if (event instanceof TaskAttemptEvent) {
    TaskAttemptEvent attemptEvent = (TaskAttemptEvent) event;
    TaskAttemptId attemptID = ((TaskAttemptEvent) event).getTaskAttemptID();
    if (attemptEvent.getType() == this.attemptEventTypeToWait
        && attemptID.getTaskId().getId() == 0 && attemptID.getId() == 0 ) {
      try {
        latch.await();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
  super.dispatch(event);
}
 
Example 6
Source Project: tez   Source File: TestTaskCommunicatorManager1.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 5000)
public void testTaskEventRoutingTaskAttemptOnly() throws Exception {
  List<TezEvent> events = Arrays.asList(
    new TezEvent(new TaskAttemptCompletedEvent(), new EventMetaData(EventProducerConsumerType.SYSTEM,
        "v1", "v2", taskAttemptID))
  );
  generateHeartbeat(events, 0, 1, 0, new ArrayList<TezEvent>());

  ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(1)).handle(arg.capture());
  final List<Event> argAllValues = arg.getAllValues();

  final Event event = argAllValues.get(0);
  // Route to TaskAttempt directly rather than through Vertex
  assertEquals("only event should be route event", TaskAttemptEventType.TA_DONE,
      event.getType());
}
 
Example 7
Source Project: big-c   Source File: TestRMContainerAllocator.java    License: Apache License 2.0 6 votes vote down vote up
private static AppContext createAppContext(
    ApplicationAttemptId appAttemptId, Job job) {
  AppContext context = mock(AppContext.class);
  ApplicationId appId = appAttemptId.getApplicationId();
  when(context.getApplicationID()).thenReturn(appId);
  when(context.getApplicationAttemptId()).thenReturn(appAttemptId);
  when(context.getJob(isA(JobId.class))).thenReturn(job);
  when(context.getClusterInfo()).thenReturn(
    new ClusterInfo(Resource.newInstance(10240, 1)));
  when(context.getEventHandler()).thenReturn(new EventHandler() {
    @Override
    public void handle(Event event) {
      // Only capture interesting events.
      if (event instanceof TaskAttemptContainerAssignedEvent) {
        events.add((TaskAttemptContainerAssignedEvent) event);
      } else if (event instanceof TaskAttemptKillEvent) {
        taskAttemptKillEvents.add((TaskAttemptKillEvent)event);
      } else if (event instanceof JobUpdatedNodesEvent) {
        jobUpdatedNodeEvents.add((JobUpdatedNodesEvent)event);
      } else if (event instanceof JobEvent) {
        jobEvents.add((JobEvent)event);
      }
    }
  });
  return context;
}
 
Example 8
Source Project: incubator-tajo   Source File: TajoAsyncDispatcher.java    License: Apache License 2.0 6 votes vote down vote up
Runnable createThread() {
  return new Runnable() {
    @Override
    public void run() {
      while (!stopped && !Thread.currentThread().isInterrupted()) {
        Event event;
        try {
          event = eventQueue.take();
          if(LOG.isDebugEnabled()) {
            LOG.debug(id + ",event take:" + event.getType() + "," + event);
          }
        } catch(InterruptedException ie) {
          if (!stopped) {
            LOG.warn("AsyncDispatcher thread interrupted");
          }
          return;
        }
        dispatch(event);
      }
    }
  };
}
 
Example 9
Source Project: incubator-tajo   Source File: TajoAsyncDispatcher.java    License: Apache License 2.0 6 votes vote down vote up
public void handle(Event event) {
  /* all this method does is enqueue all the events onto the queue */
  int qSize = eventQueue.size();
  if (qSize !=0 && qSize %1000 == 0) {
    LOG.info("Size of event-queue is " + qSize);
  }
  int remCapacity = eventQueue.remainingCapacity();
  if (remCapacity < 1000) {
    LOG.warn("Very low remaining capacity in the event-queue: "
        + remCapacity);
  }
  try {
    if(LOG.isDebugEnabled()) {
      LOG.debug(id + ",add event:" +
          event.getType() + "," + event + "," +
          (eventHandlingThread == null ? "null" : eventHandlingThread.isAlive()));
    }
    eventQueue.put(event);
  } catch (InterruptedException e) {
    if (!stopped) {
      LOG.warn("AsyncDispatcher thread interrupted", e);
    }
    throw new YarnRuntimeException(e);
  }
}
 
Example 10
Source Project: tez   Source File: TestTaskImpl.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test(timeout = 5000)
public void testTaskSucceedAndRetroActiveKilled() {
  TezTaskID taskId = getNewTaskID();
  scheduleTaskAttempt(taskId);
  launchTaskAttempt(mockTask.getLastAttempt().getID());
  updateAttemptState(mockTask.getLastAttempt(), TaskAttemptState.RUNNING);

  mockTask.handle(createTaskTASucceededEvent(mockTask.getLastAttempt().getID()));

  // The task should now have succeeded
  assertTaskSucceededState();
  verify(mockTask.stateChangeNotifier).taskSucceeded(any(String.class), eq(taskId),
      eq(mockTask.getLastAttempt().getID().getId()));

  eventHandler.events.clear();
  // Now kill the attempt after it has succeeded
  mockTask.handle(createTaskTAKilledEvent(mockTask.getLastAttempt().getID()));

  // The task should still be in the scheduled state
  assertTaskScheduledState();
  Event event = eventHandler.events.get(0);
  Assert.assertEquals(VertexEventType.V_TASK_RESCHEDULED, event.getType());
}
 
Example 11
Source Project: tez   Source File: TestTaskImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 20000)
public void testKilledAttemptUpdatesDAGScheduler() {
  TezTaskID taskId = getNewTaskID();
  scheduleTaskAttempt(taskId);
  MockTaskAttemptImpl firstAttempt = mockTask.getLastAttempt();
  launchTaskAttempt(firstAttempt.getID());
  updateAttemptState(firstAttempt, TaskAttemptState.RUNNING);

  // Add a speculative task attempt
  mockTask.handle(createTaskTAAddSpecAttempt(firstAttempt.getID()));
  MockTaskAttemptImpl specAttempt = mockTask.getLastAttempt();
  launchTaskAttempt(specAttempt.getID());
  updateAttemptState(specAttempt, TaskAttemptState.RUNNING);
  assertEquals(2, mockTask.getAttemptList().size());

  // Have the first task succeed
  eventHandler.events.clear();
  mockTask.handle(createTaskTASucceededEvent(firstAttempt.getID()));
  verifyOutgoingEvents(eventHandler.events, DAGEventType.DAG_SCHEDULER_UPDATE,
      VertexEventType.V_TASK_COMPLETED, VertexEventType.V_TASK_ATTEMPT_COMPLETED);

  // The task should now have succeeded and sent kill to other attempt
  assertTaskSucceededState();
  verify(mockTask.stateChangeNotifier).taskSucceeded(any(String.class), eq(taskId),
      eq(firstAttempt.getID().getId()));
  @SuppressWarnings("rawtypes")
  Event event = eventHandler.events.get(eventHandler.events.size()-1);
  assertEquals(TaskAttemptEventType.TA_KILL_REQUEST, event.getType());
  assertEquals(specAttempt.getID(),
      ((TaskAttemptEventKillRequest) event).getTaskAttemptID());

  eventHandler.events.clear();
  // Emulate the spec attempt being killed
  mockTask.handle(createTaskTAKilledEvent(specAttempt.getID()));
  assertTaskSucceededState();
  verifyOutgoingEvents(eventHandler.events, DAGEventType.DAG_SCHEDULER_UPDATE,
      VertexEventType.V_TASK_ATTEMPT_COMPLETED);
}
 
Example 12
Source Project: hadoop   Source File: TestDelegationTokenRenewer.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout=60000)
public void testAppRejectionWithCancelledDelegationToken() throws Exception {
  MyFS dfs = (MyFS)FileSystem.get(conf);
  LOG.info("dfs="+(Object)dfs.hashCode() + ";conf="+conf.hashCode());

  MyToken token = dfs.getDelegationToken("user1");
  token.cancelToken();

  Credentials ts = new Credentials();
  ts.addToken(token.getKind(), token);
  
  // register the tokens for renewal
  ApplicationId appId =  BuilderUtils.newApplicationId(0, 0);
  delegationTokenRenewer.addApplicationAsync(appId, ts, true, "user");
  int waitCnt = 20;
  while (waitCnt-- >0) {
    if (!eventQueue.isEmpty()) {
      Event evt = eventQueue.take();
      if (evt.getType() == RMAppEventType.APP_REJECTED) {
        Assert.assertTrue(
            ((RMAppEvent) evt).getApplicationId().equals(appId));
        return;
      }
    } else {
      Thread.sleep(500);
    }
  }
  fail("App submission with a cancelled token should have failed");
}
 
Example 13
Source Project: hadoop   Source File: TestRMApplicationHistoryWriter.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Override
protected void dispatch(Event event) {
  if (event instanceof WritingApplicationHistoryEvent) {
    WritingApplicationHistoryEvent ashEvent =
        (WritingApplicationHistoryEvent) event;
    switch (ashEvent.getType()) {
      case APP_START:
        incrementCounts(((WritingApplicationStartEvent) event)
          .getApplicationId());
        break;
      case APP_FINISH:
        incrementCounts(((WritingApplicationFinishEvent) event)
          .getApplicationId());
        break;
      case APP_ATTEMPT_START:
        incrementCounts(((WritingApplicationAttemptStartEvent) event)
          .getApplicationAttemptId().getApplicationId());
        break;
      case APP_ATTEMPT_FINISH:
        incrementCounts(((WritingApplicationAttemptFinishEvent) event)
          .getApplicationAttemptId().getApplicationId());
        break;
      case CONTAINER_START:
        incrementCounts(((WritingContainerStartEvent) event)
          .getContainerId().getApplicationAttemptId().getApplicationId());
        break;
      case CONTAINER_FINISH:
        incrementCounts(((WritingContainerFinishEvent) event)
          .getContainerId().getApplicationAttemptId().getApplicationId());
        break;
    }
  }
  super.dispatch(event);
}
 
Example 14
Source Project: hadoop   Source File: TestRMNMRPCResponseId.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
  Configuration conf = new Configuration();
  // Dispatcher that processes events inline
  Dispatcher dispatcher = new InlineDispatcher();
  dispatcher.register(SchedulerEventType.class, new EventHandler<Event>() {
    @Override
    public void handle(Event event) {
      ; // ignore
    }
  });
  RMContext context =
      new RMContextImpl(dispatcher, null, null, null, null,
        null, new RMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInRM(conf), null, null);
  dispatcher.register(RMNodeEventType.class,
      new ResourceManager.NodeEventDispatcher(context));
  NodesListManager nodesListManager = new NodesListManager(context);
  nodesListManager.init(conf);
  
  context.getContainerTokenSecretManager().rollMasterKey();
  context.getNMTokenSecretManager().rollMasterKey();
  resourceTrackerService = new ResourceTrackerService(context,
      nodesListManager, new NMLivelinessMonitor(dispatcher),
      context.getContainerTokenSecretManager(),
      context.getNMTokenSecretManager());
  resourceTrackerService.init(conf);
}
 
Example 15
Source Project: incubator-tez   Source File: TestAMContainer.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test
public void testContainerPreemptedAtRunning() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();

  wc.assignTaskAttempt(wc.taskAttemptID);
  wc.containerLaunched();
  wc.pullTaskToRun();
  wc.verifyState(AMContainerState.RUNNING);

  wc.containerCompleted(ContainerExitStatus.PREEMPTED);
  wc.verifyState(AMContainerState.COMPLETED);
  verify(wc.tal).registerRunningContainer(wc.containerID);
  verify(wc.tal).unregisterRunningContainer(wc.containerID);
  verify(wc.chh).register(wc.containerID);
  verify(wc.chh).unregister(wc.containerID);

  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED_BY_SYSTEM);

  assertFalse(wc.amContainer.isInErrorState());

  // Pending task complete. (Ideally, container should be dead at this point
  // and this event should not be generated. Network timeout on NM-RM heartbeat
  // can cause it to be genreated)
  wc.taskAttemptSucceeded(wc.taskAttemptID);
  wc.verifyNoOutgoingEvents();
  wc.verifyHistoryStopEvent();

  assertFalse(wc.amContainer.isInErrorState());
}
 
Example 16
Source Project: hadoop   Source File: RMStateStoreTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void handle(Event event) {
  if (event instanceof RMAppAttemptEvent) {
    RMAppAttemptEvent rmAppAttemptEvent = (RMAppAttemptEvent) event;
    assertEquals(attemptId, rmAppAttemptEvent.getApplicationAttemptId());
  }
  notified = true;
  synchronized (this) {
    notifyAll();
  }
}
 
Example 17
Source Project: incubator-tez   Source File: TestTaskAttempt.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void handle(Event event) {
  if (event instanceof DAGEvent) {
    DAGEvent je = ((DAGEvent) event);
    if (DAGEventType.INTERNAL_ERROR == je.getType()) {
      internalError = true;
    }
  }
}
 
Example 18
Source Project: tez   Source File: TestAMContainer.java    License: Apache License 2.0 5 votes vote down vote up
private Event findEventByType(List<Event> events, Enum<?> type) {
  for (Event event : events) {
    if (event.getType() == type) {
      return event;
    }
  }
  return null;
}
 
Example 19
Source Project: incubator-tez   Source File: TestTaskAttempt.java    License: Apache License 2.0 5 votes vote down vote up
private void verifyEventType(List<Event> events,
    Class<? extends Event> eventClass, int expectedOccurences) {
  int count = 0;
  for (Event e : events) {
    if (eventClass.isInstance(e)) {
      count++;
    }
  }
  assertEquals(
      "Mismatch in num occurences of event: " + eventClass.getCanonicalName(),
      expectedOccurences, count);
}
 
Example 20
Source Project: incubator-tez   Source File: TestAMContainer.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test
public void testAllocationAtRunning() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();
  wc.containerLaunched();
  wc.assignTaskAttempt(wc.taskAttemptID);
  wc.pullTaskToRun();
  wc.verifyState(AMContainerState.RUNNING);

  TezTaskAttemptID taID2 = TezTaskAttemptID.getInstance(wc.taskID, 2);
  wc.assignTaskAttempt(taID2);

  wc.verifyState(AMContainerState.STOP_REQUESTED);
  verify(wc.tal).unregisterRunningContainer(wc.containerID);
  verify(wc.chh).unregister(wc.containerID);
  // 1 for NM stop request. 2 TERMINATING to TaskAttempt.
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      NMCommunicatorEventType.CONTAINER_STOP_REQUEST,
      TaskAttemptEventType.TA_CONTAINER_TERMINATING,
      TaskAttemptEventType.TA_CONTAINER_TERMINATING);
  assertTrue(wc.amContainer.isInErrorState());

  wc.nmStopSent();
  wc.containerCompleted(false);
  wc.verifyHistoryStopEvent();
  // 1 Inform scheduler. 2 TERMINATED to TaskAttempt.
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED);

  assertNull(wc.amContainer.getRunningTaskAttempt());
  assertEquals(0, wc.amContainer.getQueuedTaskAttempts().size());
  assertEquals(2, wc.amContainer.getAllTaskAttempts().size());
}
 
Example 21
Source Project: hadoop   Source File: TestTaskAttempt.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleRackRequest() throws Exception {
  TaskAttemptImpl.RequestContainerTransition rct =
      new TaskAttemptImpl.RequestContainerTransition(false);

  EventHandler eventHandler = mock(EventHandler.class);
  String[] hosts = new String[3];
  hosts[0] = "host1";
  hosts[1] = "host2";
  hosts[2] = "host3";
  TaskSplitMetaInfo splitInfo =
      new TaskSplitMetaInfo(hosts, 0, 128 * 1024 * 1024l);

  TaskAttemptImpl mockTaskAttempt =
      createMapTaskAttemptImplForTest(eventHandler, splitInfo);
  TaskAttemptEvent mockTAEvent = mock(TaskAttemptEvent.class);

  rct.transition(mockTaskAttempt, mockTAEvent);

  ArgumentCaptor<Event> arg = ArgumentCaptor.forClass(Event.class);
  verify(eventHandler, times(2)).handle(arg.capture());
  if (!(arg.getAllValues().get(1) instanceof ContainerRequestEvent)) {
    Assert.fail("Second Event not of type ContainerRequestEvent");
  }
  ContainerRequestEvent cre =
      (ContainerRequestEvent) arg.getAllValues().get(1);
  String[] requestedRacks = cre.getRacks();
  //Only a single occurrence of /DefaultRack
  assertEquals(1, requestedRacks.length);
}
 
Example 22
Source Project: hadoop   Source File: TestTaskAttempt.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void handle(Event event) {
  if (event instanceof JobEvent) {
    JobEvent je = ((JobEvent) event);
    if (JobEventType.INTERNAL_ERROR == je.getType()) {
      internalError = true;
    }
  }
}
 
Example 23
Source Project: tez   Source File: TestAMContainer.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout=5000)
public void testNodeFailedAtRunning() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();
  wc.containerLaunched();
  wc.assignTaskAttempt(wc.taskAttemptID);
  wc.verifyState(AMContainerState.RUNNING);

  wc.nodeFailed();
  // Expecting a complete event from the RM
  wc.verifyState(AMContainerState.STOPPING);
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(4);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_NODE_FAILED,
      TaskAttemptEventType.TA_CONTAINER_TERMINATING,
      AMSchedulerEventType.S_CONTAINER_DEALLOCATE,
      AMNodeEventType.N_CONTAINER_COMPLETED);

  for (Event event : outgoingEvents) {
    if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
      TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
      assertTrue(nfEvent.getDiagnosticInfo().contains("nodeFailed"));
    }
  }

  wc.containerCompleted();
  wc.verifyHistoryStopEvent();
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED);

  assertFalse(wc.amContainer.isInErrorState());
}
 
Example 24
Source Project: incubator-tez   Source File: TestTaskSchedulerHelpers.java    License: Apache License 2.0 5 votes vote down vote up
public Event verifyInvocation(Class<? extends Event> eventClass) {
  for (Event e : events) {
    if (e.getClass().getName().equals(eventClass.getName())) {
      return e;
    }
  }
  fail("Expected Event: " + eventClass.getName() + " not sent");
  return null;
}
 
Example 25
Source Project: tez   Source File: TestAMContainer.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test (timeout=5000)
public void testContainerCompletedAtLaunchingSpecificError() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();


  wc.assignTaskAttempt(wc.taskAttemptID);

  wc.containerCompleted(ContainerExitStatus.ABORTED, TaskAttemptTerminationCause.NODE_FAILED, "NodeFailed");
  wc.verifyState(AMContainerState.COMPLETED);
  verify(wc.tal).registerRunningContainer(wc.containerID, 0);
  verifyUnregisterRunningContainer(wc.tal, wc.containerID, 0, ContainerEndReason.NODE_FAILED,
      "NodeFailed");

  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED,
      AMNodeEventType.N_CONTAINER_COMPLETED);
  Assert.assertEquals(TaskAttemptTerminationCause.NODE_FAILED,
      ((TaskAttemptEventContainerTerminated)outgoingEvents.get(0)).getTerminationCause());

  assertFalse(wc.amContainer.isInErrorState());

  // Container launched generated by NM call.
  wc.containerLaunched();
  wc.verifyNoOutgoingEvents();

  assertFalse(wc.amContainer.isInErrorState());
}
 
Example 26
Source Project: tez   Source File: TestTaskSchedulerManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testContainerInternalPreempted() throws IOException, ServicePluginException {
  Configuration conf = new Configuration(false);
  schedulerHandler.init(conf);
  schedulerHandler.start();

  AMContainer mockAmContainer = mock(AMContainer.class);
  when(mockAmContainer.getTaskSchedulerIdentifier()).thenReturn(0);
  when(mockAmContainer.getContainerLauncherIdentifier()).thenReturn(0);
  when(mockAmContainer.getTaskCommunicatorIdentifier()).thenReturn(0);
  ContainerId mockCId = mock(ContainerId.class);
  verify(mockTaskScheduler, times(0)).deallocateContainer((ContainerId) any());
  when(mockAMContainerMap.get(mockCId)).thenReturn(mockAmContainer);
  schedulerHandler.preemptContainer(0, mockCId);
  verify(mockTaskScheduler, times(1)).deallocateContainer(mockCId);
  assertEquals(1, mockEventHandler.events.size());
  Event event = mockEventHandler.events.get(0);
  assertEquals(AMContainerEventType.C_COMPLETED, event.getType());
  AMContainerEventCompleted completedEvent = (AMContainerEventCompleted) event;
  assertEquals(mockCId, completedEvent.getContainerId());
  assertEquals("Container preempted internally", completedEvent.getDiagnostics());
  assertTrue(completedEvent.isPreempted());
  Assert.assertFalse(completedEvent.isDiskFailed());
  assertEquals(TaskAttemptTerminationCause.INTERNAL_PREEMPTION,
      completedEvent.getTerminationCause());

  schedulerHandler.stop();
  schedulerHandler.close();
}
 
Example 27
Source Project: tez   Source File: TestTaskAttempt.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void handle(Event event) {
  if (event instanceof DAGEvent) {
    DAGEvent je = ((DAGEvent) event);
    if (DAGEventType.INTERNAL_ERROR == je.getType()) {
      internalError = true;
    }
  }
}
 
Example 28
Source Project: tez   Source File: TestAMContainer.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout=5000)
public void testSingleSuccessfulTaskFlowStopRequest() {
  WrappedContainer wc = new WrappedContainer();

  wc.verifyState(AMContainerState.ALLOCATED);

  wc.launchContainer();
  wc.assignTaskAttempt(wc.taskAttemptID);
  wc.containerLaunched();
  wc.taskAttemptSucceeded(wc.taskAttemptID);

  wc.stopRequest();
  wc.verifyState(AMContainerState.STOP_REQUESTED);
  // Event to NM to stop the container.

  List<Event> outgoingEvents = wc.verifyCountAndGetOutgoingEvents(2);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      ContainerLauncherEventType.CONTAINER_STOP_REQUEST,
      AMNodeEventType.N_CONTAINER_COMPLETED);

  wc.nmStopSent();
  wc.verifyState(AMContainerState.STOPPING);
  wc.verifyNoOutgoingEvents();

  wc.containerCompleted();
  wc.verifyHistoryStopEvent();
  wc.verifyState(AMContainerState.COMPLETED);
  wc.verifyNoOutgoingEvents();
  verifyUnregisterRunningContainer(wc.tal, wc.containerID, 0, ContainerEndReason.OTHER,
      "received a STOP_REQUEST");
  verify(wc.chh).unregister(wc.containerID);

  assertNull(wc.amContainer.getCurrentTaskAttempt());
  assertEquals(1, wc.amContainer.getAllTaskAttempts().size());
  assertFalse(wc.amContainer.isInErrorState());
}
 
Example 29
Source Project: big-c   Source File: TestRMApplicationHistoryWriter.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Override
protected void dispatch(Event event) {
  if (event instanceof WritingApplicationHistoryEvent) {
    WritingApplicationHistoryEvent ashEvent =
        (WritingApplicationHistoryEvent) event;
    switch (ashEvent.getType()) {
      case APP_START:
        incrementCounts(((WritingApplicationStartEvent) event)
          .getApplicationId());
        break;
      case APP_FINISH:
        incrementCounts(((WritingApplicationFinishEvent) event)
          .getApplicationId());
        break;
      case APP_ATTEMPT_START:
        incrementCounts(((WritingApplicationAttemptStartEvent) event)
          .getApplicationAttemptId().getApplicationId());
        break;
      case APP_ATTEMPT_FINISH:
        incrementCounts(((WritingApplicationAttemptFinishEvent) event)
          .getApplicationAttemptId().getApplicationId());
        break;
      case CONTAINER_START:
        incrementCounts(((WritingContainerStartEvent) event)
          .getContainerId().getApplicationAttemptId().getApplicationId());
        break;
      case CONTAINER_FINISH:
        incrementCounts(((WritingContainerFinishEvent) event)
          .getContainerId().getApplicationAttemptId().getApplicationId());
        break;
    }
  }
  super.dispatch(event);
}
 
Example 30
Source Project: incubator-tez   Source File: TestAMContainer.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Test
public void testNodeFailedAtIdle() {
  WrappedContainer wc = new WrappedContainer();
  List<Event> outgoingEvents;

  wc.launchContainer();
  wc.containerLaunched();
  wc.assignTaskAttempt(wc.taskAttemptID);
  wc.verifyState(AMContainerState.IDLE);

  wc.nodeFailed();
  // Expecting a complete event from the RM
  wc.verifyState(AMContainerState.STOPPING);
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(3);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_NODE_FAILED,
      TaskAttemptEventType.TA_CONTAINER_TERMINATING,
      AMSchedulerEventType.S_CONTAINER_DEALLOCATE);

  for (Event event : outgoingEvents) {
    if (event.getType() == TaskAttemptEventType.TA_NODE_FAILED) {
      TaskAttemptEventNodeFailed nfEvent = (TaskAttemptEventNodeFailed) event;
      assertEquals("nodeFailed", nfEvent.getDiagnosticInfo());
    }
  }

  wc.containerCompleted(false);
  wc.verifyHistoryStopEvent();
  outgoingEvents = wc.verifyCountAndGetOutgoingEvents(1);
  verifyUnOrderedOutgoingEventTypes(outgoingEvents,
      TaskAttemptEventType.TA_CONTAINER_TERMINATED);

  assertFalse(wc.amContainer.isInErrorState());
}