Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: MiniYARNCluster.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void initResourceManager(int index, Configuration conf) {
  if (HAUtil.isHAEnabled(conf)) {
    conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
  }
  resourceManagers[index].init(conf);
  resourceManagers[index].getRMContext().getDispatcher().register(
      RMAppAttemptEventType.class,
      new EventHandler<RMAppAttemptEvent>() {
        public void handle(RMAppAttemptEvent event) {
          if (event instanceof RMAppAttemptRegistrationEvent) {
            appMasters.put(event.getApplicationAttemptId(),
                event.getTimestamp());
          } else if (event instanceof RMAppAttemptUnregistrationEvent) {
            appMasters.remove(event.getApplicationAttemptId());
          }
        }
      });
}
 
Example 2
Source Project: hadoop   Source File: RMStateStore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void transition(RMStateStore store, RMStateStoreEvent event) {
  if (!(event instanceof RMStateStoreAppAttemptEvent)) {
    // should never happen
    LOG.error("Illegal event type: " + event.getClass());
    return;
  }
  ApplicationAttemptStateData attemptState =
      ((RMStateStoreAppAttemptEvent) event).getAppAttemptState();
  try {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Storing info for attempt: " + attemptState.getAttemptId());
    }
    store.storeApplicationAttemptStateInternal(attemptState.getAttemptId(),
        attemptState);
    store.notifyApplicationAttempt(new RMAppAttemptEvent
           (attemptState.getAttemptId(),
           RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
  } catch (Exception e) {
    LOG.error("Error storing appAttempt: " + attemptState.getAttemptId(), e);
    store.notifyStoreOperationFailed(e);
  }
}
 
Example 3
Source Project: hadoop   Source File: RMStateStore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void transition(RMStateStore store, RMStateStoreEvent event) {
  if (!(event instanceof RMStateUpdateAppAttemptEvent)) {
    // should never happen
    LOG.error("Illegal event type: " + event.getClass());
    return;
  }
  ApplicationAttemptStateData attemptState =
      ((RMStateUpdateAppAttemptEvent) event).getAppAttemptState();
  try {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Updating info for attempt: " + attemptState.getAttemptId());
    }
    store.updateApplicationAttemptStateInternal(attemptState.getAttemptId(),
        attemptState);
    store.notifyApplicationAttempt(new RMAppAttemptEvent
           (attemptState.getAttemptId(),
           RMAppAttemptEventType.ATTEMPT_UPDATE_SAVED));
  } catch (Exception e) {
    LOG.error("Error updating appAttempt: " + attemptState.getAttemptId(), e);
    store.notifyStoreOperationFailed(e);
  }
}
 
Example 4
Source Project: hadoop   Source File: MockRM.java    License: Apache License 2.0 6 votes vote down vote up
public MockAM sendAMLaunched(ApplicationAttemptId appAttemptId)
    throws Exception {
  MockAM am = new MockAM(getRMContext(), masterService, appAttemptId);
  am.waitForState(RMAppAttemptState.ALLOCATED);
  //create and set AMRMToken
  Token<AMRMTokenIdentifier> amrmToken =
      this.rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken(
        appAttemptId);
  ((RMAppAttemptImpl) this.rmContext.getRMApps()
    .get(appAttemptId.getApplicationId()).getRMAppAttempt(appAttemptId))
    .setAMRMToken(amrmToken);
  getRMContext()
      .getDispatcher()
      .getEventHandler()
      .handle(
          new RMAppAttemptEvent(appAttemptId, RMAppAttemptEventType.LAUNCHED));
  return am;
}
 
Example 5
Source Project: big-c   Source File: MiniYARNCluster.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void initResourceManager(int index, Configuration conf) {
  if (HAUtil.isHAEnabled(conf)) {
    conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
  }
  resourceManagers[index].init(conf);
  resourceManagers[index].getRMContext().getDispatcher().register(
      RMAppAttemptEventType.class,
      new EventHandler<RMAppAttemptEvent>() {
        public void handle(RMAppAttemptEvent event) {
          if (event instanceof RMAppAttemptRegistrationEvent) {
            appMasters.put(event.getApplicationAttemptId(),
                event.getTimestamp());
          } else if (event instanceof RMAppAttemptUnregistrationEvent) {
            appMasters.remove(event.getApplicationAttemptId());
          }
        }
      });
}
 
Example 6
Source Project: big-c   Source File: RMStateStore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void transition(RMStateStore store, RMStateStoreEvent event) {
  if (!(event instanceof RMStateStoreAppAttemptEvent)) {
    // should never happen
    LOG.error("Illegal event type: " + event.getClass());
    return;
  }
  ApplicationAttemptStateData attemptState =
      ((RMStateStoreAppAttemptEvent) event).getAppAttemptState();
  try {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Storing info for attempt: " + attemptState.getAttemptId());
    }
    store.storeApplicationAttemptStateInternal(attemptState.getAttemptId(),
        attemptState);
    store.notifyApplicationAttempt(new RMAppAttemptEvent
           (attemptState.getAttemptId(),
           RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
  } catch (Exception e) {
    LOG.error("Error storing appAttempt: " + attemptState.getAttemptId(), e);
    store.notifyStoreOperationFailed(e);
  }
}
 
Example 7
Source Project: big-c   Source File: RMStateStore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void transition(RMStateStore store, RMStateStoreEvent event) {
  if (!(event instanceof RMStateUpdateAppAttemptEvent)) {
    // should never happen
    LOG.error("Illegal event type: " + event.getClass());
    return;
  }
  ApplicationAttemptStateData attemptState =
      ((RMStateUpdateAppAttemptEvent) event).getAppAttemptState();
  try {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Updating info for attempt: " + attemptState.getAttemptId());
    }
    store.updateApplicationAttemptStateInternal(attemptState.getAttemptId(),
        attemptState);
    store.notifyApplicationAttempt(new RMAppAttemptEvent
           (attemptState.getAttemptId(),
           RMAppAttemptEventType.ATTEMPT_UPDATE_SAVED));
  } catch (Exception e) {
    LOG.error("Error updating appAttempt: " + attemptState.getAttemptId(), e);
    store.notifyStoreOperationFailed(e);
  }
}
 
Example 8
Source Project: big-c   Source File: MockRM.java    License: Apache License 2.0 6 votes vote down vote up
public MockAM sendAMLaunched(ApplicationAttemptId appAttemptId)
    throws Exception {
  MockAM am = new MockAM(getRMContext(), masterService, appAttemptId);
  am.waitForState(RMAppAttemptState.ALLOCATED);
  //create and set AMRMToken
  Token<AMRMTokenIdentifier> amrmToken =
      this.rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken(
        appAttemptId);
  ((RMAppAttemptImpl) this.rmContext.getRMApps()
    .get(appAttemptId.getApplicationId()).getRMAppAttempt(appAttemptId))
    .setAMRMToken(amrmToken);
  getRMContext()
      .getDispatcher()
      .getEventHandler()
      .handle(
          new RMAppAttemptEvent(appAttemptId, RMAppAttemptEventType.LAUNCHED));
  return am;
}
 
Example 9
Source Project: hadoop   Source File: RMAppAttemptUnregistrationEvent.java    License: Apache License 2.0 5 votes vote down vote up
public RMAppAttemptUnregistrationEvent(ApplicationAttemptId appAttemptId,
    String trackingUrl, FinalApplicationStatus finalStatus, String diagnostics) {
  super(appAttemptId, RMAppAttemptEventType.UNREGISTERED);
  this.finalTrackingUrl = trackingUrl;
  this.finalStatus = finalStatus;
  this.diagnostics = diagnostics;
}
 
Example 10
Source Project: hadoop   Source File: RMAppAttemptRegistrationEvent.java    License: Apache License 2.0 5 votes vote down vote up
public RMAppAttemptRegistrationEvent(ApplicationAttemptId appAttemptId,
    String host, int rpcPort, String trackingUrl) {
  super(appAttemptId, RMAppAttemptEventType.REGISTERED);
  this.appAttemptId = appAttemptId;
  this.host = host;
  this.rpcport = rpcPort;
  this.trackingurl = trackingUrl;
}
 
Example 11
Source Project: hadoop   Source File: FifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public synchronized void
    addApplicationAttempt(ApplicationAttemptId appAttemptId,
        boolean transferStateFromPreviousAttempt,
        boolean isAttemptRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(appAttemptId.getApplicationId());
  String user = application.getUser();
  // TODO: Fix store
  FiCaSchedulerApp schedulerApp =
      new FiCaSchedulerApp(appAttemptId, user, DEFAULT_QUEUE,
        activeUsersManager, this.rmContext);

  if (transferStateFromPreviousAttempt) {
    schedulerApp.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(schedulerApp);

  metrics.submitAppAttempt(user);
  LOG.info("Added Application Attempt " + appAttemptId
      + " to scheduler from user " + application.getUser());
  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(appAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(appAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 
Example 12
Source Project: hadoop   Source File: CapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt,
    boolean isAttemptRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(applicationAttemptId.getApplicationId());
  CSQueue queue = (CSQueue) application.getQueue();

  FiCaSchedulerApp attempt =
      new FiCaSchedulerApp(applicationAttemptId, application.getUser(),
        queue, queue.getActiveUsersManager(), rmContext);
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  queue.submitApplicationAttempt(attempt, application.getUser());
  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user " + application.getUser() + " in queue "
      + queue.getQueueName());
  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(applicationAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 
Example 13
Source Project: big-c   Source File: RMAppAttemptUnregistrationEvent.java    License: Apache License 2.0 5 votes vote down vote up
public RMAppAttemptUnregistrationEvent(ApplicationAttemptId appAttemptId,
    String trackingUrl, FinalApplicationStatus finalStatus, String diagnostics) {
  super(appAttemptId, RMAppAttemptEventType.UNREGISTERED);
  this.finalTrackingUrl = trackingUrl;
  this.finalStatus = finalStatus;
  this.diagnostics = diagnostics;
}
 
Example 14
Source Project: big-c   Source File: RMAppAttemptRegistrationEvent.java    License: Apache License 2.0 5 votes vote down vote up
public RMAppAttemptRegistrationEvent(ApplicationAttemptId appAttemptId,
    String host, int rpcPort, String trackingUrl) {
  super(appAttemptId, RMAppAttemptEventType.REGISTERED);
  this.appAttemptId = appAttemptId;
  this.host = host;
  this.rpcport = rpcPort;
  this.trackingurl = trackingUrl;
}
 
Example 15
Source Project: big-c   Source File: FifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public synchronized void
    addApplicationAttempt(ApplicationAttemptId appAttemptId,
        boolean transferStateFromPreviousAttempt,
        boolean isAttemptRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(appAttemptId.getApplicationId());
  String user = application.getUser();
  // TODO: Fix store
  FiCaSchedulerApp schedulerApp =
      new FiCaSchedulerApp(appAttemptId, user, DEFAULT_QUEUE,
        activeUsersManager, this.rmContext);

  if (transferStateFromPreviousAttempt) {
    schedulerApp.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(schedulerApp);

  metrics.submitAppAttempt(user);
  LOG.info("Added Application Attempt " + appAttemptId
      + " to scheduler from user " + application.getUser());
  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(appAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(appAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 
Example 16
Source Project: big-c   Source File: CapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt,
    boolean isAttemptRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(applicationAttemptId.getApplicationId());
  CSQueue queue = (CSQueue) application.getQueue();

  FiCaSchedulerApp attempt =
      new FiCaSchedulerApp(applicationAttemptId, application.getUser(),
        queue, queue.getActiveUsersManager(), rmContext);
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  queue.submitApplicationAttempt(attempt, application.getUser());
  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user " + application.getUser() + " in queue "
      + queue.getQueueName());
  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(applicationAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 
Example 17
private synchronized void initResourceManager(int index, Configuration conf) {
    if (HAUtil.isHAEnabled(conf)) {
        conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
    }

    if (conf.get(YarnConfiguration.RM_HOSTNAME) == null) {
        conf.set(YarnConfiguration.RM_HOSTNAME, "0.0.0.0");
    }
    LOG.info("*** "+YarnConfiguration.RM_HOSTNAME+" is set to: "+conf.get(YarnConfiguration.RM_HOSTNAME));

    if (conf.get(YarnConfiguration.RM_ADDRESS) == null) {
        conf.set(YarnConfiguration.RM_ADDRESS,
                   YarnConfiguration.DEFAULT_RM_ADDRESS);
    }
    LOG.info("*** "+YarnConfiguration.RM_ADDRESS+" is set to: "+conf.get(YarnConfiguration.RM_ADDRESS));

    if (conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS) == null) {
        WebAppUtils
            .setNMWebAppHostNameAndPort(conf,
                                        MiniYARNClusterSplice.getHostname(), 0);
    }
    LOG.info("*** "+YarnConfiguration.RM_WEBAPP_ADDRESS+" is set to: "+conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS));

    resourceManagers[index].init(conf);
    resourceManagers[index].getRMContext().getDispatcher().register(
        RMAppAttemptEventType.class,
        new EventHandler<RMAppAttemptEvent>() {
            public void handle(RMAppAttemptEvent event) {
                if (event instanceof RMAppAttemptRegistrationEvent) {
                    appMasters.put(event.getApplicationAttemptId(),
                                   event.getTimestamp());
                } else if (event instanceof RMAppAttemptUnregistrationEvent) {
                    appMasters.remove(event.getApplicationAttemptId());
                }
            }
        });
}
 
Example 18
Source Project: hadoop   Source File: RMAppImpl.java    License: Apache License 2.0 4 votes vote down vote up
private void recoverAppAttempts() {
  for (RMAppAttempt attempt : getAppAttempts().values()) {
    attempt.handle(new RMAppAttemptEvent(attempt.getAppAttemptId(),
      RMAppAttemptEventType.RECOVER));
  }
}
 
Example 19
Source Project: hadoop   Source File: RMAppImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void transition(RMAppImpl app, RMAppEvent event) {
  app.stateBeforeKilling = app.getState();
  app.handler.handle(new RMAppAttemptEvent(app.currentAttempt
    .getAppAttemptId(), RMAppAttemptEventType.KILL));
}
 
Example 20
public RMAppAttemptContainerAllocatedEvent(ApplicationAttemptId appAttemptId) {
  super(appAttemptId, RMAppAttemptEventType.CONTAINER_ALLOCATED);
}
 
Example 21
Source Project: hadoop   Source File: RMAppAttemptContainerFinishedEvent.java    License: Apache License 2.0 4 votes vote down vote up
public RMAppAttemptContainerFinishedEvent(ApplicationAttemptId appAttemptId, 
    ContainerStatus containerStatus, NodeId nodeId) {
  super(appAttemptId, RMAppAttemptEventType.CONTAINER_FINISHED);
  this.containerStatus = containerStatus;
  this.nodeId = nodeId;
}
 
Example 22
Source Project: hadoop   Source File: RMAppAttemptLaunchFailedEvent.java    License: Apache License 2.0 4 votes vote down vote up
public RMAppAttemptLaunchFailedEvent(ApplicationAttemptId appAttemptId,
    String message) {
  super(appAttemptId, RMAppAttemptEventType.LAUNCH_FAILED);
  this.message = message;
}
 
Example 23
Source Project: hadoop   Source File: RMAppAttemptStatusupdateEvent.java    License: Apache License 2.0 4 votes vote down vote up
public RMAppAttemptStatusupdateEvent(ApplicationAttemptId appAttemptId,
    float progress) {
  super(appAttemptId, RMAppAttemptEventType.STATUS_UPDATE);
  this.progress = progress;
}
 
Example 24
Source Project: hadoop   Source File: FairScheduler.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Add a new application attempt to the scheduler.
 */
protected synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt,
    boolean isAttemptRecovering) {
  SchedulerApplication<FSAppAttempt> application =
      applications.get(applicationAttemptId.getApplicationId());
  String user = application.getUser();
  FSLeafQueue queue = (FSLeafQueue) application.getQueue();

  FSAppAttempt attempt =
      new FSAppAttempt(this, applicationAttemptId, user,
          queue, new ActiveUsersManager(getRootQueueMetrics()),
          rmContext);
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(application
        .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  boolean runnable = maxRunningEnforcer.canAppBeRunnable(queue, user);
  queue.addApp(attempt, runnable);
  if (runnable) {
    maxRunningEnforcer.trackRunnableApp(attempt);
  } else {
    maxRunningEnforcer.trackNonRunnableApp(attempt);
  }
  
  queue.getMetrics().submitAppAttempt(user);

  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user: " + user);

  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(applicationAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 
Example 25
Source Project: hadoop   Source File: TestRMContainerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testReleaseWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);
  ConcurrentMap<ApplicationId, RMApp> rmApps =
      spy(new ConcurrentHashMap<ApplicationId, RMApp>());
  RMApp rmApp = mock(RMApp.class);
  when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
  Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any());

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getRMApps()).thenReturn(rmApps);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify RELEASED and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.RELEASED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.RELEASED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
  assertEquals(SchedulerUtils.RELEASED_CONTAINER,
      rmContainer.getDiagnosticsInfo());
  assertEquals(ContainerExitStatus.ABORTED,
      rmContainer.getContainerExitStatus());
  assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState());
  verify(writer).containerFinished(any(RMContainer.class));
  verify(publisher).containerFinished(any(RMContainer.class), anyLong());

  ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor
      .forClass(RMAppAttemptContainerFinishedEvent.class);
  verify(appAttemptEventHandler).handle(captor.capture());
  RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue();
  assertEquals(appAttemptId, cfEvent.getApplicationAttemptId());
  assertEquals(containerStatus, cfEvent.getContainerStatus());
  assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType());
  
  // In RELEASED state. A FINIHSED event may come in.
  rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils
      .createAbnormalContainerStatus(containerId, "FinishedContainer"),
      RMContainerEventType.FINISHED));
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
}
 
Example 26
Source Project: hadoop   Source File: TestRMContainerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testExpireWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify EXPIRE and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.EXPIRED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.EXPIRE));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  verify(writer, never()).containerFinished(any(RMContainer.class));
  verify(publisher, never()).containerFinished(any(RMContainer.class),
      anyLong());
}
 
Example 27
Source Project: hadoop   Source File: TestRM.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Validate killing an application when it is at accepted state.
 * @throws Exception exception
 */
@Test (timeout = 60000)
public void testApplicationKillAtAcceptedState() throws Exception {

  final Dispatcher dispatcher = new AsyncDispatcher() {
    @Override
    public EventHandler getEventHandler() {

      class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
        @Override
        public boolean matches(Object argument) {
          if (argument instanceof RMAppAttemptEvent) {
            if (((RMAppAttemptEvent) argument).getType().equals(
              RMAppAttemptEventType.KILL)) {
              return true;
            }
          }
          return false;
        }
      }

      EventHandler handler = spy(super.getEventHandler());
      doNothing().when(handler).handle(argThat(new EventArgMatcher()));
      return handler;
    }
  };

  MockRM rm = new MockRM(conf) {
    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };

  // test metrics
  QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
  int appsKilled = metrics.getAppsKilled();
  int appsSubmitted = metrics.getAppsSubmitted();

  rm.start();
  
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm1.registerNode();

  // a failed app
  RMApp application = rm.submitApp(200);
  MockAM am = MockRM.launchAM(application, rm, nm1);
  am.waitForState(RMAppAttemptState.LAUNCHED);
  nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.RUNNING);
  rm.waitForState(application.getApplicationId(), RMAppState.ACCEPTED);

  // Now kill the application before new attempt is launched, the app report
  // returns the invalid AM host and port.
  KillApplicationRequest request =
      KillApplicationRequest.newInstance(application.getApplicationId());
  rm.getClientRMService().forceKillApplication(request);

  // Specific test for YARN-1689 follows
  // Now let's say a race causes AM to register now. This should not crash RM.
  am.registerAppAttempt(false);

  // We explicitly intercepted the kill-event to RMAppAttempt, so app should
  // still be in KILLING state.
  rm.waitForState(application.getApplicationId(), RMAppState.KILLING);
  // AM should now be in running
  rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.RUNNING);

  // Simulate that appAttempt is killed.
  rm.getRMContext().getDispatcher().getEventHandler().handle(
      new RMAppEvent(application.getApplicationId(),
        RMAppEventType.ATTEMPT_KILLED));
  rm.waitForState(application.getApplicationId(), RMAppState.KILLED);

  // test metrics
  metrics = rm.getResourceScheduler().getRootQueueMetrics();
  Assert.assertEquals(appsKilled + 1, metrics.getAppsKilled());
  Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted());
}
 
Example 28
Source Project: hadoop   Source File: TestRM.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout = 30000)
public void testKillFinishingApp() throws Exception{

  // this dispatcher ignores RMAppAttemptEventType.KILL event
  final Dispatcher dispatcher = new AsyncDispatcher() {
    @Override
    public EventHandler getEventHandler() {

      class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
        @Override
        public boolean matches(Object argument) {
          if (argument instanceof RMAppAttemptEvent) {
            if (((RMAppAttemptEvent) argument).getType().equals(
              RMAppAttemptEventType.KILL)) {
              return true;
            }
          }
          return false;
        }
      }

      EventHandler handler = spy(super.getEventHandler());
      doNothing().when(handler).handle(argThat(new EventArgMatcher()));
      return handler;
    }
  };

  MockRM rm1 = new MockRM(conf){
    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  rm1.killApp(app1.getApplicationId());

  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  am1.unregisterAppAttempt(req,true);

  rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHING);
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
  rm1.waitForState(app1.getApplicationId(), RMAppState.FINISHED);
}
 
Example 29
Source Project: hadoop   Source File: TestRM.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout = 30000)
public void testKillFailingApp() throws Exception{

  // this dispatcher ignores RMAppAttemptEventType.KILL event
  final Dispatcher dispatcher = new AsyncDispatcher() {
    @Override
    public EventHandler getEventHandler() {

      class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
        @Override
        public boolean matches(Object argument) {
          if (argument instanceof RMAppAttemptEvent) {
            if (((RMAppAttemptEvent) argument).getType().equals(
              RMAppAttemptEventType.KILL)) {
              return true;
            }
          }
          return false;
        }
      }

      EventHandler handler = spy(super.getEventHandler());
      doNothing().when(handler).handle(argThat(new EventArgMatcher()));
      return handler;
    }
  };

  MockRM rm1 = new MockRM(conf){
    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  rm1.killApp(app1.getApplicationId());

  // fail the app by sending container_finished event.
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
  // app is killed, not launching a new attempt
  rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
}
 
Example 30
Source Project: hadoop   Source File: TestRMAppTransitions.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
  if (isSecurityEnabled) {
    authMethod = AuthenticationMethod.KERBEROS;
  }
  SecurityUtil.setAuthenticationMethod(authMethod, conf);
  UserGroupInformation.setConfiguration(conf);

  rmDispatcher = new DrainDispatcher();
  ContainerAllocationExpirer containerAllocationExpirer = 
      mock(ContainerAllocationExpirer.class);
  AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
  AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class);
  store = mock(RMStateStore.class);
  writer = mock(RMApplicationHistoryWriter.class);
  DelegationTokenRenewer renewer = mock(DelegationTokenRenewer.class);
  RMContext realRMContext = 
      new RMContextImpl(rmDispatcher,
        containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
        renewer, new AMRMTokenSecretManager(conf, this.rmContext),
        new RMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInRM(conf),
        new ClientToAMTokenSecretManagerInRM(),
        writer);
  ((RMContextImpl)realRMContext).setStateStore(store);
  publisher = mock(SystemMetricsPublisher.class);
  ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher);

  this.rmContext = spy(realRMContext);

  ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
  doReturn(null).when(resourceScheduler)
            .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
  doReturn(resourceScheduler).when(rmContext).getScheduler();

  rmDispatcher.register(RMAppAttemptEventType.class,
      new TestApplicationAttemptEventDispatcher(this.rmContext));

  rmDispatcher.register(RMAppEventType.class,
      new TestApplicationEventDispatcher(rmContext));
  
  rmDispatcher.register(RMAppManagerEventType.class,
      new TestApplicationManagerEventDispatcher());
  
  schedulerDispatcher = new TestSchedulerEventDispatcher();
  rmDispatcher.register(SchedulerEventType.class,
      schedulerDispatcher);
  
  rmDispatcher.init(conf);
  rmDispatcher.start();
}