org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MiniYARNCluster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private synchronized void initResourceManager(int index, Configuration conf) {
  if (HAUtil.isHAEnabled(conf)) {
    conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
  }
  resourceManagers[index].init(conf);
  resourceManagers[index].getRMContext().getDispatcher().register(
      RMAppAttemptEventType.class,
      new EventHandler<RMAppAttemptEvent>() {
        public void handle(RMAppAttemptEvent event) {
          if (event instanceof RMAppAttemptRegistrationEvent) {
            appMasters.put(event.getApplicationAttemptId(),
                event.getTimestamp());
          } else if (event instanceof RMAppAttemptUnregistrationEvent) {
            appMasters.remove(event.getApplicationAttemptId());
          }
        }
      });
}
 
Example #2
Source File: MockRM.java    From big-c with Apache License 2.0 6 votes vote down vote up
public MockAM sendAMLaunched(ApplicationAttemptId appAttemptId)
    throws Exception {
  MockAM am = new MockAM(getRMContext(), masterService, appAttemptId);
  am.waitForState(RMAppAttemptState.ALLOCATED);
  //create and set AMRMToken
  Token<AMRMTokenIdentifier> amrmToken =
      this.rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken(
        appAttemptId);
  ((RMAppAttemptImpl) this.rmContext.getRMApps()
    .get(appAttemptId.getApplicationId()).getRMAppAttempt(appAttemptId))
    .setAMRMToken(amrmToken);
  getRMContext()
      .getDispatcher()
      .getEventHandler()
      .handle(
          new RMAppAttemptEvent(appAttemptId, RMAppAttemptEventType.LAUNCHED));
  return am;
}
 
Example #3
Source File: RMStateStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void transition(RMStateStore store, RMStateStoreEvent event) {
  if (!(event instanceof RMStateUpdateAppAttemptEvent)) {
    // should never happen
    LOG.error("Illegal event type: " + event.getClass());
    return;
  }
  ApplicationAttemptStateData attemptState =
      ((RMStateUpdateAppAttemptEvent) event).getAppAttemptState();
  try {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Updating info for attempt: " + attemptState.getAttemptId());
    }
    store.updateApplicationAttemptStateInternal(attemptState.getAttemptId(),
        attemptState);
    store.notifyApplicationAttempt(new RMAppAttemptEvent
           (attemptState.getAttemptId(),
           RMAppAttemptEventType.ATTEMPT_UPDATE_SAVED));
  } catch (Exception e) {
    LOG.error("Error updating appAttempt: " + attemptState.getAttemptId(), e);
    store.notifyStoreOperationFailed(e);
  }
}
 
Example #4
Source File: RMStateStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void transition(RMStateStore store, RMStateStoreEvent event) {
  if (!(event instanceof RMStateStoreAppAttemptEvent)) {
    // should never happen
    LOG.error("Illegal event type: " + event.getClass());
    return;
  }
  ApplicationAttemptStateData attemptState =
      ((RMStateStoreAppAttemptEvent) event).getAppAttemptState();
  try {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Storing info for attempt: " + attemptState.getAttemptId());
    }
    store.storeApplicationAttemptStateInternal(attemptState.getAttemptId(),
        attemptState);
    store.notifyApplicationAttempt(new RMAppAttemptEvent
           (attemptState.getAttemptId(),
           RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
  } catch (Exception e) {
    LOG.error("Error storing appAttempt: " + attemptState.getAttemptId(), e);
    store.notifyStoreOperationFailed(e);
  }
}
 
Example #5
Source File: MiniYARNCluster.java    From big-c with Apache License 2.0 6 votes vote down vote up
private synchronized void initResourceManager(int index, Configuration conf) {
  if (HAUtil.isHAEnabled(conf)) {
    conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
  }
  resourceManagers[index].init(conf);
  resourceManagers[index].getRMContext().getDispatcher().register(
      RMAppAttemptEventType.class,
      new EventHandler<RMAppAttemptEvent>() {
        public void handle(RMAppAttemptEvent event) {
          if (event instanceof RMAppAttemptRegistrationEvent) {
            appMasters.put(event.getApplicationAttemptId(),
                event.getTimestamp());
          } else if (event instanceof RMAppAttemptUnregistrationEvent) {
            appMasters.remove(event.getApplicationAttemptId());
          }
        }
      });
}
 
Example #6
Source File: MockRM.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public MockAM sendAMLaunched(ApplicationAttemptId appAttemptId)
    throws Exception {
  MockAM am = new MockAM(getRMContext(), masterService, appAttemptId);
  am.waitForState(RMAppAttemptState.ALLOCATED);
  //create and set AMRMToken
  Token<AMRMTokenIdentifier> amrmToken =
      this.rmContext.getAMRMTokenSecretManager().createAndGetAMRMToken(
        appAttemptId);
  ((RMAppAttemptImpl) this.rmContext.getRMApps()
    .get(appAttemptId.getApplicationId()).getRMAppAttempt(appAttemptId))
    .setAMRMToken(amrmToken);
  getRMContext()
      .getDispatcher()
      .getEventHandler()
      .handle(
          new RMAppAttemptEvent(appAttemptId, RMAppAttemptEventType.LAUNCHED));
  return am;
}
 
Example #7
Source File: RMStateStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void transition(RMStateStore store, RMStateStoreEvent event) {
  if (!(event instanceof RMStateUpdateAppAttemptEvent)) {
    // should never happen
    LOG.error("Illegal event type: " + event.getClass());
    return;
  }
  ApplicationAttemptStateData attemptState =
      ((RMStateUpdateAppAttemptEvent) event).getAppAttemptState();
  try {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Updating info for attempt: " + attemptState.getAttemptId());
    }
    store.updateApplicationAttemptStateInternal(attemptState.getAttemptId(),
        attemptState);
    store.notifyApplicationAttempt(new RMAppAttemptEvent
           (attemptState.getAttemptId(),
           RMAppAttemptEventType.ATTEMPT_UPDATE_SAVED));
  } catch (Exception e) {
    LOG.error("Error updating appAttempt: " + attemptState.getAttemptId(), e);
    store.notifyStoreOperationFailed(e);
  }
}
 
Example #8
Source File: RMStateStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void transition(RMStateStore store, RMStateStoreEvent event) {
  if (!(event instanceof RMStateStoreAppAttemptEvent)) {
    // should never happen
    LOG.error("Illegal event type: " + event.getClass());
    return;
  }
  ApplicationAttemptStateData attemptState =
      ((RMStateStoreAppAttemptEvent) event).getAppAttemptState();
  try {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Storing info for attempt: " + attemptState.getAttemptId());
    }
    store.storeApplicationAttemptStateInternal(attemptState.getAttemptId(),
        attemptState);
    store.notifyApplicationAttempt(new RMAppAttemptEvent
           (attemptState.getAttemptId(),
           RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
  } catch (Exception e) {
    LOG.error("Error storing appAttempt: " + attemptState.getAttemptId(), e);
    store.notifyStoreOperationFailed(e);
  }
}
 
Example #9
Source File: MiniYARNClusterSplice.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
private synchronized void initResourceManager(int index, Configuration conf) {
    if (HAUtil.isHAEnabled(conf)) {
        conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
    }

    if (conf.get(YarnConfiguration.RM_HOSTNAME) == null) {
        conf.set(YarnConfiguration.RM_HOSTNAME, "0.0.0.0");
    }
    LOG.info("*** "+YarnConfiguration.RM_HOSTNAME+" is set to: "+conf.get(YarnConfiguration.RM_HOSTNAME));

    if (conf.get(YarnConfiguration.RM_ADDRESS) == null) {
        conf.set(YarnConfiguration.RM_ADDRESS,
                   YarnConfiguration.DEFAULT_RM_ADDRESS);
    }
    LOG.info("*** "+YarnConfiguration.RM_ADDRESS+" is set to: "+conf.get(YarnConfiguration.RM_ADDRESS));

    if (conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS) == null) {
        WebAppUtils
            .setNMWebAppHostNameAndPort(conf,
                                        MiniYARNClusterSplice.getHostname(), 0);
    }
    LOG.info("*** "+YarnConfiguration.RM_WEBAPP_ADDRESS+" is set to: "+conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS));

    resourceManagers[index].init(conf);
    resourceManagers[index].getRMContext().getDispatcher().register(
        RMAppAttemptEventType.class,
        new EventHandler<RMAppAttemptEvent>() {
            public void handle(RMAppAttemptEvent event) {
                if (event instanceof RMAppAttemptRegistrationEvent) {
                    appMasters.put(event.getApplicationAttemptId(),
                                   event.getTimestamp());
                } else if (event instanceof RMAppAttemptUnregistrationEvent) {
                    appMasters.remove(event.getApplicationAttemptId());
                }
            }
        });
}
 
Example #10
Source File: CapacityScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt,
    boolean isAttemptRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(applicationAttemptId.getApplicationId());
  CSQueue queue = (CSQueue) application.getQueue();

  FiCaSchedulerApp attempt =
      new FiCaSchedulerApp(applicationAttemptId, application.getUser(),
        queue, queue.getActiveUsersManager(), rmContext);
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  queue.submitApplicationAttempt(attempt, application.getUser());
  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user " + application.getUser() + " in queue "
      + queue.getQueueName());
  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(applicationAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 
Example #11
Source File: FifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public synchronized void
    addApplicationAttempt(ApplicationAttemptId appAttemptId,
        boolean transferStateFromPreviousAttempt,
        boolean isAttemptRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(appAttemptId.getApplicationId());
  String user = application.getUser();
  // TODO: Fix store
  FiCaSchedulerApp schedulerApp =
      new FiCaSchedulerApp(appAttemptId, user, DEFAULT_QUEUE,
        activeUsersManager, this.rmContext);

  if (transferStateFromPreviousAttempt) {
    schedulerApp.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(schedulerApp);

  metrics.submitAppAttempt(user);
  LOG.info("Added Application Attempt " + appAttemptId
      + " to scheduler from user " + application.getUser());
  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(appAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(appAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 
Example #12
Source File: RMAppAttemptRegistrationEvent.java    From big-c with Apache License 2.0 5 votes vote down vote up
public RMAppAttemptRegistrationEvent(ApplicationAttemptId appAttemptId,
    String host, int rpcPort, String trackingUrl) {
  super(appAttemptId, RMAppAttemptEventType.REGISTERED);
  this.appAttemptId = appAttemptId;
  this.host = host;
  this.rpcport = rpcPort;
  this.trackingurl = trackingUrl;
}
 
Example #13
Source File: RMAppAttemptUnregistrationEvent.java    From big-c with Apache License 2.0 5 votes vote down vote up
public RMAppAttemptUnregistrationEvent(ApplicationAttemptId appAttemptId,
    String trackingUrl, FinalApplicationStatus finalStatus, String diagnostics) {
  super(appAttemptId, RMAppAttemptEventType.UNREGISTERED);
  this.finalTrackingUrl = trackingUrl;
  this.finalStatus = finalStatus;
  this.diagnostics = diagnostics;
}
 
Example #14
Source File: CapacityScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt,
    boolean isAttemptRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(applicationAttemptId.getApplicationId());
  CSQueue queue = (CSQueue) application.getQueue();

  FiCaSchedulerApp attempt =
      new FiCaSchedulerApp(applicationAttemptId, application.getUser(),
        queue, queue.getActiveUsersManager(), rmContext);
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  queue.submitApplicationAttempt(attempt, application.getUser());
  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user " + application.getUser() + " in queue "
      + queue.getQueueName());
  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(applicationAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 
Example #15
Source File: RMAppAttemptRegistrationEvent.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public RMAppAttemptRegistrationEvent(ApplicationAttemptId appAttemptId,
    String host, int rpcPort, String trackingUrl) {
  super(appAttemptId, RMAppAttemptEventType.REGISTERED);
  this.appAttemptId = appAttemptId;
  this.host = host;
  this.rpcport = rpcPort;
  this.trackingurl = trackingUrl;
}
 
Example #16
Source File: RMAppAttemptUnregistrationEvent.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public RMAppAttemptUnregistrationEvent(ApplicationAttemptId appAttemptId,
    String trackingUrl, FinalApplicationStatus finalStatus, String diagnostics) {
  super(appAttemptId, RMAppAttemptEventType.UNREGISTERED);
  this.finalTrackingUrl = trackingUrl;
  this.finalStatus = finalStatus;
  this.diagnostics = diagnostics;
}
 
Example #17
Source File: FifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public synchronized void
    addApplicationAttempt(ApplicationAttemptId appAttemptId,
        boolean transferStateFromPreviousAttempt,
        boolean isAttemptRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(appAttemptId.getApplicationId());
  String user = application.getUser();
  // TODO: Fix store
  FiCaSchedulerApp schedulerApp =
      new FiCaSchedulerApp(appAttemptId, user, DEFAULT_QUEUE,
        activeUsersManager, this.rmContext);

  if (transferStateFromPreviousAttempt) {
    schedulerApp.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(schedulerApp);

  metrics.submitAppAttempt(user);
  LOG.info("Added Application Attempt " + appAttemptId
      + " to scheduler from user " + application.getUser());
  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(appAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(appAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 
Example #18
Source File: TestRMContainerImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testExpireWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify EXPIRE and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.EXPIRED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.EXPIRE));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  verify(writer, never()).containerFinished(any(RMContainer.class));
  verify(publisher, never()).containerFinished(any(RMContainer.class),
      anyLong());
}
 
Example #19
Source File: RMAppAttemptContainerFinishedEvent.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public RMAppAttemptContainerFinishedEvent(ApplicationAttemptId appAttemptId, 
    ContainerStatus containerStatus, NodeId nodeId) {
  super(appAttemptId, RMAppAttemptEventType.CONTAINER_FINISHED);
  this.containerStatus = containerStatus;
  this.nodeId = nodeId;
}
 
Example #20
Source File: FairScheduler.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Add a new application attempt to the scheduler.
 */
protected synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt,
    boolean isAttemptRecovering) {
  SchedulerApplication<FSAppAttempt> application =
      applications.get(applicationAttemptId.getApplicationId());
  String user = application.getUser();
  FSLeafQueue queue = (FSLeafQueue) application.getQueue();

  FSAppAttempt attempt =
      new FSAppAttempt(this, applicationAttemptId, user,
          queue, new ActiveUsersManager(getRootQueueMetrics()),
          rmContext);
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(application
        .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  boolean runnable = maxRunningEnforcer.canAppBeRunnable(queue, user);
  queue.addApp(attempt, runnable);
  if (runnable) {
    maxRunningEnforcer.trackRunnableApp(attempt);
  } else {
    maxRunningEnforcer.trackNonRunnableApp(attempt);
  }
  
  queue.getMetrics().submitAppAttempt(user);

  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user: " + user);

  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(applicationAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 
Example #21
Source File: RMAppAttemptContainerAllocatedEvent.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public RMAppAttemptContainerAllocatedEvent(ApplicationAttemptId appAttemptId) {
  super(appAttemptId, RMAppAttemptEventType.CONTAINER_ALLOCATED);
}
 
Example #22
Source File: TestRMContainerImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testReleaseWhileRunning() {

  DrainDispatcher drainDispatcher = new DrainDispatcher();
  EventHandler<RMAppAttemptEvent> appAttemptEventHandler = mock(EventHandler.class);
  EventHandler generic = mock(EventHandler.class);
  drainDispatcher.register(RMAppAttemptEventType.class,
      appAttemptEventHandler);
  drainDispatcher.register(RMNodeEventType.class, generic);
  drainDispatcher.init(new YarnConfiguration());
  drainDispatcher.start();
  NodeId nodeId = BuilderUtils.newNodeId("host", 3425);
  ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
  ContainerAllocationExpirer expirer = mock(ContainerAllocationExpirer.class);

  Resource resource = BuilderUtils.newResource(512, 1);
  Priority priority = BuilderUtils.newPriority(5);

  Container container = BuilderUtils.newContainer(containerId, nodeId,
      "host:3465", resource, priority, null);
  ConcurrentMap<ApplicationId, RMApp> rmApps =
      spy(new ConcurrentHashMap<ApplicationId, RMApp>());
  RMApp rmApp = mock(RMApp.class);
  when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
  Mockito.doReturn(rmApp).when(rmApps).get((ApplicationId)Matchers.any());

  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
  RMContext rmContext = mock(RMContext.class);
  when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
  when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
  when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
  when(rmContext.getRMApps()).thenReturn(rmApps);
  when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
  when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
  RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
      nodeId, "user", rmContext);

  assertEquals(RMContainerState.NEW, rmContainer.getState());
  assertEquals(resource, rmContainer.getAllocatedResource());
  assertEquals(nodeId, rmContainer.getAllocatedNode());
  assertEquals(priority, rmContainer.getAllocatedPriority());
  verify(writer).containerStarted(any(RMContainer.class));
  verify(publisher).containerCreated(any(RMContainer.class), anyLong());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.START));
  drainDispatcher.await();
  assertEquals(RMContainerState.ALLOCATED, rmContainer.getState());
  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.ACQUIRED));
  drainDispatcher.await();
  assertEquals(RMContainerState.ACQUIRED, rmContainer.getState());

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RUNNING, rmContainer.getState());
  assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
      rmContainer.getLogURL());

  // In RUNNING state. Verify RELEASED and associated actions.
  reset(appAttemptEventHandler);
  ContainerStatus containerStatus = SchedulerUtils
      .createAbnormalContainerStatus(containerId,
          SchedulerUtils.RELEASED_CONTAINER);
  rmContainer.handle(new RMContainerFinishedEvent(containerId,
      containerStatus, RMContainerEventType.RELEASED));
  drainDispatcher.await();
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
  assertEquals(SchedulerUtils.RELEASED_CONTAINER,
      rmContainer.getDiagnosticsInfo());
  assertEquals(ContainerExitStatus.ABORTED,
      rmContainer.getContainerExitStatus());
  assertEquals(ContainerState.COMPLETE, rmContainer.getContainerState());
  verify(writer).containerFinished(any(RMContainer.class));
  verify(publisher).containerFinished(any(RMContainer.class), anyLong());

  ArgumentCaptor<RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor
      .forClass(RMAppAttemptContainerFinishedEvent.class);
  verify(appAttemptEventHandler).handle(captor.capture());
  RMAppAttemptContainerFinishedEvent cfEvent = captor.getValue();
  assertEquals(appAttemptId, cfEvent.getApplicationAttemptId());
  assertEquals(containerStatus, cfEvent.getContainerStatus());
  assertEquals(RMAppAttemptEventType.CONTAINER_FINISHED, cfEvent.getType());
  
  // In RELEASED state. A FINIHSED event may come in.
  rmContainer.handle(new RMContainerFinishedEvent(containerId, SchedulerUtils
      .createAbnormalContainerStatus(containerId, "FinishedContainer"),
      RMContainerEventType.FINISHED));
  assertEquals(RMContainerState.RELEASED, rmContainer.getState());
}
 
Example #23
Source File: RMAppAttemptLaunchFailedEvent.java    From big-c with Apache License 2.0 4 votes vote down vote up
public RMAppAttemptLaunchFailedEvent(ApplicationAttemptId appAttemptId,
    String message) {
  super(appAttemptId, RMAppAttemptEventType.LAUNCH_FAILED);
  this.message = message;
}
 
Example #24
Source File: TestRM.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Validate killing an application when it is at accepted state.
 * @throws Exception exception
 */
@Test (timeout = 60000)
public void testApplicationKillAtAcceptedState() throws Exception {

  final Dispatcher dispatcher = new AsyncDispatcher() {
    @Override
    public EventHandler getEventHandler() {

      class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
        @Override
        public boolean matches(Object argument) {
          if (argument instanceof RMAppAttemptEvent) {
            if (((RMAppAttemptEvent) argument).getType().equals(
              RMAppAttemptEventType.KILL)) {
              return true;
            }
          }
          return false;
        }
      }

      EventHandler handler = spy(super.getEventHandler());
      doNothing().when(handler).handle(argThat(new EventArgMatcher()));
      return handler;
    }
  };

  MockRM rm = new MockRM(conf) {
    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };

  // test metrics
  QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
  int appsKilled = metrics.getAppsKilled();
  int appsSubmitted = metrics.getAppsSubmitted();

  rm.start();
  
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm1.registerNode();

  // a failed app
  RMApp application = rm.submitApp(200);
  MockAM am = MockRM.launchAM(application, rm, nm1);
  am.waitForState(RMAppAttemptState.LAUNCHED);
  nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.RUNNING);
  rm.waitForState(application.getApplicationId(), RMAppState.ACCEPTED);

  // Now kill the application before new attempt is launched, the app report
  // returns the invalid AM host and port.
  KillApplicationRequest request =
      KillApplicationRequest.newInstance(application.getApplicationId());
  rm.getClientRMService().forceKillApplication(request);

  // Specific test for YARN-1689 follows
  // Now let's say a race causes AM to register now. This should not crash RM.
  am.registerAppAttempt(false);

  // We explicitly intercepted the kill-event to RMAppAttempt, so app should
  // still be in KILLING state.
  rm.waitForState(application.getApplicationId(), RMAppState.KILLING);
  // AM should now be in running
  rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.RUNNING);

  // Simulate that appAttempt is killed.
  rm.getRMContext().getDispatcher().getEventHandler().handle(
      new RMAppEvent(application.getApplicationId(),
        RMAppEventType.ATTEMPT_KILLED));
  rm.waitForState(application.getApplicationId(), RMAppState.KILLED);

  // test metrics
  metrics = rm.getResourceScheduler().getRootQueueMetrics();
  Assert.assertEquals(appsKilled + 1, metrics.getAppsKilled());
  Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted());
}
 
Example #25
Source File: TestRM.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test (timeout = 30000)
public void testKillFinishingApp() throws Exception{

  // this dispatcher ignores RMAppAttemptEventType.KILL event
  final Dispatcher dispatcher = new AsyncDispatcher() {
    @Override
    public EventHandler getEventHandler() {

      class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
        @Override
        public boolean matches(Object argument) {
          if (argument instanceof RMAppAttemptEvent) {
            if (((RMAppAttemptEvent) argument).getType().equals(
              RMAppAttemptEventType.KILL)) {
              return true;
            }
          }
          return false;
        }
      }

      EventHandler handler = spy(super.getEventHandler());
      doNothing().when(handler).handle(argThat(new EventArgMatcher()));
      return handler;
    }
  };

  MockRM rm1 = new MockRM(conf){
    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  rm1.killApp(app1.getApplicationId());

  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  am1.unregisterAppAttempt(req,true);

  rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHING);
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
  rm1.waitForState(app1.getApplicationId(), RMAppState.FINISHED);
}
 
Example #26
Source File: TestRM.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test (timeout = 30000)
public void testKillFailingApp() throws Exception{

  // this dispatcher ignores RMAppAttemptEventType.KILL event
  final Dispatcher dispatcher = new AsyncDispatcher() {
    @Override
    public EventHandler getEventHandler() {

      class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
        @Override
        public boolean matches(Object argument) {
          if (argument instanceof RMAppAttemptEvent) {
            if (((RMAppAttemptEvent) argument).getType().equals(
              RMAppAttemptEventType.KILL)) {
              return true;
            }
          }
          return false;
        }
      }

      EventHandler handler = spy(super.getEventHandler());
      doNothing().when(handler).handle(argThat(new EventArgMatcher()));
      return handler;
    }
  };

  MockRM rm1 = new MockRM(conf){
    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  rm1.killApp(app1.getApplicationId());

  // fail the app by sending container_finished event.
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
  // app is killed, not launching a new attempt
  rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
}
 
Example #27
Source File: RMAppImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public void transition(RMAppImpl app, RMAppEvent event) {
  app.stateBeforeKilling = app.getState();
  app.handler.handle(new RMAppAttemptEvent(app.currentAttempt
    .getAppAttemptId(), RMAppAttemptEventType.KILL));
}
 
Example #28
Source File: TestRMAppTransitions.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE;
  if (isSecurityEnabled) {
    authMethod = AuthenticationMethod.KERBEROS;
  }
  SecurityUtil.setAuthenticationMethod(authMethod, conf);
  UserGroupInformation.setConfiguration(conf);

  rmDispatcher = new DrainDispatcher();
  ContainerAllocationExpirer containerAllocationExpirer = 
      mock(ContainerAllocationExpirer.class);
  AMLivelinessMonitor amLivelinessMonitor = mock(AMLivelinessMonitor.class);
  AMLivelinessMonitor amFinishingMonitor = mock(AMLivelinessMonitor.class);
  store = mock(RMStateStore.class);
  writer = mock(RMApplicationHistoryWriter.class);
  DelegationTokenRenewer renewer = mock(DelegationTokenRenewer.class);
  RMContext realRMContext = 
      new RMContextImpl(rmDispatcher,
        containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
        renewer, new AMRMTokenSecretManager(conf, this.rmContext),
        new RMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInRM(conf),
        new ClientToAMTokenSecretManagerInRM(),
        writer);
  ((RMContextImpl)realRMContext).setStateStore(store);
  publisher = mock(SystemMetricsPublisher.class);
  ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher);

  this.rmContext = spy(realRMContext);

  ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
  doReturn(null).when(resourceScheduler)
            .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
  doReturn(resourceScheduler).when(rmContext).getScheduler();

  rmDispatcher.register(RMAppAttemptEventType.class,
      new TestApplicationAttemptEventDispatcher(this.rmContext));

  rmDispatcher.register(RMAppEventType.class,
      new TestApplicationEventDispatcher(rmContext));
  
  rmDispatcher.register(RMAppManagerEventType.class,
      new TestApplicationManagerEventDispatcher());
  
  schedulerDispatcher = new TestSchedulerEventDispatcher();
  rmDispatcher.register(SchedulerEventType.class,
      schedulerDispatcher);
  
  rmDispatcher.init(conf);
  rmDispatcher.start();
}
 
Example #29
Source File: TestRMAppTransitions.java    From big-c with Apache License 2.0 4 votes vote down vote up
private void sendAttemptUpdateSavedEvent(RMApp application) {
  application.getCurrentAppAttempt().handle(
      new RMAppAttemptEvent(application.getCurrentAppAttempt().getAppAttemptId(),
          RMAppAttemptEventType.ATTEMPT_UPDATE_SAVED));
}
 
Example #30
Source File: RMAppImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private void recoverAppAttempts() {
  for (RMAppAttempt attempt : getAppAttempts().values()) {
    attempt.handle(new RMAppAttemptEvent(attempt.getAppAttemptId(),
      RMAppAttemptEventType.RECOVER));
  }
}