Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationAttemptId#getApplicationId()

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationAttemptId#getApplicationId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   File: TestFairScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyQueueName() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // only default queue
  assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());

  // submit app with empty queue
  ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "", "user1");
  scheduler.handle(appAddedEvent);

  // submission rejected
  assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
  assertNull(scheduler.getSchedulerApp(appAttemptId));
  assertEquals(0, resourceManager.getRMContext().getRMApps().size());
}
 
Example 2
Source Project: big-c   File: FiCaSchedulerNode.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void unreserveResource(
    SchedulerApplicationAttempt application) {

  // adding NP checks as this can now be called for preemption
  if (getReservedContainer() != null
      && getReservedContainer().getContainer() != null
      && getReservedContainer().getContainer().getId() != null
      && getReservedContainer().getContainer().getId()
        .getApplicationAttemptId() != null) {

    // Cannot unreserve for wrong application...
    ApplicationAttemptId reservedApplication =
        getReservedContainer().getContainer().getId()
          .getApplicationAttemptId();
    if (!reservedApplication.equals(
        application.getApplicationAttemptId())) {
      throw new IllegalStateException("Trying to unreserve " +
          " for application " + application.getApplicationAttemptId() +
          " when currently reserved " +
          " for application " + reservedApplication.getApplicationId() +
          " on node " + this);
    }
  }
  setReservedContainer(null);
}
 
Example 3
Source Project: hadoop   File: AbstractYarnScheduler.java    License: Apache License 2.0 6 votes vote down vote up
public synchronized List<Container> getTransferredContainers(
    ApplicationAttemptId currentAttempt) {
  ApplicationId appId = currentAttempt.getApplicationId();
  SchedulerApplication<T> app = applications.get(appId);
  List<Container> containerList = new ArrayList<Container>();
  RMApp appImpl = this.rmContext.getRMApps().get(appId);
  if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) {
    return containerList;
  }
  Collection<RMContainer> liveContainers =
      app.getCurrentAppAttempt().getLiveContainers();
  ContainerId amContainerId =
      rmContext.getRMApps().get(appId).getCurrentAppAttempt()
        .getMasterContainer().getId();
  for (RMContainer rmContainer : liveContainers) {
    if (!rmContainer.getContainerId().equals(amContainerId)) {
      containerList.add(rmContainer.getContainer());
    }
  }
  return containerList;
}
 
Example 4
Source Project: hadoop   File: FiCaSchedulerNode.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void unreserveResource(
    SchedulerApplicationAttempt application) {

  // adding NP checks as this can now be called for preemption
  if (getReservedContainer() != null
      && getReservedContainer().getContainer() != null
      && getReservedContainer().getContainer().getId() != null
      && getReservedContainer().getContainer().getId()
        .getApplicationAttemptId() != null) {

    // Cannot unreserve for wrong application...
    ApplicationAttemptId reservedApplication =
        getReservedContainer().getContainer().getId()
          .getApplicationAttemptId();
    if (!reservedApplication.equals(
        application.getApplicationAttemptId())) {
      throw new IllegalStateException("Trying to unreserve " +
          " for application " + application.getApplicationAttemptId() +
          " when currently reserved " +
          " for application " + reservedApplication.getApplicationId() +
          " on node " + this);
    }
  }
  setReservedContainer(null);
}
 
Example 5
Source Project: hadoop   File: TestZKRMStateStore.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDuplicateRMAppDeletion() throws Exception {
  TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester();
  long submitTime = System.currentTimeMillis();
  long startTime = System.currentTimeMillis() + 1234;
  RMStateStore store = zkTester.getRMStateStore();
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);

  ApplicationAttemptId attemptIdRemoved = ConverterUtils
      .toApplicationAttemptId("appattempt_1352994193343_0002_000001");
  ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
  storeApp(store, appIdRemoved, submitTime, startTime);
  storeAttempt(store, attemptIdRemoved,
      "container_1352994193343_0002_01_000001", null, null, dispatcher);

  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appIdRemoved);
  ApplicationStateData appStateRemoved =
      ApplicationStateData.newInstance(
          submitTime, startTime, context, "user1");
  appStateRemoved.attempts.put(attemptIdRemoved, null);
  store.removeApplicationStateInternal(appStateRemoved);
  try {
    store.removeApplicationStateInternal(appStateRemoved);
  } catch (KeeperException.NoNodeException nne) {
    Assert.fail("NoNodeException should not happen.");
  }
  store.close();
}
 
Example 6
Source Project: big-c   File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAppAdditionAndRemoval() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
  ApplicationAttemptId attemptId =createAppAttemptId(1, 1);
  AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), "default",
    "user1");
  scheduler.handle(appAddedEvent);
  AppAttemptAddedSchedulerEvent attemptAddedEvent =
      new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false);
  scheduler.handle(attemptAddedEvent);

  // Scheduler should have two queues (the default and the one created for user1)
  assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());

  // That queue should have one app
  assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
      .getNumRunnableApps());

  AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent(
      createAppAttemptId(1, 1), RMAppAttemptState.FINISHED, false);

  // Now remove app
  scheduler.handle(appRemovedEvent1);

  // Queue should have no apps
  assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
      .getNumRunnableApps());
}
 
Example 7
Source Project: tez   File: HistoryEventProtoConverter.java    License: Apache License 2.0 5 votes vote down vote up
private HistoryEventProto.Builder makeBuilderForEvent(HistoryEvent event, long time,
    TezDAGID dagId, ApplicationId appId, ApplicationAttemptId appAttemptId, TezVertexID vertexId,
    TezTaskID taskId, TezTaskAttemptID taskAttemptId, String user) {
  HistoryEventProto.Builder builder = HistoryEventProto.newBuilder();
  builder.setEventType(event.getEventType().name());
  builder.setEventTime(time);
  if (taskAttemptId != null) {
    builder.setTaskAttemptId(taskAttemptId.toString());
    taskId = taskAttemptId.getTaskID();
  }
  if (taskId != null) {
    builder.setTaskId(taskId.toString());
    vertexId = taskId.getVertexID();
  }
  if (vertexId != null) {
    builder.setVertexId(vertexId.toString());
    dagId = vertexId.getDAGId();
  }
  if (dagId != null) {
    builder.setDagId(dagId.toString());
    if (appId == null) {
      appId = dagId.getApplicationId();
    }
  }
  if (appAttemptId != null) {
    builder.setAppAttemptId(appAttemptId.toString());
    if (appId == null) {
      appId = appAttemptId.getApplicationId();
    }
  }
  if (appId != null) {
    builder.setAppId(appId.toString());
  }
  if (user != null) {
    builder.setUser(user);
  }
  return builder;
}
 
Example 8
Source Project: big-c   File: TestZKRMStateStore.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDuplicateRMAppDeletion() throws Exception {
  TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester();
  long submitTime = System.currentTimeMillis();
  long startTime = System.currentTimeMillis() + 1234;
  RMStateStore store = zkTester.getRMStateStore();
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);

  ApplicationAttemptId attemptIdRemoved = ConverterUtils
      .toApplicationAttemptId("appattempt_1352994193343_0002_000001");
  ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
  storeApp(store, appIdRemoved, submitTime, startTime);
  storeAttempt(store, attemptIdRemoved,
      "container_1352994193343_0002_01_000001", null, null, dispatcher);

  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appIdRemoved);
  ApplicationStateData appStateRemoved =
      ApplicationStateData.newInstance(
          submitTime, startTime, context, "user1");
  appStateRemoved.attempts.put(attemptIdRemoved, null);
  store.removeApplicationStateInternal(appStateRemoved);
  try {
    store.removeApplicationStateInternal(appStateRemoved);
  } catch (KeeperException.NoNodeException nne) {
    Assert.fail("NoNodeException should not happen.");
  }
  store.close();
}
 
Example 9
Source Project: big-c   File: NMTokenSecretManagerInNM.java    License: Apache License 2.0 5 votes vote down vote up
public synchronized void recover()
    throws IOException {
  RecoveredNMTokensState state = stateStore.loadNMTokensState();
  MasterKey key = state.getCurrentMasterKey();
  if (key != null) {
    super.currentMasterKey =
        new MasterKeyData(key, createSecretKey(key.getBytes().array()));
  }

  key = state.getPreviousMasterKey();
  if (key != null) {
    previousMasterKey =
        new MasterKeyData(key, createSecretKey(key.getBytes().array()));
  }

  // restore the serial number from the current master key
  if (super.currentMasterKey != null) {
    super.serialNo = super.currentMasterKey.getMasterKey().getKeyId() + 1;
  }

  for (Map.Entry<ApplicationAttemptId, MasterKey> entry :
       state.getApplicationMasterKeys().entrySet()) {
    key = entry.getValue();
    oldMasterKeys.put(entry.getKey(),
        new MasterKeyData(key, createSecretKey(key.getBytes().array())));
  }

  // reconstruct app to app attempts map
  appToAppAttemptMap.clear();
  for (ApplicationAttemptId attempt : oldMasterKeys.keySet()) {
    ApplicationId app = attempt.getApplicationId();
    List<ApplicationAttemptId> attempts = appToAppAttemptMap.get(app);
    if (attempts == null) {
      attempts = new ArrayList<ApplicationAttemptId>();
      appToAppAttemptMap.put(app, attempts);
    }
    attempts.add(attempt);
  }
}
 
Example 10
Source Project: big-c   File: FairSchedulerTestBase.java    License: Apache License 2.0 5 votes vote down vote up
protected void createApplicationWithAMResource(ApplicationAttemptId attId,
    String queue, String user, Resource amResource) {
  RMContext rmContext = resourceManager.getRMContext();
  RMApp rmApp = new RMAppImpl(attId.getApplicationId(), rmContext, conf,
      null, null, null, ApplicationSubmissionContext.newInstance(null, null,
      null, null, null, false, false, 0, amResource, null), null, null,
      0, null, null, null);
  rmContext.getRMApps().put(attId.getApplicationId(), rmApp);
  AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
      attId.getApplicationId(), queue, user);
  scheduler.handle(appAddedEvent);
  AppAttemptAddedSchedulerEvent attempAddedEvent =
      new AppAttemptAddedSchedulerEvent(attId, false);
  scheduler.handle(attempAddedEvent);
}
 
Example 11
Source Project: hadoop   File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testQueueuNameWithPeriods() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // only default queue
  assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());

  // submit app with queue name (.A)
  ApplicationAttemptId appAttemptId1 = createAppAttemptId(1, 1);
  AppAddedSchedulerEvent appAddedEvent1 =
      new AppAddedSchedulerEvent(appAttemptId1.getApplicationId(), ".A", "user1");
  scheduler.handle(appAddedEvent1);
  // submission rejected
  assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
  assertNull(scheduler.getSchedulerApp(appAttemptId1));
  assertEquals(0, resourceManager.getRMContext().getRMApps().size());

  // submit app with queue name (A.)
  ApplicationAttemptId appAttemptId2 = createAppAttemptId(2, 1);
  AppAddedSchedulerEvent appAddedEvent2 =
      new AppAddedSchedulerEvent(appAttemptId2.getApplicationId(), "A.", "user1");
  scheduler.handle(appAddedEvent2);
  // submission rejected
  assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
  assertNull(scheduler.getSchedulerApp(appAttemptId2));
  assertEquals(0, resourceManager.getRMContext().getRMApps().size());

  // submit app with queue name (A.B)
  ApplicationAttemptId appAttemptId3 = createAppAttemptId(3, 1);
  AppAddedSchedulerEvent appAddedEvent3 =
      new AppAddedSchedulerEvent(appAttemptId3.getApplicationId(), "A.B", "user1");
  scheduler.handle(appAddedEvent3);
  // submission accepted
  assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());
  assertNull(scheduler.getSchedulerApp(appAttemptId3));
  assertEquals(0, resourceManager.getRMContext().getRMApps().size());
}
 
Example 12
Source Project: hadoop   File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAppAdditionAndRemoval() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
  ApplicationAttemptId attemptId =createAppAttemptId(1, 1);
  AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attemptId.getApplicationId(), "default",
    "user1");
  scheduler.handle(appAddedEvent);
  AppAttemptAddedSchedulerEvent attemptAddedEvent =
      new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false);
  scheduler.handle(attemptAddedEvent);

  // Scheduler should have two queues (the default and the one created for user1)
  assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());

  // That queue should have one app
  assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
      .getNumRunnableApps());

  AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent(
      createAppAttemptId(1, 1), RMAppAttemptState.FINISHED, false);

  // Now remove app
  scheduler.handle(appRemovedEvent1);

  // Queue should have no apps
  assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
      .getNumRunnableApps());
}
 
Example 13
Source Project: big-c   File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMoveRunnableApp() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  QueueManager queueMgr = scheduler.getQueueManager();
  FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
  FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);

  ApplicationAttemptId appAttId =
      createSchedulingRequest(1024, 1, "queue1", "user1", 3);
  ApplicationId appId = appAttId.getApplicationId();
  RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(1024));
  NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  scheduler.handle(nodeEvent);
  scheduler.handle(updateEvent);
  
  assertEquals(Resource.newInstance(1024, 1), oldQueue.getResourceUsage());
  scheduler.update();
  assertEquals(Resource.newInstance(3072, 3), oldQueue.getDemand());
  
  scheduler.moveApplication(appId, "queue2");
  FSAppAttempt app = scheduler.getSchedulerApp(appAttId);
  assertSame(targetQueue, app.getQueue());
  assertFalse(oldQueue.isRunnableApp(app));
  assertTrue(targetQueue.isRunnableApp(app));
  assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage());
  assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage());
  assertEquals(0, oldQueue.getNumRunnableApps());
  assertEquals(1, targetQueue.getNumRunnableApps());
  assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps());
  
  scheduler.update();
  assertEquals(Resource.newInstance(0, 0), oldQueue.getDemand());
  assertEquals(Resource.newInstance(3072, 3), targetQueue.getDemand());
}
 
Example 14
Source Project: hadoop   File: RMAppAttemptImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void transition(RMAppAttemptImpl appAttempt,
    RMAppAttemptEvent event) {
  ApplicationAttemptId appAttemptId = appAttempt.getAppAttemptId();

  // Tell the AMS. Unregister from the ApplicationMasterService
  appAttempt.masterService.unregisterAttempt(appAttemptId);

  // Tell the application and the scheduler
  ApplicationId applicationId = appAttemptId.getApplicationId();
  RMAppEvent appEvent = null;
  boolean keepContainersAcrossAppAttempts = false;
  switch (finalAttemptState) {
    case FINISHED:
    {
      appEvent = new RMAppFinishedAttemptEvent(applicationId,
          appAttempt.getDiagnostics());
    }
    break;
    case KILLED:
    {
      appAttempt.invalidateAMHostAndPort();
      appEvent =
          new RMAppFailedAttemptEvent(applicationId,
              RMAppEventType.ATTEMPT_KILLED,
              "Application killed by user.", false);
    }
    break;
    case FAILED:
    {
      appAttempt.invalidateAMHostAndPort();

      if (appAttempt.submissionContext
        .getKeepContainersAcrossApplicationAttempts()
          && !appAttempt.submissionContext.getUnmanagedAM()) {
        // See if we should retain containers for non-unmanaged applications
        if (!appAttempt.shouldCountTowardsMaxAttemptRetry()) {
          // Premption, hardware failures, NM resync doesn't count towards
          // app-failures and so we should retain containers.
          keepContainersAcrossAppAttempts = true;
        } else if (!appAttempt.maybeLastAttempt) {
          // Not preemption, hardware failures or NM resync.
          // Not last-attempt too - keep containers.
          keepContainersAcrossAppAttempts = true;
        }
      }
      appEvent =
          new RMAppFailedAttemptEvent(applicationId,
            RMAppEventType.ATTEMPT_FAILED, appAttempt.getDiagnostics(),
            keepContainersAcrossAppAttempts);

    }
    break;
    default:
    {
      LOG.error("Cannot get this state!! Error!!");
    }
    break;
  }

  appAttempt.eventHandler.handle(appEvent);
  appAttempt.eventHandler.handle(new AppAttemptRemovedSchedulerEvent(
    appAttemptId, finalAttemptState, keepContainersAcrossAppAttempts));
  appAttempt.removeCredentials(appAttempt);

  appAttempt.rmContext.getRMApplicationHistoryWriter()
      .applicationAttemptFinished(appAttempt, finalAttemptState);
  appAttempt.rmContext.getSystemMetricsPublisher()
      .appAttemptFinished(appAttempt, finalAttemptState,
          appAttempt.rmContext.getRMApps().get(
              appAttempt.applicationAttemptId.getApplicationId()),
          System.currentTimeMillis());
}
 
Example 15
Source Project: big-c   File: TestLogAggregationService.java    License: Apache License 2.0 4 votes vote down vote up
private void verifyLocalFileDeletion(
    LogAggregationService logAggregationService) throws Exception {
  logAggregationService.init(this.conf);
  logAggregationService.start();

  ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);

  // AppLogDir should be created
  File app1LogDir =
      new File(localLogDir, ConverterUtils.toString(application1));
  app1LogDir.mkdir();
  logAggregationService
      .handle(new LogHandlerAppStartedEvent(
          application1, this.user, null,
          ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls));

  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(application1, 1);
  ContainerId container11 = BuilderUtils.newContainerId(appAttemptId, 1);
  // Simulate log-file creation
  writeContainerLogs(app1LogDir, container11, new String[] { "stdout",
      "stderr", "syslog" });
  logAggregationService.handle(
      new LogHandlerContainerFinishedEvent(container11, 0));

  logAggregationService.handle(new LogHandlerAppFinishedEvent(
      application1));

  logAggregationService.stop();
  assertEquals(0, logAggregationService.getNumAggregators());
  // ensure filesystems were closed
  verify(logAggregationService).closeFileSystems(
      any(UserGroupInformation.class));
  verify(delSrvc).delete(eq(user), eq((Path) null),
    eq(new Path(app1LogDir.getAbsolutePath())));
  delSrvc.stop();
  
  String containerIdStr = ConverterUtils.toString(container11);
  File containerLogDir = new File(app1LogDir, containerIdStr);
  for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
    File f = new File(containerLogDir, fileType);
    Assert.assertFalse("check "+f, f.exists());
  }

  Assert.assertFalse(app1LogDir.exists());

  Path logFilePath =
      logAggregationService.getRemoteNodeLogFileForApp(application1,
          this.user);

  Assert.assertTrue("Log file [" + logFilePath + "] not found", new File(
      logFilePath.toUri().getPath()).exists());
  
  dispatcher.await();
  
  ApplicationEvent expectedEvents[] = new ApplicationEvent[]{
      new ApplicationEvent(
          appAttemptId.getApplicationId(),
          ApplicationEventType.APPLICATION_LOG_HANDLING_INITED),
      new ApplicationEvent(
          appAttemptId.getApplicationId(),
          ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED)
  };

  checkEvents(appEventHandler, expectedEvents, true, "getType",
      "getApplicationID");
}
 
Example 16
Source Project: big-c   File: RMAppAttemptImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void transition(RMAppAttemptImpl appAttempt,
    RMAppAttemptEvent event) {
  ApplicationAttemptId appAttemptId = appAttempt.getAppAttemptId();

  // Tell the AMS. Unregister from the ApplicationMasterService
  appAttempt.masterService.unregisterAttempt(appAttemptId);

  // Tell the application and the scheduler
  ApplicationId applicationId = appAttemptId.getApplicationId();
  RMAppEvent appEvent = null;
  boolean keepContainersAcrossAppAttempts = false;
  switch (finalAttemptState) {
    case FINISHED:
    {
      appEvent = new RMAppFinishedAttemptEvent(applicationId,
          appAttempt.getDiagnostics());
    }
    break;
    case KILLED:
    {
      appAttempt.invalidateAMHostAndPort();
      appEvent =
          new RMAppFailedAttemptEvent(applicationId,
              RMAppEventType.ATTEMPT_KILLED,
              "Application killed by user.", false);
    }
    break;
    case FAILED:
    {
      appAttempt.invalidateAMHostAndPort();

      if (appAttempt.submissionContext
        .getKeepContainersAcrossApplicationAttempts()
          && !appAttempt.submissionContext.getUnmanagedAM()) {
        // See if we should retain containers for non-unmanaged applications
        if (!appAttempt.shouldCountTowardsMaxAttemptRetry()) {
          // Premption, hardware failures, NM resync doesn't count towards
          // app-failures and so we should retain containers.
          keepContainersAcrossAppAttempts = true;
        } else if (!appAttempt.maybeLastAttempt) {
          // Not preemption, hardware failures or NM resync.
          // Not last-attempt too - keep containers.
          keepContainersAcrossAppAttempts = true;
        }
      }
      appEvent =
          new RMAppFailedAttemptEvent(applicationId,
            RMAppEventType.ATTEMPT_FAILED, appAttempt.getDiagnostics(),
            keepContainersAcrossAppAttempts);

    }
    break;
    default:
    {
      LOG.error("Cannot get this state!! Error!!");
    }
    break;
  }

  appAttempt.eventHandler.handle(appEvent);
  appAttempt.eventHandler.handle(new AppAttemptRemovedSchedulerEvent(
    appAttemptId, finalAttemptState, keepContainersAcrossAppAttempts));
  appAttempt.removeCredentials(appAttempt);

  appAttempt.rmContext.getRMApplicationHistoryWriter()
      .applicationAttemptFinished(appAttempt, finalAttemptState);
  appAttempt.rmContext.getSystemMetricsPublisher()
      .appAttemptFinished(appAttempt, finalAttemptState,
          appAttempt.rmContext.getRMApps().get(
              appAttempt.applicationAttemptId.getApplicationId()),
          System.currentTimeMillis());
}
 
Example 17
Source Project: big-c   File: ApplicationMasterService.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {

  ApplicationAttemptId applicationAttemptId =
      authorizeRequest().getApplicationAttemptId();
  ApplicationId appId = applicationAttemptId.getApplicationId();

  RMApp rmApp =
      rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
  // checking whether the app exits in RMStateStore at first not to throw
  // ApplicationDoesNotExistInCacheException before and after
  // RM work-preserving restart.
  if (rmApp.isAppFinalStateStored()) {
    LOG.info(rmApp.getApplicationId() + " unregistered successfully. ");
    return FinishApplicationMasterResponse.newInstance(true);
  }

  AllocateResponseLock lock = responseMap.get(applicationAttemptId);
  if (lock == null) {
    throwApplicationDoesNotExistInCacheException(applicationAttemptId);
  }

  // Allow only one thread in AM to do finishApp at a time.
  synchronized (lock) {
    if (!hasApplicationMasterRegistered(applicationAttemptId)) {
      String message =
          "Application Master is trying to unregister before registering for: "
              + appId;
      LOG.error(message);
      RMAuditLogger.logFailure(
          this.rmContext.getRMApps()
              .get(appId).getUser(),
          AuditConstants.UNREGISTER_AM, "", "ApplicationMasterService",
          message, appId,
          applicationAttemptId);
      throw new ApplicationMasterNotRegisteredException(message);
    }

    this.amLivelinessMonitor.receivedPing(applicationAttemptId);

    rmContext.getDispatcher().getEventHandler().handle(
        new RMAppAttemptUnregistrationEvent(applicationAttemptId, request
            .getTrackingUrl(), request.getFinalApplicationStatus(), request
            .getDiagnostics()));

    // For UnmanagedAMs, return true so they don't retry
    return FinishApplicationMasterResponse.newInstance(
        rmApp.getApplicationSubmissionContext().getUnmanagedAM());
  }
}
 
Example 18
Source Project: big-c   File: ClientRMService.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public GetContainerReportResponse getContainerReport(
    GetContainerReportRequest request) throws YarnException, IOException {
  ContainerId containerId = request.getContainerId();
  ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
  ApplicationId appId = appAttemptId.getApplicationId();
  UserGroupInformation callerUGI;
  try {
    callerUGI = UserGroupInformation.getCurrentUser();
  } catch (IOException ie) {
    LOG.info("Error getting UGI ", ie);
    throw RPCUtil.getRemoteException(ie);
  }
  RMApp application = this.rmContext.getRMApps().get(appId);
  if (application == null) {
    // If the RM doesn't have the application, throw
    // ApplicationNotFoundException and let client to handle.
    throw new ApplicationNotFoundException("Application with id '" + appId
        + "' doesn't exist in RM.");
  }
  boolean allowAccess = checkAccess(callerUGI, application.getUser(),
      ApplicationAccessType.VIEW_APP, application);
  GetContainerReportResponse response = null;
  if (allowAccess) {
    RMAppAttempt appAttempt = application.getAppAttempts().get(appAttemptId);
    if (appAttempt == null) {
      throw new ApplicationAttemptNotFoundException(
          "ApplicationAttempt with id '" + appAttemptId +
          "' doesn't exist in RM.");
    }
    RMContainer rmConatiner = this.rmContext.getScheduler().getRMContainer(
        containerId);
    if (rmConatiner == null) {
      throw new ContainerNotFoundException("Container with id '" + containerId
          + "' doesn't exist in RM.");
    }
    response = GetContainerReportResponse.newInstance(rmConatiner
        .createContainerReport());
  } else {
    throw new YarnException("User " + callerUGI.getShortUserName()
        + " does not have privilage to see this aplication " + appId);
  }
  return response;
}
 
Example 19
Source Project: hadoop   File: TestFifoScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=2000)
public void testNodeLocalAssignment() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  Configuration conf = new Configuration();
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.rollMasterKey();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  
  FifoScheduler scheduler = new FifoScheduler();
  RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
      null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
      scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(new Configuration(), rmContext);

  RMNode node0 = MockNodes.newNodeInfo(1,
      Resources.createResource(1024 * 64), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
  scheduler.handle(nodeEvent1);

  int _appId = 1;
  int _appAttemptId = 1;
  ApplicationAttemptId appAttemptId = createAppAttemptId(_appId,
      _appAttemptId);

  createMockRMApp(appAttemptId, rmContext);

  AppAddedSchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1",
          "user1");
  scheduler.handle(appEvent);
  AppAttemptAddedSchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  int memory = 64;
  int nConts = 3;
  int priority = 20;

  List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
  ResourceRequest nodeLocal = createResourceRequest(memory,
      node0.getHostName(), priority, nConts);
  ResourceRequest rackLocal = createResourceRequest(memory,
      node0.getRackName(), priority, nConts);
  ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
      nConts);
  ask.add(nodeLocal);
  ask.add(rackLocal);
  ask.add(any);
  scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);

  NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);

  // Before the node update event, there are 3 local requests outstanding
  Assert.assertEquals(3, nodeLocal.getNumContainers());

  scheduler.handle(node0Update);

  // After the node update event, check that there are no more local requests
  // outstanding
  Assert.assertEquals(0, nodeLocal.getNumContainers());
  //Also check that the containers were scheduled
  SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
  Assert.assertEquals(3, info.getLiveContainers().size());
  scheduler.stop();
}
 
Example 20
Source Project: hadoop   File: TestFifoScheduler.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=2000)
public void testUpdateResourceOnNode() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  Configuration conf = new Configuration();
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.rollMasterKey();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  
  FifoScheduler scheduler = new FifoScheduler(){
    @SuppressWarnings("unused")
    public Map<NodeId, FiCaSchedulerNode> getNodes(){
      return nodes;
    }
  };
  RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
      null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
      scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(new Configuration(), rmContext);
  RMNode node0 = MockNodes.newNodeInfo(1,
      Resources.createResource(2048, 4, 4), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
  scheduler.handle(nodeEvent1);
  
  Method method = scheduler.getClass().getDeclaredMethod("getNodes");
  @SuppressWarnings("unchecked")
  Map<NodeId, FiCaSchedulerNode> schedulerNodes = 
      (Map<NodeId, FiCaSchedulerNode>) method.invoke(scheduler);
  assertEquals(schedulerNodes.values().size(), 1);
  
  Resource newResource = Resources.createResource(1024, 4, 4);
  
  NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new 
      NodeResourceUpdateSchedulerEvent(node0, ResourceOption.newInstance(
          newResource, RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT));
  scheduler.handle(node0ResourceUpdate);
  
  // SchedulerNode's total resource and available resource are changed.
  assertEquals(schedulerNodes.get(node0.getNodeID()).getTotalResource()
      .getMemory(), 1024);
  assertEquals(schedulerNodes.get(node0.getNodeID()).
      getAvailableResource().getMemory(), 1024);
  QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
  Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
  
  int _appId = 1;
  int _appAttemptId = 1;
  ApplicationAttemptId appAttemptId = createAppAttemptId(_appId,
      _appAttemptId);
  createMockRMApp(appAttemptId, rmContext);

  AppAddedSchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1",
        "user1");
  scheduler.handle(appEvent);
  AppAttemptAddedSchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  int memory = 1024;
  int priority = 1;

  List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
  ResourceRequest nodeLocal = createResourceRequest(memory,
      node0.getHostName(), priority, 1);
  ResourceRequest rackLocal = createResourceRequest(memory,
      node0.getRackName(), priority, 1);
  ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
      1);
  ask.add(nodeLocal);
  ask.add(rackLocal);
  ask.add(any);
  scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);

  // Before the node update event, there are one local request
  Assert.assertEquals(1, nodeLocal.getNumContainers());

  NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
  // Now schedule.
  scheduler.handle(node0Update);

  // After the node update event, check no local request
  Assert.assertEquals(0, nodeLocal.getNumContainers());
  // Also check that one container was scheduled
  SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
  Assert.assertEquals(1, info.getLiveContainers().size());
  // And check the default Queue now is full.
  queueInfo = scheduler.getQueueInfo(null, false, false);
  Assert.assertEquals(1.0f, queueInfo.getCurrentCapacity(), 0.0f);
}