Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: RMWebServices.java    License: Apache License 2.0 6 votes vote down vote up
private RMApp getRMAppForAppId(String appId) {

    if (appId == null || appId.isEmpty()) {
      throw new NotFoundException("appId, " + appId + ", is empty or null");
    }
    ApplicationId id;
    try {
      id = ConverterUtils.toApplicationId(recordFactory, appId);
    } catch (NumberFormatException e) {
      throw new NotFoundException("appId is invalid");
    }
    if (id == null) {
      throw new NotFoundException("appId is invalid");
    }
    RMApp app = rm.getRMContext().getRMApps().get(id);
    if (app == null) {
      throw new NotFoundException("app with id: " + appId + " not found");
    }
    return app;
  }
 
Example 2
Source Project: big-c   Source File: AbstractYarnScheduler.java    License: Apache License 2.0 6 votes vote down vote up
public synchronized List<Container> getTransferredContainers(
    ApplicationAttemptId currentAttempt) {
  ApplicationId appId = currentAttempt.getApplicationId();
  SchedulerApplication<T> app = applications.get(appId);
  List<Container> containerList = new ArrayList<Container>();
  RMApp appImpl = this.rmContext.getRMApps().get(appId);
  if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) {
    return containerList;
  }
  Collection<RMContainer> liveContainers =
      app.getCurrentAppAttempt().getLiveContainers();
  ContainerId amContainerId =
      rmContext.getRMApps().get(appId).getCurrentAppAttempt()
        .getMasterContainer().getId();
  for (RMContainer rmContainer : liveContainers) {
    if (!rmContainer.getContainerId().equals(amContainerId)) {
      containerList.add(rmContainer.getContainer());
    }
  }
  return containerList;
}
 
Example 3
Source Project: big-c   Source File: TestRMWebServicesApps.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAppsQueryFinalStatus() throws JSONException, Exception {
  rm.start();
  MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
  RMApp app1 = rm.submitApp(CONTAINER_MB);
  amNodeManager.nodeHeartbeat(true);
  WebResource r = resource();

  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString())
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  System.out.println(json.toString());
  JSONObject apps = json.getJSONObject("apps");
  assertEquals("incorrect number of elements", 1, apps.length());
  JSONArray array = apps.getJSONArray("app");
  assertEquals("incorrect number of elements", 1, array.length());
  verifyAppInfo(array.getJSONObject(0), app1);
  rm.stop();
}
 
Example 4
Source Project: big-c   Source File: TestFairScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAssignToQueue() throws Exception {
  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
  RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);
  
  FSLeafQueue queue1 = scheduler.assignToQueue(rmApp1, "default", "asterix");
  FSLeafQueue queue2 = scheduler.assignToQueue(rmApp2, "notdefault", "obelix");
  
  // assert FSLeafQueue's name is the correct name is the one set in the RMApp
  assertEquals(rmApp1.getQueue(), queue1.getName());
  assertEquals("root.asterix", rmApp1.getQueue());
  assertEquals(rmApp2.getQueue(), queue2.getName());
  assertEquals("root.notdefault", rmApp2.getQueue());
}
 
Example 5
Source Project: hadoop   Source File: ResourceManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void handle(RMAppAttemptEvent event) {
  ApplicationAttemptId appAttemptID = event.getApplicationAttemptId();
  ApplicationId appAttemptId = appAttemptID.getApplicationId();
  RMApp rmApp = this.rmContext.getRMApps().get(appAttemptId);
  if (rmApp != null) {
    RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptID);
    if (rmAppAttempt != null) {
      try {
        rmAppAttempt.handle(event);
      } catch (Throwable t) {
        LOG.error("Error in handling event type " + event.getType()
            + " for applicationAttempt " + appAttemptId, t);
      }
    }
  }
}
 
Example 6
Source Project: big-c   Source File: SystemMetricsPublisher.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public void appAttemptFinished(RMAppAttempt appAttempt,
    RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) {
  if (publishSystemMetrics) {
    dispatcher.getEventHandler().handle(
        new AppAttemptFinishedEvent(
            appAttempt.getAppAttemptId(),
            appAttempt.getTrackingUrl(),
            appAttempt.getOriginalTrackingUrl(),
            appAttempt.getDiagnostics(),
            // app will get the final status from app attempt, or create one
            // based on app state if it doesn't exist
            app.getFinalApplicationStatus(),
            RMServerUtils.createApplicationAttemptState(appAttemtpState),
            finishedTime));
  }
}
 
Example 7
Source Project: hadoop   Source File: AbstractYarnScheduler.java    License: Apache License 2.0 6 votes vote down vote up
public synchronized List<Container> getTransferredContainers(
    ApplicationAttemptId currentAttempt) {
  ApplicationId appId = currentAttempt.getApplicationId();
  SchedulerApplication<T> app = applications.get(appId);
  List<Container> containerList = new ArrayList<Container>();
  RMApp appImpl = this.rmContext.getRMApps().get(appId);
  if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) {
    return containerList;
  }
  Collection<RMContainer> liveContainers =
      app.getCurrentAppAttempt().getLiveContainers();
  ContainerId amContainerId =
      rmContext.getRMApps().get(appId).getCurrentAppAttempt()
        .getMasterContainer().getId();
  for (RMContainer rmContainer : liveContainers) {
    if (!rmContainer.getContainerId().equals(amContainerId)) {
      containerList.add(rmContainer.getContainer());
    }
  }
  return containerList;
}
 
Example 8
Source Project: big-c   Source File: RMStateStoreTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected RMApp storeApp(RMStateStore store, ApplicationId appId,
    long submitTime,
    long startTime) throws Exception {
  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appId);

  RMApp mockApp = mock(RMApp.class);
  when(mockApp.getApplicationId()).thenReturn(appId);
  when(mockApp.getSubmitTime()).thenReturn(submitTime);
  when(mockApp.getStartTime()).thenReturn(startTime);
  when(mockApp.getApplicationSubmissionContext()).thenReturn(context);
  when(mockApp.getUser()).thenReturn("test");
  store.storeNewApplication(mockApp);
  return mockApp;
}
 
Example 9
Source Project: hadoop   Source File: TestKillApplicationWithRMHA.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 20000)
public void testKillAppWhenFailoverHappensAtRunningState()
    throws Exception {
  startRMs();
  MockNM nm1 = new MockNM("127.0.0.1:1234", 15120,
      rm1.getResourceTrackerService());
  nm1.registerNode();

  // create app and launch the AM
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = launchAM(app0, rm1, nm1);

  // failover and kill application
  // The application is at RUNNING State when failOver happens.
  // Since RMStateStore has already saved ApplicationState, the active RM
  // will load the ApplicationState. After that, the application will be at
  // ACCEPTED State. Because the application is not at Final State,
  // KillApplicationResponse.getIsKillCompleted is expected to return false.
  failOverAndKillApp(app0.getApplicationId(),
      am0.getApplicationAttemptId(), RMAppState.RUNNING,
      RMAppAttemptState.RUNNING, RMAppState.ACCEPTED);
}
 
Example 10
Source Project: hadoop   Source File: TestMoveApplication.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 10000)
    public
    void testMoveSuccessful() throws Exception {
  MockRM rm1 = new MockRM(conf);
  rm1.start();
  RMApp app = rm1.submitApp(1024);
  ClientRMService clientRMService = rm1.getClientRMService();
  // FIFO scheduler does not support moves
  clientRMService
    .moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest
      .newInstance(app.getApplicationId(), "newqueue"));

  RMApp rmApp = rm1.getRMContext().getRMApps().get(app.getApplicationId());
  assertEquals("newqueue", rmApp.getQueue());
  rm1.stop();
}
 
Example 11
Source Project: hadoop   Source File: TestSystemMetricsPublisher.java    License: Apache License 2.0 6 votes vote down vote up
private static RMApp createRMApp(ApplicationId appId) {
  RMApp app = mock(RMApp.class);
  when(app.getApplicationId()).thenReturn(appId);
  when(app.getName()).thenReturn("test app");
  when(app.getApplicationType()).thenReturn("test app type");
  when(app.getUser()).thenReturn("test user");
  when(app.getQueue()).thenReturn("test queue");
  when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
  when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
  when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
  when(app.getDiagnostics()).thenReturn(
      new StringBuilder("test diagnostics info"));
  RMAppAttempt appAttempt = mock(RMAppAttempt.class);
  when(appAttempt.getAppAttemptId()).thenReturn(
      ApplicationAttemptId.newInstance(appId, 1));
  when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
  when(app.getFinalApplicationStatus()).thenReturn(
      FinalApplicationStatus.UNDEFINED);
  when(app.getRMAppMetrics()).thenReturn(
      new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE));
  return app;
}
 
Example 12
Source Project: hadoop   Source File: TestAppManager.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 30000)
public void testRMAppSubmitDuplicateApplicationId() throws Exception {
  ApplicationId appId = MockApps.newAppID(0);
  asContext.setApplicationId(appId);
  RMApp appOrig = rmContext.getRMApps().get(appId);
  Assert.assertTrue("app name matches but shouldn't", "testApp1" != appOrig.getName());

  // our testApp1 should be rejected and original app with same id should be left in place
  try {
    appMonitor.submitApplication(asContext, "test");
    Assert.fail("Exception is expected when applicationId is duplicate.");
  } catch (YarnException e) {
    Assert.assertTrue("The thrown exception is not the expectd one.",
        e.getMessage().contains("Cannot add a duplicate!"));
  }

  // make sure original app didn't get removed
  RMApp app = rmContext.getRMApps().get(appId);
  Assert.assertNotNull("app is null", app);
  Assert.assertEquals("app id doesn't match", appId, app.getApplicationId());
  Assert.assertEquals("app state doesn't match", RMAppState.FINISHED, app.getState());
}
 
Example 13
Source Project: hadoop   Source File: RMStateStoreTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected RMApp storeApp(RMStateStore store, ApplicationId appId,
    long submitTime,
    long startTime) throws Exception {
  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appId);

  RMApp mockApp = mock(RMApp.class);
  when(mockApp.getApplicationId()).thenReturn(appId);
  when(mockApp.getSubmitTime()).thenReturn(submitTime);
  when(mockApp.getStartTime()).thenReturn(startTime);
  when(mockApp.getApplicationSubmissionContext()).thenReturn(context);
  when(mockApp.getUser()).thenReturn("test");
  store.storeNewApplication(mockApp);
  return mockApp;
}
 
Example 14
Source Project: hadoop   Source File: TestRMWebServicesApps.java    License: Apache License 2.0 6 votes vote down vote up
public void testAppsHelper(String path, RMApp app, String media)
    throws JSONException, Exception {
  WebResource r = resource();

  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path(path).accept(media).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject apps = json.getJSONObject("apps");
  assertEquals("incorrect number of elements", 1, apps.length());
  JSONArray array = apps.getJSONArray("app");
  assertEquals("incorrect number of elements", 1, array.length());
  verifyAppInfo(array.getJSONObject(0), app);

}
 
Example 15
Source Project: big-c   Source File: TestRMRestart.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 60000)
public void testAppSubmissionWithOldDelegationTokenAfterRMRestart()
    throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032");
  UserGroupInformation.setConfiguration(conf);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  MockRM rm1 = new TestSecurityMockRM(conf, memStore);
  rm1.start();

  GetDelegationTokenRequest request1 =
      GetDelegationTokenRequest.newInstance("renewer1");
  UserGroupInformation.getCurrentUser().setAuthenticationMethod(
      AuthMethod.KERBEROS);
  GetDelegationTokenResponse response1 =
      rm1.getClientRMService().getDelegationToken(request1);
  Token<RMDelegationTokenIdentifier> token1 =
      ConverterUtils.convertFromYarn(response1.getRMDelegationToken(), rmAddr);

  // start new RM
  MockRM rm2 = new TestSecurityMockRM(conf, memStore);
  rm2.start();

  // submit an app with the old delegation token got from previous RM.
  Credentials ts = new Credentials();
  ts.addToken(token1.getService(), token1);
  RMApp app = rm2.submitApp(200, "name", "user",
      new HashMap<ApplicationAccessType, String>(), false, "default", 1, ts);
  rm2.waitForState(app.getApplicationId(), RMAppState.ACCEPTED);
}
 
Example 16
Source Project: hadoop   Source File: ResourceTrackerService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Helper method to handle received ContainerStatus. If this corresponds to
 * the completion of a master-container of a managed AM,
 * we call the handler for RMAppAttemptContainerFinishedEvent.
 */
@SuppressWarnings("unchecked")
@VisibleForTesting
void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) {
  ApplicationAttemptId appAttemptId =
      containerStatus.getContainerId().getApplicationAttemptId();
  RMApp rmApp =
      rmContext.getRMApps().get(appAttemptId.getApplicationId());
  if (rmApp == null) {
    LOG.error("Received finished container : "
        + containerStatus.getContainerId()
        + " for unknown application " + appAttemptId.getApplicationId()
        + " Skipping.");
    return;
  }

  if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Ignoring container completion status for unmanaged AM "
          + rmApp.getApplicationId());
    }
    return;
  }

  RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId);
  Container masterContainer = rmAppAttempt.getMasterContainer();
  if (masterContainer.getId().equals(containerStatus.getContainerId())
      && containerStatus.getContainerState() == ContainerState.COMPLETE) {
    ContainerStatus status =
        ContainerStatus.newInstance(containerStatus.getContainerId(),
          containerStatus.getContainerState(), containerStatus.getDiagnostics(),
          containerStatus.getContainerExitStatus());
    // sending master container finished event.
    RMAppAttemptContainerFinishedEvent evt =
        new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
            nodeId);
    rmContext.getDispatcher().getEventHandler().handle(evt);
  }
}
 
Example 17
Source Project: hadoop   Source File: RMWebServices.java    License: Apache License 2.0 5 votes vote down vote up
protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) {
  // Check for the authorization.
  UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
  if (callerUGI != null
      && !(this.rm.getApplicationACLsManager().checkAccess(callerUGI,
            ApplicationAccessType.VIEW_APP, app.getUser(),
            app.getApplicationId()) ||
          this.rm.getQueueACLsManager().checkAccess(callerUGI,
            QueueACL.ADMINISTER_QUEUE, app.getQueue()))) {
    return false;
  }
  return true;
}
 
Example 18
Source Project: hadoop   Source File: TestContainerAllocation.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testContainerTokenGeneratedOnPullRequest() throws Exception {
  MockRM rm1 = new MockRM(conf);
  rm1.start();
  MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  // request a container.
  am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
  ContainerId containerId2 =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);

  RMContainer container =
      rm1.getResourceScheduler().getRMContainer(containerId2);
  // no container token is generated.
  Assert.assertEquals(containerId2, container.getContainerId());
  Assert.assertNull(container.getContainer().getContainerToken());

  // acquire the container.
  List<Container> containers =
      am1.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers();
  Assert.assertEquals(containerId2, containers.get(0).getId());
  // container token is generated.
  Assert.assertNotNull(containers.get(0).getContainerToken());
  rm1.stop();
}
 
Example 19
Source Project: big-c   Source File: MockRM.java    License: Apache License 2.0 5 votes vote down vote up
public RMApp submitApp(int masterMemory, String name, String user,
    Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
    int maxAppAttempts, Credentials ts, String appType,
    boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
    ApplicationId applicationId) throws Exception {
  return submitApp(masterMemory, name, user, acls, unmanaged, queue,
    maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
    isAppIdProvided, applicationId, 0, null, true);
}
 
Example 20
Source Project: hadoop   Source File: RMAppAttemptBlock.java    License: Apache License 2.0 5 votes vote down vote up
private RMAppAttempt getRMAppAttempt() {
  ApplicationId appId = this.appAttemptId.getApplicationId();
  RMAppAttempt attempt = null;
  RMApp rmApp = rm.getRMContext().getRMApps().get(appId);
  if (rmApp != null) { 
    attempt = rmApp.getAppAttempts().get(appAttemptId);
  }
  return attempt;
}
 
Example 21
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 5 votes vote down vote up
private List<ApplicationAttemptId> getSchedulerApps(
    Map<ApplicationId, RMApp> apps) {
  List<ApplicationAttemptId> schedApps = new ArrayList<ApplicationAttemptId>();
  // Return app IDs for the apps in testqueue (as defined in getRMApps)
  schedApps.add(ApplicationAttemptId.newInstance(getApplicationId(1), 0));
  schedApps.add(ApplicationAttemptId.newInstance(getApplicationId(3), 0));
  return schedApps;
}
 
Example 22
Source Project: hadoop   Source File: MockRM.java    License: Apache License 2.0 5 votes vote down vote up
public RMApp submitApp(int masterMemory, String name, String user,
    Map<ApplicationAccessType, String> acls, String queue, 
    boolean waitForAccepted) throws Exception {
  return submitApp(masterMemory, name, user, acls, false, queue,
    super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null,
      waitForAccepted);
}
 
Example 23
Source Project: big-c   Source File: TestContainerAllocation.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testContainerTokenGeneratedOnPullRequest() throws Exception {
  MockRM rm1 = new MockRM(conf);
  rm1.start();
  MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  // request a container.
  am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
  ContainerId containerId2 =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);

  RMContainer container =
      rm1.getResourceScheduler().getRMContainer(containerId2);
  // no container token is generated.
  Assert.assertEquals(containerId2, container.getContainerId());
  Assert.assertNull(container.getContainer().getContainerToken());

  // acquire the container.
  List<Container> containers =
      am1.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers();
  Assert.assertEquals(containerId2, containers.get(0).getId());
  // container token is generated.
  Assert.assertNotNull(containers.get(0).getContainerToken());
  rm1.stop();
}
 
Example 24
Source Project: big-c   Source File: TestRMWebServicesApps.java    License: Apache License 2.0 5 votes vote down vote up
public void testSingleAppsHelper(String path, RMApp app, String media)
    throws JSONException, Exception {
  WebResource r = resource();
  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("apps").path(path).accept(media).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);

  assertEquals("incorrect number of elements", 1, json.length());
  verifyAppInfo(json.getJSONObject("app"), app);
}
 
Example 25
Source Project: big-c   Source File: TestRMWebServicesApps.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAppsQueryFinishEnd() throws JSONException, Exception {
  rm.start();
  MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
  RMApp app1 = rm.submitApp(CONTAINER_MB);
  amNodeManager.nodeHeartbeat(true);
  // finish App
  MockAM am = rm
      .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId());
  am.registerAppAttempt();
  am.unregisterAppAttempt();
  amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
      1, ContainerState.COMPLETE);

  rm.submitApp(CONTAINER_MB);
  rm.submitApp(CONTAINER_MB);
  long end = System.currentTimeMillis();

  WebResource r = resource();
  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("apps").queryParam("finishedTimeEnd", String.valueOf(end))
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject apps = json.getJSONObject("apps");
  assertEquals("incorrect number of elements", 1, apps.length());
  JSONArray array = apps.getJSONArray("app");
  assertEquals("incorrect number of elements", 3, array.length());
  rm.stop();
}
 
Example 26
Source Project: hadoop   Source File: TestRMWebServicesApps.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAppAttemtpsDefault() throws JSONException, Exception {
  rm.start();
  MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
  RMApp app1 = rm.submitApp(CONTAINER_MB);
  amNodeManager.nodeHeartbeat(true);
  testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, "");
  rm.stop();
}
 
Example 27
Source Project: hadoop   Source File: FairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Helper method that attempts to assign the app to a queue. The method is
 * responsible to call the appropriate event-handler if the app is rejected.
 */
@VisibleForTesting
FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) {
  FSLeafQueue queue = null;
  String appRejectMsg = null;

  try {
    QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy();
    queueName = placementPolicy.assignAppToQueue(queueName, user);
    if (queueName == null) {
      appRejectMsg = "Application rejected by queue placement policy";
    } else {
      queue = queueMgr.getLeafQueue(queueName, true);
      if (queue == null) {
        appRejectMsg = queueName + " is not a leaf queue";
      }
    }
  } catch (IOException ioe) {
    appRejectMsg = "Error assigning app to queue " + queueName;
  }

  if (appRejectMsg != null && rmApp != null) {
    LOG.error(appRejectMsg);
    rmContext.getDispatcher().getEventHandler().handle(
        new RMAppRejectedEvent(rmApp.getApplicationId(), appRejectMsg));
    return null;
  }

  if (rmApp != null) {
    rmApp.setQueue(queue.getName());
  } else {
    LOG.error("Couldn't find RM app to set queue name on");
  }
  return queue;
}
 
Example 28
Source Project: hadoop   Source File: FiCaSchedulerApp.java    License: Apache License 2.0 5 votes vote down vote up
public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId, 
    String user, Queue queue, ActiveUsersManager activeUsersManager,
    RMContext rmContext) {
  super(applicationAttemptId, user, queue, activeUsersManager, rmContext);
  
  RMApp rmApp = rmContext.getRMApps().get(getApplicationId());
  
  Resource amResource;
  String partition;

  if (rmApp == null || rmApp.getAMResourceRequest() == null) {
    // the rmApp may be undefined (the resource manager checks for this too)
    // and unmanaged applications do not provide an amResource request
    // in these cases, provide a default using the scheduler
    amResource = rmContext.getScheduler().getMinimumResourceCapability();
    partition = CommonNodeLabelsManager.NO_LABEL;
  } else {
    amResource = rmApp.getAMResourceRequest().getCapability();
    partition =
        (rmApp.getAMResourceRequest().getNodeLabelExpression() == null)
        ? CommonNodeLabelsManager.NO_LABEL
        : rmApp.getAMResourceRequest().getNodeLabelExpression();
  }
  
  setAppAMNodePartitionName(partition);
  setAMResource(partition, amResource);
}
 
Example 29
Source Project: hadoop   Source File: RMAppManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * create a summary of the application's runtime.
 * 
 * @param app {@link RMApp} whose summary is to be created, cannot
 *            be <code>null</code>.
 */
public static SummaryBuilder createAppSummary(RMApp app) {
  String trackingUrl = "N/A";
  String host = "N/A";
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  if (attempt != null) {
    trackingUrl = attempt.getTrackingUrl();
    host = attempt.getHost();
  }
  RMAppMetrics metrics = app.getRMAppMetrics();
  SummaryBuilder summary = new SummaryBuilder()
      .add("appId", app.getApplicationId())
      .add("name", app.getName())
      .add("user", app.getUser())
      .add("queue", app.getQueue())
      .add("state", app.getState())
      .add("trackingUrl", trackingUrl)
      .add("appMasterHost", host)
      .add("startTime", app.getStartTime())
      .add("finishTime", app.getFinishTime())
      .add("finalStatus", app.getFinalApplicationStatus())
      .add("memorySeconds", metrics.getMemorySeconds())
      .add("vcoreSeconds", metrics.getVcoreSeconds())
      .add("gcoreSeconds", metrics.getGcoreSeconds())
      .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
      .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
      .add("preemptedResources", metrics.getResourcePreempted())
      .add("applicationType", app.getApplicationType());
  return summary;
}
 
Example 30
Source Project: big-c   Source File: TestRMRestart.java    License: Apache License 2.0 5 votes vote down vote up
private void finishApplicationMaster(RMApp rmApp, MockRM rm, MockNM nm,
    MockAM am) throws Exception {
  final FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  finishApplicationMaster(rmApp, rm, nm, am, req);
}