org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestMoveApplication.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 10000)
    public
    void testMoveSuccessful() throws Exception {
  MockRM rm1 = new MockRM(conf);
  rm1.start();
  RMApp app = rm1.submitApp(1024);
  ClientRMService clientRMService = rm1.getClientRMService();
  // FIFO scheduler does not support moves
  clientRMService
    .moveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest
      .newInstance(app.getApplicationId(), "newqueue"));

  RMApp rmApp = rm1.getRMContext().getRMApps().get(app.getApplicationId());
  assertEquals("newqueue", rmApp.getQueue());
  rm1.stop();
}
 
Example #2
Source File: RMStateStoreTestBase.java    From hadoop with Apache License 2.0 6 votes vote down vote up
protected RMApp storeApp(RMStateStore store, ApplicationId appId,
    long submitTime,
    long startTime) throws Exception {
  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appId);

  RMApp mockApp = mock(RMApp.class);
  when(mockApp.getApplicationId()).thenReturn(appId);
  when(mockApp.getSubmitTime()).thenReturn(submitTime);
  when(mockApp.getStartTime()).thenReturn(startTime);
  when(mockApp.getApplicationSubmissionContext()).thenReturn(context);
  when(mockApp.getUser()).thenReturn("test");
  store.storeNewApplication(mockApp);
  return mockApp;
}
 
Example #3
Source File: RMWebServices.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private RMApp getRMAppForAppId(String appId) {

    if (appId == null || appId.isEmpty()) {
      throw new NotFoundException("appId, " + appId + ", is empty or null");
    }
    ApplicationId id;
    try {
      id = ConverterUtils.toApplicationId(recordFactory, appId);
    } catch (NumberFormatException e) {
      throw new NotFoundException("appId is invalid");
    }
    if (id == null) {
      throw new NotFoundException("appId is invalid");
    }
    RMApp app = rm.getRMContext().getRMApps().get(id);
    if (app == null) {
      throw new NotFoundException("app with id: " + appId + " not found");
    }
    return app;
  }
 
Example #4
Source File: TestRMWebServicesApps.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void testAppsHelper(String path, RMApp app, String media)
    throws JSONException, Exception {
  WebResource r = resource();

  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path(path).accept(media).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject apps = json.getJSONObject("apps");
  assertEquals("incorrect number of elements", 1, apps.length());
  JSONArray array = apps.getJSONArray("app");
  assertEquals("incorrect number of elements", 1, array.length());
  verifyAppInfo(array.getJSONObject(0), app);

}
 
Example #5
Source File: AbstractYarnScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
public synchronized List<Container> getTransferredContainers(
    ApplicationAttemptId currentAttempt) {
  ApplicationId appId = currentAttempt.getApplicationId();
  SchedulerApplication<T> app = applications.get(appId);
  List<Container> containerList = new ArrayList<Container>();
  RMApp appImpl = this.rmContext.getRMApps().get(appId);
  if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) {
    return containerList;
  }
  Collection<RMContainer> liveContainers =
      app.getCurrentAppAttempt().getLiveContainers();
  ContainerId amContainerId =
      rmContext.getRMApps().get(appId).getCurrentAppAttempt()
        .getMasterContainer().getId();
  for (RMContainer rmContainer : liveContainers) {
    if (!rmContainer.getContainerId().equals(amContainerId)) {
      containerList.add(rmContainer.getContainer());
    }
  }
  return containerList;
}
 
Example #6
Source File: TestRMWebServicesApps.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testAppsQueryFinalStatus() throws JSONException, Exception {
  rm.start();
  MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
  RMApp app1 = rm.submitApp(CONTAINER_MB);
  amNodeManager.nodeHeartbeat(true);
  WebResource r = resource();

  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString())
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  System.out.println(json.toString());
  JSONObject apps = json.getJSONObject("apps");
  assertEquals("incorrect number of elements", 1, apps.length());
  JSONArray array = apps.getJSONArray("app");
  assertEquals("incorrect number of elements", 1, array.length());
  verifyAppInfo(array.getJSONObject(0), app1);
  rm.stop();
}
 
Example #7
Source File: TestFairScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testAssignToQueue() throws Exception {
  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
  RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);
  
  FSLeafQueue queue1 = scheduler.assignToQueue(rmApp1, "default", "asterix");
  FSLeafQueue queue2 = scheduler.assignToQueue(rmApp2, "notdefault", "obelix");
  
  // assert FSLeafQueue's name is the correct name is the one set in the RMApp
  assertEquals(rmApp1.getQueue(), queue1.getName());
  assertEquals("root.asterix", rmApp1.getQueue());
  assertEquals(rmApp2.getQueue(), queue2.getName());
  assertEquals("root.notdefault", rmApp2.getQueue());
}
 
Example #8
Source File: ResourceManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void handle(RMAppAttemptEvent event) {
  ApplicationAttemptId appAttemptID = event.getApplicationAttemptId();
  ApplicationId appAttemptId = appAttemptID.getApplicationId();
  RMApp rmApp = this.rmContext.getRMApps().get(appAttemptId);
  if (rmApp != null) {
    RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptID);
    if (rmAppAttempt != null) {
      try {
        rmAppAttempt.handle(event);
      } catch (Throwable t) {
        LOG.error("Error in handling event type " + event.getType()
            + " for applicationAttempt " + appAttemptId, t);
      }
    }
  }
}
 
Example #9
Source File: SystemMetricsPublisher.java    From big-c with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public void appAttemptFinished(RMAppAttempt appAttempt,
    RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) {
  if (publishSystemMetrics) {
    dispatcher.getEventHandler().handle(
        new AppAttemptFinishedEvent(
            appAttempt.getAppAttemptId(),
            appAttempt.getTrackingUrl(),
            appAttempt.getOriginalTrackingUrl(),
            appAttempt.getDiagnostics(),
            // app will get the final status from app attempt, or create one
            // based on app state if it doesn't exist
            app.getFinalApplicationStatus(),
            RMServerUtils.createApplicationAttemptState(appAttemtpState),
            finishedTime));
  }
}
 
Example #10
Source File: TestAppManager.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 30000)
public void testRMAppSubmitDuplicateApplicationId() throws Exception {
  ApplicationId appId = MockApps.newAppID(0);
  asContext.setApplicationId(appId);
  RMApp appOrig = rmContext.getRMApps().get(appId);
  Assert.assertTrue("app name matches but shouldn't", "testApp1" != appOrig.getName());

  // our testApp1 should be rejected and original app with same id should be left in place
  try {
    appMonitor.submitApplication(asContext, "test");
    Assert.fail("Exception is expected when applicationId is duplicate.");
  } catch (YarnException e) {
    Assert.assertTrue("The thrown exception is not the expectd one.",
        e.getMessage().contains("Cannot add a duplicate!"));
  }

  // make sure original app didn't get removed
  RMApp app = rmContext.getRMApps().get(appId);
  Assert.assertNotNull("app is null", app);
  Assert.assertEquals("app id doesn't match", appId, app.getApplicationId());
  Assert.assertEquals("app state doesn't match", RMAppState.FINISHED, app.getState());
}
 
Example #11
Source File: AbstractYarnScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public synchronized List<Container> getTransferredContainers(
    ApplicationAttemptId currentAttempt) {
  ApplicationId appId = currentAttempt.getApplicationId();
  SchedulerApplication<T> app = applications.get(appId);
  List<Container> containerList = new ArrayList<Container>();
  RMApp appImpl = this.rmContext.getRMApps().get(appId);
  if (appImpl.getApplicationSubmissionContext().getUnmanagedAM()) {
    return containerList;
  }
  Collection<RMContainer> liveContainers =
      app.getCurrentAppAttempt().getLiveContainers();
  ContainerId amContainerId =
      rmContext.getRMApps().get(appId).getCurrentAppAttempt()
        .getMasterContainer().getId();
  for (RMContainer rmContainer : liveContainers) {
    if (!rmContainer.getContainerId().equals(amContainerId)) {
      containerList.add(rmContainer.getContainer());
    }
  }
  return containerList;
}
 
Example #12
Source File: RMStateStoreTestBase.java    From big-c with Apache License 2.0 6 votes vote down vote up
protected RMApp storeApp(RMStateStore store, ApplicationId appId,
    long submitTime,
    long startTime) throws Exception {
  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appId);

  RMApp mockApp = mock(RMApp.class);
  when(mockApp.getApplicationId()).thenReturn(appId);
  when(mockApp.getSubmitTime()).thenReturn(submitTime);
  when(mockApp.getStartTime()).thenReturn(startTime);
  when(mockApp.getApplicationSubmissionContext()).thenReturn(context);
  when(mockApp.getUser()).thenReturn("test");
  store.storeNewApplication(mockApp);
  return mockApp;
}
 
Example #13
Source File: TestKillApplicationWithRMHA.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 20000)
public void testKillAppWhenFailoverHappensAtRunningState()
    throws Exception {
  startRMs();
  MockNM nm1 = new MockNM("127.0.0.1:1234", 15120,
      rm1.getResourceTrackerService());
  nm1.registerNode();

  // create app and launch the AM
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = launchAM(app0, rm1, nm1);

  // failover and kill application
  // The application is at RUNNING State when failOver happens.
  // Since RMStateStore has already saved ApplicationState, the active RM
  // will load the ApplicationState. After that, the application will be at
  // ACCEPTED State. Because the application is not at Final State,
  // KillApplicationResponse.getIsKillCompleted is expected to return false.
  failOverAndKillApp(app0.getApplicationId(),
      am0.getApplicationAttemptId(), RMAppState.RUNNING,
      RMAppAttemptState.RUNNING, RMAppState.ACCEPTED);
}
 
Example #14
Source File: TestSystemMetricsPublisher.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static RMApp createRMApp(ApplicationId appId) {
  RMApp app = mock(RMApp.class);
  when(app.getApplicationId()).thenReturn(appId);
  when(app.getName()).thenReturn("test app");
  when(app.getApplicationType()).thenReturn("test app type");
  when(app.getUser()).thenReturn("test user");
  when(app.getQueue()).thenReturn("test queue");
  when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
  when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
  when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
  when(app.getDiagnostics()).thenReturn(
      new StringBuilder("test diagnostics info"));
  RMAppAttempt appAttempt = mock(RMAppAttempt.class);
  when(appAttempt.getAppAttemptId()).thenReturn(
      ApplicationAttemptId.newInstance(appId, 1));
  when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
  when(app.getFinalApplicationStatus()).thenReturn(
      FinalApplicationStatus.UNDEFINED);
  when(app.getRMAppMetrics()).thenReturn(
      new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE));
  return app;
}
 
Example #15
Source File: FairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Helper method that attempts to assign the app to a queue. The method is
 * responsible to call the appropriate event-handler if the app is rejected.
 */
@VisibleForTesting
FSLeafQueue assignToQueue(RMApp rmApp, String queueName, String user) {
  FSLeafQueue queue = null;
  String appRejectMsg = null;

  try {
    QueuePlacementPolicy placementPolicy = allocConf.getPlacementPolicy();
    queueName = placementPolicy.assignAppToQueue(queueName, user);
    if (queueName == null) {
      appRejectMsg = "Application rejected by queue placement policy";
    } else {
      queue = queueMgr.getLeafQueue(queueName, true);
      if (queue == null) {
        appRejectMsg = queueName + " is not a leaf queue";
      }
    }
  } catch (IOException ioe) {
    appRejectMsg = "Error assigning app to queue " + queueName;
  }

  if (appRejectMsg != null && rmApp != null) {
    LOG.error(appRejectMsg);
    rmContext.getDispatcher().getEventHandler().handle(
        new RMAppRejectedEvent(rmApp.getApplicationId(), appRejectMsg));
    return null;
  }

  if (rmApp != null) {
    rmApp.setQueue(queue.getName());
  } else {
    LOG.error("Couldn't find RM app to set queue name on");
  }
  return queue;
}
 
Example #16
Source File: TestContainerAllocation.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 30000)
public void testAMContainerAllocationWhenDNSUnavailable() throws Exception {
  MockRM rm1 = new MockRM(conf) {
    @Override
    protected RMSecretManagerService createRMSecretManagerService() {
      return new TestRMSecretManagerService(conf, rmContext);
    }
  };
  rm1.start();

  MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
  SecurityUtilTestHelper.setTokenServiceUseIp(true);
  RMApp app1 = rm1.submitApp(200);
  RMAppAttempt attempt = app1.getCurrentAppAttempt();
  nm1.nodeHeartbeat(true);

  // fetching am container will fail, keep retrying 5 times.
  while (numRetries <= 5) {
    nm1.nodeHeartbeat(true);
    Thread.sleep(1000);
    Assert.assertEquals(RMAppAttemptState.SCHEDULED,
      attempt.getAppAttemptState());
    System.out.println("Waiting for am container to be allocated.");
  }

  SecurityUtilTestHelper.setTokenServiceUseIp(false);
  rm1.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.ALLOCATED);
  MockRM.launchAndRegisterAM(app1, rm1, nm1);
}
 
Example #17
Source File: TestRMRestart.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 60000)
public void testAppSubmissionWithOldDelegationTokenAfterRMRestart()
    throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  conf.set(YarnConfiguration.RM_ADDRESS, "localhost:8032");
  UserGroupInformation.setConfiguration(conf);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  MockRM rm1 = new TestSecurityMockRM(conf, memStore);
  rm1.start();

  GetDelegationTokenRequest request1 =
      GetDelegationTokenRequest.newInstance("renewer1");
  UserGroupInformation.getCurrentUser().setAuthenticationMethod(
      AuthMethod.KERBEROS);
  GetDelegationTokenResponse response1 =
      rm1.getClientRMService().getDelegationToken(request1);
  Token<RMDelegationTokenIdentifier> token1 =
      ConverterUtils.convertFromYarn(response1.getRMDelegationToken(), rmAddr);

  // start new RM
  MockRM rm2 = new TestSecurityMockRM(conf, memStore);
  rm2.start();

  // submit an app with the old delegation token got from previous RM.
  Credentials ts = new Credentials();
  ts.addToken(token1.getService(), token1);
  RMApp app = rm2.submitApp(200, "name", "user",
      new HashMap<ApplicationAccessType, String>(), false, "default", 1, ts);
  rm2.waitForState(app.getApplicationId(), RMAppState.ACCEPTED);
}
 
Example #18
Source File: RMWebServices.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) {
  // Check for the authorization.
  UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
  if (callerUGI != null
      && !(this.rm.getApplicationACLsManager().checkAccess(callerUGI,
            ApplicationAccessType.VIEW_APP, app.getUser(),
            app.getApplicationId()) ||
          this.rm.getQueueACLsManager().checkAccess(callerUGI,
            QueueACL.ADMINISTER_QUEUE, app.getQueue()))) {
    return false;
  }
  return true;
}
 
Example #19
Source File: ResourceTrackerService.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Helper method to handle received ContainerStatus. If this corresponds to
 * the completion of a master-container of a managed AM,
 * we call the handler for RMAppAttemptContainerFinishedEvent.
 */
@SuppressWarnings("unchecked")
@VisibleForTesting
void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) {
  ApplicationAttemptId appAttemptId =
      containerStatus.getContainerId().getApplicationAttemptId();
  RMApp rmApp =
      rmContext.getRMApps().get(appAttemptId.getApplicationId());
  if (rmApp == null) {
    LOG.error("Received finished container : "
        + containerStatus.getContainerId()
        + " for unknown application " + appAttemptId.getApplicationId()
        + " Skipping.");
    return;
  }

  if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Ignoring container completion status for unmanaged AM "
          + rmApp.getApplicationId());
    }
    return;
  }

  RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId);
  Container masterContainer = rmAppAttempt.getMasterContainer();
  if (masterContainer.getId().equals(containerStatus.getContainerId())
      && containerStatus.getContainerState() == ContainerState.COMPLETE) {
    ContainerStatus status =
        ContainerStatus.newInstance(containerStatus.getContainerId(),
          containerStatus.getContainerState(), containerStatus.getDiagnostics(),
          containerStatus.getContainerExitStatus());
    // sending master container finished event.
    RMAppAttemptContainerFinishedEvent evt =
        new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
            nodeId);
    rmContext.getDispatcher().getEventHandler().handle(evt);
  }
}
 
Example #20
Source File: TestRMRestart.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void finishApplicationMaster(RMApp rmApp, MockRM rm, MockNM nm,
    MockAM am) throws Exception {
  final FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  finishApplicationMaster(rmApp, rm, nm, am, req);
}
 
Example #21
Source File: FiCaSchedulerApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId, 
    String user, Queue queue, ActiveUsersManager activeUsersManager,
    RMContext rmContext) {
  super(applicationAttemptId, user, queue, activeUsersManager, rmContext);
  
  RMApp rmApp = rmContext.getRMApps().get(getApplicationId());
  
  Resource amResource;
  String partition;

  if (rmApp == null || rmApp.getAMResourceRequest() == null) {
    // the rmApp may be undefined (the resource manager checks for this too)
    // and unmanaged applications do not provide an amResource request
    // in these cases, provide a default using the scheduler
    amResource = rmContext.getScheduler().getMinimumResourceCapability();
    partition = CommonNodeLabelsManager.NO_LABEL;
  } else {
    amResource = rmApp.getAMResourceRequest().getCapability();
    partition =
        (rmApp.getAMResourceRequest().getNodeLabelExpression() == null)
        ? CommonNodeLabelsManager.NO_LABEL
        : rmApp.getAMResourceRequest().getNodeLabelExpression();
  }
  
  setAppAMNodePartitionName(partition);
  setAMResource(partition, amResource);
}
 
Example #22
Source File: RMAppManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * create a summary of the application's runtime.
 * 
 * @param app {@link RMApp} whose summary is to be created, cannot
 *            be <code>null</code>.
 */
public static SummaryBuilder createAppSummary(RMApp app) {
  String trackingUrl = "N/A";
  String host = "N/A";
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  if (attempt != null) {
    trackingUrl = attempt.getTrackingUrl();
    host = attempt.getHost();
  }
  RMAppMetrics metrics = app.getRMAppMetrics();
  SummaryBuilder summary = new SummaryBuilder()
      .add("appId", app.getApplicationId())
      .add("name", app.getName())
      .add("user", app.getUser())
      .add("queue", app.getQueue())
      .add("state", app.getState())
      .add("trackingUrl", trackingUrl)
      .add("appMasterHost", host)
      .add("startTime", app.getStartTime())
      .add("finishTime", app.getFinishTime())
      .add("finalStatus", app.getFinalApplicationStatus())
      .add("memorySeconds", metrics.getMemorySeconds())
      .add("vcoreSeconds", metrics.getVcoreSeconds())
      .add("gcoreSeconds", metrics.getGcoreSeconds())
      .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
      .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
      .add("preemptedResources", metrics.getResourcePreempted())
      .add("applicationType", app.getApplicationType());
  return summary;
}
 
Example #23
Source File: TestRMWebServicesApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testAppAttemtpsDefault() throws JSONException, Exception {
  rm.start();
  MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
  RMApp app1 = rm.submitApp(CONTAINER_MB);
  amNodeManager.nodeHeartbeat(true);
  testAppAttemptsHelper(app1.getApplicationId().toString() + "/", app1, "");
  rm.stop();
}
 
Example #24
Source File: TestRMWebServicesApps.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testAppsQueryFinishEnd() throws JSONException, Exception {
  rm.start();
  MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
  RMApp app1 = rm.submitApp(CONTAINER_MB);
  amNodeManager.nodeHeartbeat(true);
  // finish App
  MockAM am = rm
      .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId());
  am.registerAppAttempt();
  am.unregisterAppAttempt();
  amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
      1, ContainerState.COMPLETE);

  rm.submitApp(CONTAINER_MB);
  rm.submitApp(CONTAINER_MB);
  long end = System.currentTimeMillis();

  WebResource r = resource();
  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("apps").queryParam("finishedTimeEnd", String.valueOf(end))
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  JSONObject apps = json.getJSONObject("apps");
  assertEquals("incorrect number of elements", 1, apps.length());
  JSONArray array = apps.getJSONArray("app");
  assertEquals("incorrect number of elements", 3, array.length());
  rm.stop();
}
 
Example #25
Source File: MockRM.java    From big-c with Apache License 2.0 5 votes vote down vote up
public RMApp submitApp(int masterMemory, String name, String user,
    Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
    int maxAppAttempts, Credentials ts, String appType,
    boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
    ApplicationId applicationId) throws Exception {
  return submitApp(masterMemory, name, user, acls, unmanaged, queue,
    maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
    isAppIdProvided, applicationId, 0, null, true);
}
 
Example #26
Source File: TestRMApplicationHistoryWriter.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteApplication() throws Exception {
  RMApp app = createRMApp(ApplicationId.newInstance(0, 1));

  writer.applicationStarted(app);
  ApplicationHistoryData appHD = null;
  for (int i = 0; i < MAX_RETRIES; ++i) {
    appHD = store.getApplication(ApplicationId.newInstance(0, 1));
    if (appHD != null) {
      break;
    } else {
      Thread.sleep(100);
    }
  }
  Assert.assertNotNull(appHD);
  Assert.assertEquals("test app", appHD.getApplicationName());
  Assert.assertEquals("test app type", appHD.getApplicationType());
  Assert.assertEquals("test user", appHD.getUser());
  Assert.assertEquals("test queue", appHD.getQueue());
  Assert.assertEquals(0L, appHD.getSubmitTime());
  Assert.assertEquals(1L, appHD.getStartTime());

  writer.applicationFinished(app, RMAppState.FINISHED);
  for (int i = 0; i < MAX_RETRIES; ++i) {
    appHD = store.getApplication(ApplicationId.newInstance(0, 1));
    if (appHD.getYarnApplicationState() != null) {
      break;
    } else {
      Thread.sleep(100);
    }
  }
  Assert.assertEquals(2L, appHD.getFinishTime());
  Assert.assertEquals("test diagnostics info", appHD.getDiagnosticsInfo());
  Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
    appHD.getFinalApplicationStatus());
  Assert.assertEquals(YarnApplicationState.FINISHED,
    appHD.getYarnApplicationState());
}
 
Example #27
Source File: TestRMWebServicesApps.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void testSingleAppsHelper(String path, RMApp app, String media)
    throws JSONException, Exception {
  WebResource r = resource();
  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("apps").path(path).accept(media).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);

  assertEquals("incorrect number of elements", 1, json.length());
  verifyAppInfo(json.getJSONObject("app"), app);
}
 
Example #28
Source File: TestRMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 60000)
public void testClientRetryOnKillingApplication() throws Exception {
  MemoryRMStateStore memStore = new TestMemoryRMStateStore();
  memStore.init(conf);

  // start RM
  MockRM rm1 = createMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  RMApp app1 =
      rm1.submitApp(200, "name", "user", null, false, "default", 1, null,
        "myType");
  MockAM am1 = launchAM(app1, rm1, nm1);

  KillApplicationResponse response;
  int count = 0;
  while (true) {
    response = rm1.killApp(app1.getApplicationId());
    if (response.getIsKillCompleted()) {
      break;
    }
    Thread.sleep(100);
    count++;
  }
  // we expect at least 2 calls for killApp as the first killApp always return
  // false.
  Assert.assertTrue(count >= 1);

  rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.KILLED);
  rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
  Assert.assertEquals(1, ((TestMemoryRMStateStore) memStore).updateAttempt);
  Assert.assertEquals(2, ((TestMemoryRMStateStore) memStore).updateApp);
}
 
Example #29
Source File: TestWorkPreservingRMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Testing to confirm that retried finishApplicationMaster() doesn't throw
 * InvalidApplicationMasterRequest before and after RM restart.
 */
@Test (timeout = 20000)
public void testRetriedFinishApplicationMasterRequest()
    throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  // start RM
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // create app and launch the AM
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = MockRM.launchAM(app0, rm1, nm1);

  am0.registerAppAttempt();

  // Emulating following a scenario:
  // RM1 saves the app in RMStateStore and then crashes,
  // FinishApplicationMasterResponse#isRegistered still return false,
  // so AM still retry the 2nd RM
  MockRM.finishAMAndVerifyAppState(app0, rm1, nm1, am0);


  // start new RM
  rm2 = new MockRM(conf, memStore);
  rm2.start();

  am0.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
  am0.unregisterAppAttempt(false);
}
 
Example #30
Source File: TestContainerAllocation.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testContainerTokenGeneratedOnPullRequest() throws Exception {
  MockRM rm1 = new MockRM(conf);
  rm1.start();
  MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  // request a container.
  am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
  ContainerId containerId2 =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);

  RMContainer container =
      rm1.getResourceScheduler().getRMContainer(containerId2);
  // no container token is generated.
  Assert.assertEquals(containerId2, container.getContainerId());
  Assert.assertNull(container.getContainer().getContainerToken());

  // acquire the container.
  List<Container> containers =
      am1.allocate(new ArrayList<ResourceRequest>(),
        new ArrayList<ContainerId>()).getAllocatedContainers();
  Assert.assertEquals(containerId2, containers.get(0).getId());
  // container token is generated.
  Assert.assertNotNull(containers.get(0).getContainerToken());
  rm1.stop();
}