Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp#getState()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp#getState() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestMoveApplication.java From hadoop with Apache License 2.0

6 votes

@Test
public void testMoveRejectedByScheduler() throws Exception {
  failMove = true;
  
  // Submit application
  Application application = new Application("user1", resourceManager);
  application.submit();

  // Wait for app to be accepted
  RMApp app = resourceManager.rmContext.getRMApps()
          .get(application.getApplicationId());
  while (app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(100);
  }

  ClientRMService clientRMService = resourceManager.getClientRMService();
  try {
    // FIFO scheduler does not support moves
    clientRMService.moveApplicationAcrossQueues(
        MoveApplicationAcrossQueuesRequest.newInstance(
            application.getApplicationId(), "newqueue"));
    fail("Should have hit exception");
  } catch (YarnException ex) {
    assertEquals("Move not supported", ex.getCause().getMessage());
  }
}

Example 2

Source File: TestMoveApplication.java From big-c with Apache License 2.0

6 votes

@Test
public void testMoveRejectedByScheduler() throws Exception {
  failMove = true;
  
  // Submit application
  Application application = new Application("user1", resourceManager);
  application.submit();

  // Wait for app to be accepted
  RMApp app = resourceManager.rmContext.getRMApps()
          .get(application.getApplicationId());
  while (app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(100);
  }

  ClientRMService clientRMService = resourceManager.getClientRMService();
  try {
    // FIFO scheduler does not support moves
    clientRMService.moveApplicationAcrossQueues(
        MoveApplicationAcrossQueuesRequest.newInstance(
            application.getApplicationId(), "newqueue"));
    fail("Should have hit exception");
  } catch (YarnException ex) {
    assertEquals("Move not supported", ex.getCause().getMessage());
  }
}

Example 3

Source File: RMAppManager.java From hadoop with Apache License 2.0

5 votes

protected void writeAuditLog(ApplicationId appId) {
  RMApp app = rmContext.getRMApps().get(appId);
  String operation = "UNKONWN";
  boolean success = false;
  switch (app.getState()) {
    case FAILED: 
      operation = AuditConstants.FINISH_FAILED_APP;
      break;
    case FINISHED:
      operation = AuditConstants.FINISH_SUCCESS_APP;
      success = true;
      break;
    case KILLED: 
      operation = AuditConstants.FINISH_KILLED_APP;
      success = true;
      break;
    default:
      break;
  }
  
  if (success) {
    RMAuditLogger.logSuccess(app.getUser(), operation,
        "RMAppManager", app.getApplicationId());
  } else {
    StringBuilder diag = app.getDiagnostics(); 
    String msg = diag == null ? null : diag.toString();
    RMAuditLogger.logFailure(app.getUser(), operation, msg, "RMAppManager",
        "App failed with state: " + app.getState(), appId);
  }
}

Example 4

Source File: TestAppManager.java From hadoop with Apache License 2.0

5 votes

protected void addToCompletedApps(TestRMAppManager appMonitor, RMContext rmContext) {
  for (RMApp app : rmContext.getRMApps().values()) {
    if (app.getState() == RMAppState.FINISHED
        || app.getState() == RMAppState.KILLED 
        || app.getState() == RMAppState.FAILED) {
      appMonitor.finishApplication(app.getApplicationId());
    }
  }
}

Example 5

Source File: TestRMRestart.java From hadoop with Apache License 2.0

5 votes

private void waitForTokensToBeRenewed(MockRM rm2) throws Exception {
  int waitCnt = 20;
  boolean atleastOneAppInNEWState = true;
  while (waitCnt-- > 0 && atleastOneAppInNEWState) {
    atleastOneAppInNEWState = false;
    for (RMApp rmApp : rm2.getRMContext().getRMApps().values()) {
      if (rmApp.getState() == RMAppState.NEW) {
        Thread.sleep(1000);
        atleastOneAppInNEWState = true;
        break;
      }
    }
  }
}

Example 6

Source File: TestMoveApplication.java From hadoop with Apache License 2.0

5 votes

@Test (timeout = 10000)
public void testMoveTooLate() throws Exception {
  // Submit application
  Application application = new Application("user1", resourceManager);
  ApplicationId appId = application.getApplicationId();
  application.submit();
  
  ClientRMService clientRMService = resourceManager.getClientRMService();
  // Kill the application
  clientRMService.forceKillApplication(
      KillApplicationRequest.newInstance(appId));
  RMApp rmApp = resourceManager.getRMContext().getRMApps().get(appId);
  // wait until it's dead
  while (rmApp.getState() != RMAppState.KILLED) {
    Thread.sleep(100);
  }
  
  try {
    clientRMService.moveApplicationAcrossQueues(
        MoveApplicationAcrossQueuesRequest.newInstance(appId, "newqueue"));
    fail("Should have hit exception");
  } catch (YarnException ex) {
    assertEquals(YarnException.class,
        ex.getClass());
    assertEquals("App in KILLED state cannot be moved.", ex.getMessage());
  }
}

Example 7

Source File: RMAppManager.java From big-c with Apache License 2.0

5 votes

protected void writeAuditLog(ApplicationId appId) {
  RMApp app = rmContext.getRMApps().get(appId);
  String operation = "UNKONWN";
  boolean success = false;
  switch (app.getState()) {
    case FAILED: 
      operation = AuditConstants.FINISH_FAILED_APP;
      break;
    case FINISHED:
      operation = AuditConstants.FINISH_SUCCESS_APP;
      success = true;
      break;
    case KILLED: 
      operation = AuditConstants.FINISH_KILLED_APP;
      success = true;
      break;
    default:
      break;
  }
  
  if (success) {
    RMAuditLogger.logSuccess(app.getUser(), operation,
        "RMAppManager", app.getApplicationId());
  } else {
    StringBuilder diag = app.getDiagnostics(); 
    String msg = diag == null ? null : diag.toString();
    RMAuditLogger.logFailure(app.getUser(), operation, msg, "RMAppManager",
        "App failed with state: " + app.getState(), appId);
  }
}

Example 8

Source File: TestAppManager.java From big-c with Apache License 2.0

5 votes

protected void addToCompletedApps(TestRMAppManager appMonitor, RMContext rmContext) {
  for (RMApp app : rmContext.getRMApps().values()) {
    if (app.getState() == RMAppState.FINISHED
        || app.getState() == RMAppState.KILLED 
        || app.getState() == RMAppState.FAILED) {
      appMonitor.finishApplication(app.getApplicationId());
    }
  }
}

Example 9

Source File: TestRMRestart.java From big-c with Apache License 2.0

5 votes

private void waitForTokensToBeRenewed(MockRM rm2) throws Exception {
  int waitCnt = 20;
  boolean atleastOneAppInNEWState = true;
  while (waitCnt-- > 0 && atleastOneAppInNEWState) {
    atleastOneAppInNEWState = false;
    for (RMApp rmApp : rm2.getRMContext().getRMApps().values()) {
      if (rmApp.getState() == RMAppState.NEW) {
        Thread.sleep(1000);
        atleastOneAppInNEWState = true;
        break;
      }
    }
  }
}

Example 10

Source File: TestMoveApplication.java From big-c with Apache License 2.0

5 votes

@Test (timeout = 10000)
public void testMoveTooLate() throws Exception {
  // Submit application
  Application application = new Application("user1", resourceManager);
  ApplicationId appId = application.getApplicationId();
  application.submit();
  
  ClientRMService clientRMService = resourceManager.getClientRMService();
  // Kill the application
  clientRMService.forceKillApplication(
      KillApplicationRequest.newInstance(appId));
  RMApp rmApp = resourceManager.getRMContext().getRMApps().get(appId);
  // wait until it's dead
  while (rmApp.getState() != RMAppState.KILLED) {
    Thread.sleep(100);
  }
  
  try {
    clientRMService.moveApplicationAcrossQueues(
        MoveApplicationAcrossQueuesRequest.newInstance(appId, "newqueue"));
    fail("Should have hit exception");
  } catch (YarnException ex) {
    assertEquals(YarnException.class,
        ex.getClass());
    assertEquals("App in KILLED state cannot be moved.", ex.getMessage());
  }
}

Example 11

Source File: RMContextImplEventRunnable.java From garmadon with Apache License 2.0

4 votes

public void sendAppEvent(ApplicationId applicationId, RMApp rmApp) {
    if (cacheFinishedApp.getIfPresent(applicationId.toString()) == null) {
        Header.Builder headerBuilder = Header.newBuilder()
            .withId(applicationId.toString())
            .withApplicationID(applicationId.toString())
            .withUser(rmApp.getUser())
            .withApplicationName(rmApp.getName())
            .withFramework(rmApp.getApplicationType().toUpperCase());

        ApplicationEvent.Builder eventBuilder = ApplicationEvent.newBuilder()
            .setState(rmApp.getState().name())
            .setQueue(rmApp.getQueue());

        rmApp.getApplicationTags().stream()
            .filter(tag -> YARN_TAGS_TO_EXTRACT.stream().noneMatch(tag::startsWith) && !tag.contains(":"))
            .forEach(eventBuilder::addYarnTags);

        rmApp.getApplicationTags().stream()
            .filter(tag -> tag.contains(":") && YARN_TAGS_TO_EXTRACT.stream().anyMatch(tag::startsWith))
            .map(tag -> {
                int idx = tag.indexOf(':');
                String key = tag.substring(0, idx);
                String value = tag.substring(idx + 1);
                return new String[] {key, value};
            })
            .forEach(splitTag -> BUILDERS.get(splitTag[0]).accept(splitTag[1], eventBuilder));

        eventBuilder.setFinalStatus(rmApp.getFinalApplicationStatus().name());

        eventBuilder.setStartTime(rmApp.getStartTime());
        eventBuilder.setFinishTime(rmApp.getFinishTime());

        RMAppMetrics rmAppMetrics = rmApp.getRMAppMetrics();
        if (rmAppMetrics != null) {
            eventBuilder.setMemorySeconds(rmAppMetrics.getMemorySeconds());
            eventBuilder.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
        }

        RMAppAttempt rmAppAttempt = rmApp.getCurrentAppAttempt();
        if (rmAppAttempt != null) {
            headerBuilder.withAttemptID(rmAppAttempt.getAppAttemptId().toString());

            Container container = rmAppAttempt.getMasterContainer();
            if (container != null) {
                eventBuilder.setAmContainerId(container.getId().toString());
            }
        }

        if (rmApp.getTrackingUrl() != null) {
            eventBuilder.setTrackingUrl(normalizeTrackingUrl(rmApp.getTrackingUrl()));
        }

        if (rmApp.getOriginalTrackingUrl() != null && !"N/A".equals(rmApp.getOriginalTrackingUrl())) {
            eventBuilder.setOriginalTrackingUrl(normalizeTrackingUrl(rmApp.getOriginalTrackingUrl()));
        }

        eventHandler.accept(System.currentTimeMillis(), headerBuilder.build(), eventBuilder.build());

        if (rmApp.getState() == RMAppState.FINISHED || rmApp.getState() == RMAppState.KILLED || rmApp.getState() == RMAppState.FAILED) {
            cacheFinishedApp.put(applicationId.toString(), rmApp.getState().name());
        }
    }
}

Example 12

Source File: ClientRMService.java From hadoop with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
@Override
public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
    MoveApplicationAcrossQueuesRequest request) throws YarnException {
  ApplicationId applicationId = request.getApplicationId();

  UserGroupInformation callerUGI;
  try {
    callerUGI = UserGroupInformation.getCurrentUser();
  } catch (IOException ie) {
    LOG.info("Error getting UGI ", ie);
    RMAuditLogger.logFailure("UNKNOWN", AuditConstants.MOVE_APP_REQUEST,
        "UNKNOWN", "ClientRMService" , "Error getting UGI",
        applicationId);
    throw RPCUtil.getRemoteException(ie);
  }

  RMApp application = this.rmContext.getRMApps().get(applicationId);
  if (application == null) {
    RMAuditLogger.logFailure(callerUGI.getUserName(),
        AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService",
        "Trying to move an absent application", applicationId);
    throw new ApplicationNotFoundException("Trying to move an absent"
        + " application " + applicationId);
  }

  if (!checkAccess(callerUGI, application.getUser(),
      ApplicationAccessType.MODIFY_APP, application)) {
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.MOVE_APP_REQUEST,
        "User doesn't have permissions to "
            + ApplicationAccessType.MODIFY_APP.toString(), "ClientRMService",
        AuditConstants.UNAUTHORIZED_USER, applicationId);
    throw RPCUtil.getRemoteException(new AccessControlException("User "
        + callerUGI.getShortUserName() + " cannot perform operation "
        + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId));
  }
  
  // Moves only allowed when app is in a state that means it is tracked by
  // the scheduler
  if (EnumSet.of(RMAppState.NEW, RMAppState.NEW_SAVING, RMAppState.FAILED,
      RMAppState.FINAL_SAVING, RMAppState.FINISHING, RMAppState.FINISHED,
      RMAppState.KILLED, RMAppState.KILLING, RMAppState.FAILED)
      .contains(application.getState())) {
    String msg = "App in " + application.getState() + " state cannot be moved.";
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService", msg);
    throw new YarnException(msg);
  }

  SettableFuture<Object> future = SettableFuture.create();
  this.rmContext.getDispatcher().getEventHandler().handle(
      new RMAppMoveEvent(applicationId, request.getTargetQueue(), future));
  
  try {
    Futures.get(future, YarnException.class);
  } catch (YarnException ex) {
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService",
        ex.getMessage());
    throw ex;
  }

  RMAuditLogger.logSuccess(callerUGI.getShortUserName(), 
      AuditConstants.MOVE_APP_REQUEST, "ClientRMService" , applicationId);
  MoveApplicationAcrossQueuesResponse response = recordFactory
      .newRecordInstance(MoveApplicationAcrossQueuesResponse.class);
  return response;
}

Example 13

Source File: AMSimulator.java From hadoop with Apache License 2.0

4 votes

private void submitApp()
        throws YarnException, InterruptedException, IOException {
  // ask for new application
  GetNewApplicationRequest newAppRequest =
      Records.newRecord(GetNewApplicationRequest.class);
  GetNewApplicationResponse newAppResponse = 
      rm.getClientRMService().getNewApplication(newAppRequest);
  appId = newAppResponse.getApplicationId();
  
  // submit the application
  final SubmitApplicationRequest subAppRequest =
      Records.newRecord(SubmitApplicationRequest.class);
  ApplicationSubmissionContext appSubContext = 
      Records.newRecord(ApplicationSubmissionContext.class);
  appSubContext.setApplicationId(appId);
  appSubContext.setMaxAppAttempts(1);
  appSubContext.setQueue(queue);
  appSubContext.setPriority(Priority.newInstance(0));
  ContainerLaunchContext conLauContext = 
      Records.newRecord(ContainerLaunchContext.class);
  conLauContext.setApplicationACLs(
      new HashMap<ApplicationAccessType, String>());
  conLauContext.setCommands(new ArrayList<String>());
  conLauContext.setEnvironment(new HashMap<String, String>());
  conLauContext.setLocalResources(new HashMap<String, LocalResource>());
  conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
  appSubContext.setAMContainerSpec(conLauContext);
  appSubContext.setUnmanagedAM(true);
  subAppRequest.setApplicationSubmissionContext(appSubContext);
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws YarnException {
      rm.getClientRMService().submitApplication(subAppRequest);
      return null;
    }
  });
  LOG.info(MessageFormat.format("Submit a new application {0}", appId));
  
  // waiting until application ACCEPTED
  RMApp app = rm.getRMContext().getRMApps().get(appId);
  while(app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(10);
  }

  // Waiting until application attempt reach LAUNCHED
  // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
  this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt().getAppAttemptId();
  RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt();
  while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
    Thread.sleep(10);
  }
}

Example 14

Source File: ClientRMService.java From big-c with Apache License 2.0

4 votes

@SuppressWarnings("unchecked")
@Override
public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
    MoveApplicationAcrossQueuesRequest request) throws YarnException {
  ApplicationId applicationId = request.getApplicationId();

  UserGroupInformation callerUGI;
  try {
    callerUGI = UserGroupInformation.getCurrentUser();
  } catch (IOException ie) {
    LOG.info("Error getting UGI ", ie);
    RMAuditLogger.logFailure("UNKNOWN", AuditConstants.MOVE_APP_REQUEST,
        "UNKNOWN", "ClientRMService" , "Error getting UGI",
        applicationId);
    throw RPCUtil.getRemoteException(ie);
  }

  RMApp application = this.rmContext.getRMApps().get(applicationId);
  if (application == null) {
    RMAuditLogger.logFailure(callerUGI.getUserName(),
        AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService",
        "Trying to move an absent application", applicationId);
    throw new ApplicationNotFoundException("Trying to move an absent"
        + " application " + applicationId);
  }

  if (!checkAccess(callerUGI, application.getUser(),
      ApplicationAccessType.MODIFY_APP, application)) {
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.MOVE_APP_REQUEST,
        "User doesn't have permissions to "
            + ApplicationAccessType.MODIFY_APP.toString(), "ClientRMService",
        AuditConstants.UNAUTHORIZED_USER, applicationId);
    throw RPCUtil.getRemoteException(new AccessControlException("User "
        + callerUGI.getShortUserName() + " cannot perform operation "
        + ApplicationAccessType.MODIFY_APP.name() + " on " + applicationId));
  }
  
  // Moves only allowed when app is in a state that means it is tracked by
  // the scheduler
  if (EnumSet.of(RMAppState.NEW, RMAppState.NEW_SAVING, RMAppState.FAILED,
      RMAppState.FINAL_SAVING, RMAppState.FINISHING, RMAppState.FINISHED,
      RMAppState.KILLED, RMAppState.KILLING, RMAppState.FAILED)
      .contains(application.getState())) {
    String msg = "App in " + application.getState() + " state cannot be moved.";
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService", msg);
    throw new YarnException(msg);
  }

  SettableFuture<Object> future = SettableFuture.create();
  this.rmContext.getDispatcher().getEventHandler().handle(
      new RMAppMoveEvent(applicationId, request.getTargetQueue(), future));
  
  try {
    Futures.get(future, YarnException.class);
  } catch (YarnException ex) {
    RMAuditLogger.logFailure(callerUGI.getShortUserName(),
        AuditConstants.MOVE_APP_REQUEST, "UNKNOWN", "ClientRMService",
        ex.getMessage());
    throw ex;
  }

  RMAuditLogger.logSuccess(callerUGI.getShortUserName(), 
      AuditConstants.MOVE_APP_REQUEST, "ClientRMService" , applicationId);
  MoveApplicationAcrossQueuesResponse response = recordFactory
      .newRecordInstance(MoveApplicationAcrossQueuesResponse.class);
  return response;
}

Example 15

Source File: AMSimulator.java From big-c with Apache License 2.0

4 votes

private void submitApp()
        throws YarnException, InterruptedException, IOException {
  // ask for new application
  GetNewApplicationRequest newAppRequest =
      Records.newRecord(GetNewApplicationRequest.class);
  GetNewApplicationResponse newAppResponse = 
      rm.getClientRMService().getNewApplication(newAppRequest);
  appId = newAppResponse.getApplicationId();
  
  // submit the application
  final SubmitApplicationRequest subAppRequest =
      Records.newRecord(SubmitApplicationRequest.class);
  ApplicationSubmissionContext appSubContext = 
      Records.newRecord(ApplicationSubmissionContext.class);
  appSubContext.setApplicationId(appId);
  appSubContext.setMaxAppAttempts(1);
  appSubContext.setQueue(queue);
  appSubContext.setPriority(Priority.newInstance(0));
  ContainerLaunchContext conLauContext = 
      Records.newRecord(ContainerLaunchContext.class);
  conLauContext.setApplicationACLs(
      new HashMap<ApplicationAccessType, String>());
  conLauContext.setCommands(new ArrayList<String>());
  conLauContext.setEnvironment(new HashMap<String, String>());
  conLauContext.setLocalResources(new HashMap<String, LocalResource>());
  conLauContext.setServiceData(new HashMap<String, ByteBuffer>());
  appSubContext.setAMContainerSpec(conLauContext);
  appSubContext.setUnmanagedAM(true);
  subAppRequest.setApplicationSubmissionContext(appSubContext);
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws YarnException {
      rm.getClientRMService().submitApplication(subAppRequest);
      return null;
    }
  });
  LOG.info(MessageFormat.format("Submit a new application {0}", appId));
  
  // waiting until application ACCEPTED
  RMApp app = rm.getRMContext().getRMApps().get(appId);
  while(app.getState() != RMAppState.ACCEPTED) {
    Thread.sleep(10);
  }

  // Waiting until application attempt reach LAUNCHED
  // "Unmanaged AM must register after AM attempt reaches LAUNCHED state"
  this.appAttemptId = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt().getAppAttemptId();
  RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId)
      .getCurrentAppAttempt();
  while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) {
    Thread.sleep(10);
  }
}