Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp#getApplicationId()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp#getApplicationId() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   File: RMWebServices.java    License: Apache License 2.0 4 votes vote down vote up
protected Response killApp(RMApp app, UserGroupInformation callerUGI,
    HttpServletRequest hsr) throws IOException, InterruptedException {

  if (app == null) {
    throw new IllegalArgumentException("app cannot be null");
  }
  String userName = callerUGI.getUserName();
  final ApplicationId appid = app.getApplicationId();
  KillApplicationResponse resp = null;
  try {
    resp =
        callerUGI
          .doAs(new PrivilegedExceptionAction<KillApplicationResponse>() {
            @Override
            public KillApplicationResponse run() throws IOException,
                YarnException {
              KillApplicationRequest req =
                  KillApplicationRequest.newInstance(appid);
              return rm.getClientRMService().forceKillApplication(req);
            }
          });
  } catch (UndeclaredThrowableException ue) {
    // if the root cause is a permissions issue
    // bubble that up to the user
    if (ue.getCause() instanceof YarnException) {
      YarnException ye = (YarnException) ue.getCause();
      if (ye.getCause() instanceof AccessControlException) {
        String appId = app.getApplicationId().toString();
        String msg =
            "Unauthorized attempt to kill appid " + appId
                + " by remote user " + userName;
        return Response.status(Status.FORBIDDEN).entity(msg).build();
      } else {
        throw ue;
      }
    } else {
      throw ue;
    }
  }

  AppState ret = new AppState();
  ret.setState(app.getState().toString());

  if (resp.getIsKillCompleted()) {
    RMAuditLogger.logSuccess(userName, AuditConstants.KILL_APP_REQUEST,
      "RMWebService", app.getApplicationId());
  } else {
    return Response.status(Status.ACCEPTED).entity(ret)
      .header(HttpHeaders.LOCATION, hsr.getRequestURL()).build();
  }
  return Response.status(Status.OK).entity(ret).build();
}
 
Example 2
Source Project: hadoop   File: RMWebServices.java    License: Apache License 2.0 4 votes vote down vote up
protected Response moveApp(RMApp app, UserGroupInformation callerUGI,
    String targetQueue) throws IOException, InterruptedException {

  if (app == null) {
    throw new IllegalArgumentException("app cannot be null");
  }
  String userName = callerUGI.getUserName();
  final ApplicationId appid = app.getApplicationId();
  final String reqTargetQueue = targetQueue;
  try {
    callerUGI
      .doAs(new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws IOException,
            YarnException {
          MoveApplicationAcrossQueuesRequest req =
              MoveApplicationAcrossQueuesRequest.newInstance(appid,
                reqTargetQueue);
          rm.getClientRMService().moveApplicationAcrossQueues(req);
          return null;
        }
      });
  } catch (UndeclaredThrowableException ue) {
    // if the root cause is a permissions issue
    // bubble that up to the user
    if (ue.getCause() instanceof YarnException) {
      YarnException ye = (YarnException) ue.getCause();
      if (ye.getCause() instanceof AccessControlException) {
        String appId = app.getApplicationId().toString();
        String msg =
            "Unauthorized attempt to move appid " + appId
                + " by remote user " + userName;
        return Response.status(Status.FORBIDDEN).entity(msg).build();
      } else if (ye.getMessage().startsWith("App in")
          && ye.getMessage().endsWith("state cannot be moved.")) {
        return Response.status(Status.BAD_REQUEST).entity(ye.getMessage())
          .build();
      } else {
        throw ue;
      }
    } else {
      throw ue;
    }
  }

  AppQueue ret = new AppQueue();
  ret.setQueue(app.getQueue());
  return Response.status(Status.OK).entity(ret).build();
}
 
Example 3
Source Project: hadoop   File: AppInfo.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          allocatedGCores = usedResources.getGpuCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    preemptedResourceGCores =
        appMetrics.getResourcePreempted().getGpuCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
    gcoreSeconds = appMetrics.getGcoreSeconds();
  }
}
 
Example 4
Source Project: hadoop   File: TestRMContainerAllocator.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=60000)
public void testAMRMTokenUpdate() throws Exception {
  LOG.info("Running testAMRMTokenUpdate");

  final String rmAddr = "somermaddress:1234";
  final Configuration conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 8);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 2000);
  conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);

  final MyResourceManager rm = new MyResourceManager(conf);
  rm.start();
  AMRMTokenSecretManager secretMgr =
      rm.getRMContext().getAMRMTokenSecretManager();
  DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
      .getDispatcher();

  // Submit the application
  RMApp app = rm.submitApp(1024);
  dispatcher.await();

  MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
  amNodeManager.nodeHeartbeat(true);
  dispatcher.await();

  final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
      .getAppAttemptId();
  final ApplicationId appId = app.getApplicationId();
  rm.sendAMLaunched(appAttemptId);
  dispatcher.await();

  JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
  final Job mockJob = mock(Job.class);
  when(mockJob.getReport()).thenReturn(
      MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
          0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));

  final Token<AMRMTokenIdentifier> oldToken = rm.getRMContext().getRMApps()
      .get(appId).getRMAppAttempt(appAttemptId).getAMRMToken();
  Assert.assertNotNull("app should have a token", oldToken);
  UserGroupInformation testUgi = UserGroupInformation.createUserForTesting(
      "someuser", new String[0]);
  Token<AMRMTokenIdentifier> newToken = testUgi.doAs(
      new PrivilegedExceptionAction<Token<AMRMTokenIdentifier>>() {
        @Override
        public Token<AMRMTokenIdentifier> run() throws Exception {
          MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
              appAttemptId, mockJob);

          // Keep heartbeating until RM thinks the token has been updated
          Token<AMRMTokenIdentifier> currentToken = oldToken;
          long startTime = Time.monotonicNow();
          while (currentToken == oldToken) {
            if (Time.monotonicNow() - startTime > 20000) {
              Assert.fail("Took to long to see AMRM token change");
            }
            Thread.sleep(100);
            allocator.schedule();
            currentToken = rm.getRMContext().getRMApps().get(appId)
                .getRMAppAttempt(appAttemptId).getAMRMToken();
          }

          return currentToken;
        }
      });

  // verify there is only one AMRM token in the UGI and it matches the
  // updated token from the RM
  int tokenCount = 0;
  Token<? extends TokenIdentifier> ugiToken = null;
  for (Token<? extends TokenIdentifier> token : testUgi.getTokens()) {
    if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())) {
      ugiToken = token;
      ++tokenCount;
    }
  }

  Assert.assertEquals("too many AMRM tokens", 1, tokenCount);
  Assert.assertArrayEquals("token identifier not updated",
      newToken.getIdentifier(), ugiToken.getIdentifier());
  Assert.assertArrayEquals("token password not updated",
      newToken.getPassword(), ugiToken.getPassword());
  Assert.assertEquals("AMRM token service not updated",
      new Text(rmAddr), ugiToken.getService());
}
 
Example 5
Source Project: big-c   File: RMWebServices.java    License: Apache License 2.0 4 votes vote down vote up
protected Response killApp(RMApp app, UserGroupInformation callerUGI,
    HttpServletRequest hsr) throws IOException, InterruptedException {

  if (app == null) {
    throw new IllegalArgumentException("app cannot be null");
  }
  String userName = callerUGI.getUserName();
  final ApplicationId appid = app.getApplicationId();
  KillApplicationResponse resp = null;
  try {
    resp =
        callerUGI
          .doAs(new PrivilegedExceptionAction<KillApplicationResponse>() {
            @Override
            public KillApplicationResponse run() throws IOException,
                YarnException {
              KillApplicationRequest req =
                  KillApplicationRequest.newInstance(appid);
              return rm.getClientRMService().forceKillApplication(req);
            }
          });
  } catch (UndeclaredThrowableException ue) {
    // if the root cause is a permissions issue
    // bubble that up to the user
    if (ue.getCause() instanceof YarnException) {
      YarnException ye = (YarnException) ue.getCause();
      if (ye.getCause() instanceof AccessControlException) {
        String appId = app.getApplicationId().toString();
        String msg =
            "Unauthorized attempt to kill appid " + appId
                + " by remote user " + userName;
        return Response.status(Status.FORBIDDEN).entity(msg).build();
      } else {
        throw ue;
      }
    } else {
      throw ue;
    }
  }

  AppState ret = new AppState();
  ret.setState(app.getState().toString());

  if (resp.getIsKillCompleted()) {
    RMAuditLogger.logSuccess(userName, AuditConstants.KILL_APP_REQUEST,
      "RMWebService", app.getApplicationId());
  } else {
    return Response.status(Status.ACCEPTED).entity(ret)
      .header(HttpHeaders.LOCATION, hsr.getRequestURL()).build();
  }
  return Response.status(Status.OK).entity(ret).build();
}
 
Example 6
Source Project: big-c   File: RMWebServices.java    License: Apache License 2.0 4 votes vote down vote up
protected Response moveApp(RMApp app, UserGroupInformation callerUGI,
    String targetQueue) throws IOException, InterruptedException {

  if (app == null) {
    throw new IllegalArgumentException("app cannot be null");
  }
  String userName = callerUGI.getUserName();
  final ApplicationId appid = app.getApplicationId();
  final String reqTargetQueue = targetQueue;
  try {
    callerUGI
      .doAs(new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws IOException,
            YarnException {
          MoveApplicationAcrossQueuesRequest req =
              MoveApplicationAcrossQueuesRequest.newInstance(appid,
                reqTargetQueue);
          rm.getClientRMService().moveApplicationAcrossQueues(req);
          return null;
        }
      });
  } catch (UndeclaredThrowableException ue) {
    // if the root cause is a permissions issue
    // bubble that up to the user
    if (ue.getCause() instanceof YarnException) {
      YarnException ye = (YarnException) ue.getCause();
      if (ye.getCause() instanceof AccessControlException) {
        String appId = app.getApplicationId().toString();
        String msg =
            "Unauthorized attempt to move appid " + appId
                + " by remote user " + userName;
        return Response.status(Status.FORBIDDEN).entity(msg).build();
      } else if (ye.getMessage().startsWith("App in")
          && ye.getMessage().endsWith("state cannot be moved.")) {
        return Response.status(Status.BAD_REQUEST).entity(ye.getMessage())
          .build();
      } else {
        throw ue;
      }
    } else {
      throw ue;
    }
  }

  AppQueue ret = new AppQueue();
  ret.setQueue(app.getQueue());
  return Response.status(Status.OK).entity(ret).build();
}
 
Example 7
Source Project: big-c   File: AppInfo.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
  }
}
 
Example 8
Source Project: big-c   File: TestRMContainerAllocator.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=60000)
public void testAMRMTokenUpdate() throws Exception {
  LOG.info("Running testAMRMTokenUpdate");

  final String rmAddr = "somermaddress:1234";
  final Configuration conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 8);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 2000);
  conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, rmAddr);

  final MyResourceManager rm = new MyResourceManager(conf);
  rm.start();
  AMRMTokenSecretManager secretMgr =
      rm.getRMContext().getAMRMTokenSecretManager();
  DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
      .getDispatcher();

  // Submit the application
  RMApp app = rm.submitApp(1024);
  dispatcher.await();

  MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
  amNodeManager.nodeHeartbeat(true);
  dispatcher.await();

  final ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
      .getAppAttemptId();
  final ApplicationId appId = app.getApplicationId();
  rm.sendAMLaunched(appAttemptId);
  dispatcher.await();

  JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
  final Job mockJob = mock(Job.class);
  when(mockJob.getReport()).thenReturn(
      MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
          0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));

  final Token<AMRMTokenIdentifier> oldToken = rm.getRMContext().getRMApps()
      .get(appId).getRMAppAttempt(appAttemptId).getAMRMToken();
  Assert.assertNotNull("app should have a token", oldToken);
  UserGroupInformation testUgi = UserGroupInformation.createUserForTesting(
      "someuser", new String[0]);
  Token<AMRMTokenIdentifier> newToken = testUgi.doAs(
      new PrivilegedExceptionAction<Token<AMRMTokenIdentifier>>() {
        @Override
        public Token<AMRMTokenIdentifier> run() throws Exception {
          MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
              appAttemptId, mockJob);

          // Keep heartbeating until RM thinks the token has been updated
          Token<AMRMTokenIdentifier> currentToken = oldToken;
          long startTime = Time.monotonicNow();
          while (currentToken == oldToken) {
            if (Time.monotonicNow() - startTime > 20000) {
              Assert.fail("Took to long to see AMRM token change");
            }
            Thread.sleep(100);
            allocator.schedule();
            currentToken = rm.getRMContext().getRMApps().get(appId)
                .getRMAppAttempt(appAttemptId).getAMRMToken();
          }

          return currentToken;
        }
      });

  // verify there is only one AMRM token in the UGI and it matches the
  // updated token from the RM
  int tokenCount = 0;
  Token<? extends TokenIdentifier> ugiToken = null;
  for (Token<? extends TokenIdentifier> token : testUgi.getTokens()) {
    if (AMRMTokenIdentifier.KIND_NAME.equals(token.getKind())) {
      ugiToken = token;
      ++tokenCount;
    }
  }

  Assert.assertEquals("too many AMRM tokens", 1, tokenCount);
  Assert.assertArrayEquals("token identifier not updated",
      newToken.getIdentifier(), ugiToken.getIdentifier());
  Assert.assertArrayEquals("token password not updated",
      newToken.getPassword(), ugiToken.getPassword());
  Assert.assertEquals("AMRM token service not updated",
      new Text(rmAddr), ugiToken.getService());
}