Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics#getMemorySeconds()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics#getMemorySeconds() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: AppInfo.java From hadoop with Apache License 2.0

4 votes

@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          allocatedGCores = usedResources.getGpuCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    preemptedResourceGCores =
        appMetrics.getResourcePreempted().getGpuCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
    gcoreSeconds = appMetrics.getGcoreSeconds();
  }
}

Example 2

Source File: TestContainerResourceUsage.java From hadoop with Apache License 2.0

4 votes

@Test (timeout = 120000)
public void testUsageWithOneAttemptAndOneContainer() throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();
  
  MockNM nm =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm.registerNode();

  RMApp app0 = rm.submitApp(200);

  RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "Before app submittion, memory seconds should have been 0 but was "
                        + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() == 0);
  Assert.assertTrue(
      "Before app submission, vcore seconds should have been 0 but was "
                        + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() == 0);

  Assert.assertTrue(
      "Before app submission, gcore seconds should have been 0 but was "
                        + rmAppMetrics.getGcoreSeconds(),
      rmAppMetrics.getGcoreSeconds() == 0);

  RMAppAttempt attempt0 = app0.getCurrentAppAttempt();

  nm.nodeHeartbeat(true);
  MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId());
  am0.registerAppAttempt();

  RMContainer rmContainer =
      rm.getResourceScheduler()
         .getRMContainer(attempt0.getMasterContainer().getId());

  // Allow metrics to accumulate.
  int sleepInterval = 1000;
  int cumulativeSleepTime = 0;
  while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
    Thread.sleep(sleepInterval);
    cumulativeSleepTime += sleepInterval;
  }

  rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "While app is running, memory seconds should be >0 but is "
          + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() > 0);
  Assert.assertTrue(
      "While app is running, vcore seconds should be >0 but is "
          + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() > 0);

  MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);

  AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
  rmAppMetrics = app0.getRMAppMetrics();

  Assert.assertEquals("Unexcpected MemorySeconds value",
      ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
  Assert.assertEquals("Unexpected VcoreSeconds value",
      ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
  Assert.assertEquals("Unexpected GcoreSeconds value",
      ru.getGcoreSeconds(), rmAppMetrics.getGcoreSeconds());
  rm.stop();
}

Example 3

Source File: AppInfo.java From big-c with Apache License 2.0

4 votes

@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
  }
}

Example 4

Source File: TestContainerResourceUsage.java From big-c with Apache License 2.0

4 votes

@Test (timeout = 120000)
public void testUsageWithOneAttemptAndOneContainer() throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();
  
  MockNM nm =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm.registerNode();

  RMApp app0 = rm.submitApp(200);

  RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "Before app submittion, memory seconds should have been 0 but was "
                        + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() == 0);
  Assert.assertTrue(
      "Before app submission, vcore seconds should have been 0 but was "
                        + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() == 0);

  RMAppAttempt attempt0 = app0.getCurrentAppAttempt();

  nm.nodeHeartbeat(true);
  MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId());
  am0.registerAppAttempt();

  RMContainer rmContainer =
      rm.getResourceScheduler()
         .getRMContainer(attempt0.getMasterContainer().getId());

  // Allow metrics to accumulate.
  int sleepInterval = 1000;
  int cumulativeSleepTime = 0;
  while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
    Thread.sleep(sleepInterval);
    cumulativeSleepTime += sleepInterval;
  }

  rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "While app is running, memory seconds should be >0 but is "
          + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() > 0);
  Assert.assertTrue(
      "While app is running, vcore seconds should be >0 but is "
          + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() > 0);

  MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);

  AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
  rmAppMetrics = app0.getRMAppMetrics();

  Assert.assertEquals("Unexcpected MemorySeconds value",
      ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
  Assert.assertEquals("Unexpected VcoreSeconds value",
      ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());

  rm.stop();
}