org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void waitForAppPreemptionInfo(RMApp app, Resource preempted,
    int numAMPreempted, int numTaskPreempted,
    Resource currentAttemptPreempted, boolean currentAttemptAMPreempted,
    int numLatestAttemptTaskPreempted) throws InterruptedException {
  while (true) {
    RMAppMetrics appPM = app.getRMAppMetrics();
    RMAppAttemptMetrics attemptPM =
        app.getCurrentAppAttempt().getRMAppAttemptMetrics();

    if (appPM.getResourcePreempted().equals(preempted)
        && appPM.getNumAMContainersPreempted() == numAMPreempted
        && appPM.getNumNonAMContainersPreempted() == numTaskPreempted
        && attemptPM.getResourcePreempted().equals(currentAttemptPreempted)
        && app.getCurrentAppAttempt().getRMAppAttemptMetrics()
          .getIsPreempted() == currentAttemptAMPreempted
        && attemptPM.getNumNonAMContainersPreempted() == 
           numLatestAttemptTaskPreempted) {
      return;
    }
    Thread.sleep(500);
  }
}
 
Example #2
Source File: ApplicationFinishedEvent.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public ApplicationFinishedEvent(
    ApplicationId appId,
    String diagnosticsInfo,
    FinalApplicationStatus appStatus,
    YarnApplicationState state,
    ApplicationAttemptId latestAppAttemptId,
    long finishedTime,
    RMAppMetrics appMetrics) {
  super(SystemMetricsEventType.APP_FINISHED, finishedTime);
  this.appId = appId;
  this.diagnosticsInfo = diagnosticsInfo;
  this.appStatus = appStatus;
  this.latestAppAttemptId = latestAppAttemptId;
  this.state = state;
  this.appMetrics=appMetrics;
}
 
Example #3
Source File: ApplicationFinishedEvent.java    From big-c with Apache License 2.0 6 votes vote down vote up
public ApplicationFinishedEvent(
    ApplicationId appId,
    String diagnosticsInfo,
    FinalApplicationStatus appStatus,
    YarnApplicationState state,
    ApplicationAttemptId latestAppAttemptId,
    long finishedTime,
    RMAppMetrics appMetrics) {
  super(SystemMetricsEventType.APP_FINISHED, finishedTime);
  this.appId = appId;
  this.diagnosticsInfo = diagnosticsInfo;
  this.appStatus = appStatus;
  this.latestAppAttemptId = latestAppAttemptId;
  this.state = state;
  this.appMetrics=appMetrics;
}
 
Example #4
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void waitForAppPreemptionInfo(RMApp app, Resource preempted,
    int numAMPreempted, int numTaskPreempted,
    Resource currentAttemptPreempted, boolean currentAttemptAMPreempted,
    int numLatestAttemptTaskPreempted) throws InterruptedException {
  while (true) {
    RMAppMetrics appPM = app.getRMAppMetrics();
    RMAppAttemptMetrics attemptPM =
        app.getCurrentAppAttempt().getRMAppAttemptMetrics();

    if (appPM.getResourcePreempted().equals(preempted)
        && appPM.getNumAMContainersPreempted() == numAMPreempted
        && appPM.getNumNonAMContainersPreempted() == numTaskPreempted
        && attemptPM.getResourcePreempted().equals(currentAttemptPreempted)
        && app.getCurrentAppAttempt().getRMAppAttemptMetrics()
          .getIsPreempted() == currentAttemptAMPreempted
        && attemptPM.getNumNonAMContainersPreempted() == 
           numLatestAttemptTaskPreempted) {
      return;
    }
    Thread.sleep(500);
  }
}
 
Example #5
Source File: TestSystemMetricsPublisher.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static RMApp createRMApp(ApplicationId appId) {
  RMApp app = mock(RMApp.class);
  when(app.getApplicationId()).thenReturn(appId);
  when(app.getName()).thenReturn("test app");
  when(app.getApplicationType()).thenReturn("test app type");
  when(app.getUser()).thenReturn("test user");
  when(app.getQueue()).thenReturn("test queue");
  when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
  when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
  when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
  when(app.getDiagnostics()).thenReturn(
      new StringBuilder("test diagnostics info"));
  RMAppAttempt appAttempt = mock(RMAppAttempt.class);
  when(appAttempt.getAppAttemptId()).thenReturn(
      ApplicationAttemptId.newInstance(appId, 1));
  when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
  when(app.getFinalApplicationStatus()).thenReturn(
      FinalApplicationStatus.UNDEFINED);
  when(app.getRMAppMetrics()).thenReturn(
      new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
  return app;
}
 
Example #6
Source File: TestSystemMetricsPublisher.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static RMApp createRMApp(ApplicationId appId) {
  RMApp app = mock(RMApp.class);
  when(app.getApplicationId()).thenReturn(appId);
  when(app.getName()).thenReturn("test app");
  when(app.getApplicationType()).thenReturn("test app type");
  when(app.getUser()).thenReturn("test user");
  when(app.getQueue()).thenReturn("test queue");
  when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
  when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
  when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
  when(app.getDiagnostics()).thenReturn(
      new StringBuilder("test diagnostics info"));
  RMAppAttempt appAttempt = mock(RMAppAttempt.class);
  when(appAttempt.getAppAttemptId()).thenReturn(
      ApplicationAttemptId.newInstance(appId, 1));
  when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
  when(app.getFinalApplicationStatus()).thenReturn(
      FinalApplicationStatus.UNDEFINED);
  when(app.getRMAppMetrics()).thenReturn(
      new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE));
  return app;
}
 
Example #7
Source File: TestRMWebAppFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}
 
Example #8
Source File: SystemMetricsPublisher.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
  TimelineEntity entity =
      createApplicationEntity(event.getApplicationId());
  TimelineEvent tEvent = new TimelineEvent();
  tEvent.setEventType(
      ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  Map<String, Object> eventInfo = new HashMap<String, Object>();
  eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
      event.getDiagnosticsInfo());
  eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO,
      event.getFinalApplicationStatus().toString());
  eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
      event.getYarnApplicationState().toString());
  if (event.getLatestApplicationAttemptId() != null) {
    eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
        event.getLatestApplicationAttemptId().toString());
  }
  RMAppMetrics appMetrics = event.getAppMetrics();
  entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
      appMetrics.getVcoreSeconds());
  entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
      appMetrics.getMemorySeconds());
  
  tEvent.setEventInfo(eventInfo);
  entity.addEvent(tEvent);
  putEntity(entity);
}
 
Example #9
Source File: RMAppManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * create a summary of the application's runtime.
 * 
 * @param app {@link RMApp} whose summary is to be created, cannot
 *            be <code>null</code>.
 */
public static SummaryBuilder createAppSummary(RMApp app) {
  String trackingUrl = "N/A";
  String host = "N/A";
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  if (attempt != null) {
    trackingUrl = attempt.getTrackingUrl();
    host = attempt.getHost();
  }
  RMAppMetrics metrics = app.getRMAppMetrics();
  SummaryBuilder summary = new SummaryBuilder()
      .add("appId", app.getApplicationId())
      .add("name", app.getName())
      .add("user", app.getUser())
      .add("queue", app.getQueue())
      .add("state", app.getState())
      .add("trackingUrl", trackingUrl)
      .add("appMasterHost", host)
      .add("startTime", app.getStartTime())
      .add("finishTime", app.getFinishTime())
      .add("finalStatus", app.getFinalApplicationStatus())
      .add("memorySeconds", metrics.getMemorySeconds())
      .add("vcoreSeconds", metrics.getVcoreSeconds())
      .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
      .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
      .add("preemptedResources", metrics.getResourcePreempted())
      .add("applicationType", app.getApplicationType());
  return summary;
}
 
Example #10
Source File: TestAppManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 30000)
 public void testEscapeApplicationSummary() {
   RMApp app = mock(RMAppImpl.class);
   when(app.getApplicationId()).thenReturn(
       ApplicationId.newInstance(100L, 1));
   when(app.getName()).thenReturn("Multiline\n\n\r\rAppName");
   when(app.getUser()).thenReturn("Multiline\n\n\r\rUserName");
   when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
   when(app.getState()).thenReturn(RMAppState.RUNNING);
   when(app.getApplicationType()).thenReturn("MAPREDUCE");
   RMAppMetrics metrics =
       new RMAppMetrics(Resource.newInstance(1234, 56), 10, 1, 16384, 64);
   when(app.getRMAppMetrics()).thenReturn(metrics);

   RMAppManager.ApplicationSummary.SummaryBuilder summary =
       new RMAppManager.ApplicationSummary().createAppSummary(app);
   String msg = summary.toString();
   LOG.info("summary: " + msg);
   Assert.assertFalse(msg.contains("\n"));
   Assert.assertFalse(msg.contains("\r"));

   String escaped = "\\n\\n\\r\\r";
   Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
   Assert.assertTrue(msg.contains("memorySeconds=16384"));
   Assert.assertTrue(msg.contains("vcoreSeconds=64"));
   Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
   Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
   Assert.assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
   Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
 
Example #11
Source File: TestRMWebAppFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}
 
Example #12
Source File: TestAppManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 30000)
 public void testEscapeApplicationSummary() {
   RMApp app = mock(RMAppImpl.class);
   when(app.getApplicationId()).thenReturn(
       ApplicationId.newInstance(100L, 1));
   when(app.getName()).thenReturn("Multiline\n\n\r\rAppName");
   when(app.getUser()).thenReturn("Multiline\n\n\r\rUserName");
   when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
   when(app.getState()).thenReturn(RMAppState.RUNNING);
   when(app.getApplicationType()).thenReturn("MAPREDUCE");
   RMAppMetrics metrics =
       new RMAppMetrics(Resource.newInstance(1234, 56, 56), 10, 1, 16384, 64, 64);
   when(app.getRMAppMetrics()).thenReturn(metrics);

   RMAppManager.ApplicationSummary.SummaryBuilder summary =
       new RMAppManager.ApplicationSummary().createAppSummary(app);
   String msg = summary.toString();
   LOG.info("summary: " + msg);
   Assert.assertFalse(msg.contains("\n"));
   Assert.assertFalse(msg.contains("\r"));

   String escaped = "\\n\\n\\r\\r";
   Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
   Assert.assertTrue(msg.contains("memorySeconds=16384"));
   Assert.assertTrue(msg.contains("vcoreSeconds=64"));
   Assert.assertTrue(msg.contains("gcoreSeconds=64"));
   Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
   Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
   Assert.assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56\\, gCores:56>"));
   Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
 
Example #13
Source File: RMAppManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * create a summary of the application's runtime.
 * 
 * @param app {@link RMApp} whose summary is to be created, cannot
 *            be <code>null</code>.
 */
public static SummaryBuilder createAppSummary(RMApp app) {
  String trackingUrl = "N/A";
  String host = "N/A";
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  if (attempt != null) {
    trackingUrl = attempt.getTrackingUrl();
    host = attempt.getHost();
  }
  RMAppMetrics metrics = app.getRMAppMetrics();
  SummaryBuilder summary = new SummaryBuilder()
      .add("appId", app.getApplicationId())
      .add("name", app.getName())
      .add("user", app.getUser())
      .add("queue", app.getQueue())
      .add("state", app.getState())
      .add("trackingUrl", trackingUrl)
      .add("appMasterHost", host)
      .add("startTime", app.getStartTime())
      .add("finishTime", app.getFinishTime())
      .add("finalStatus", app.getFinalApplicationStatus())
      .add("memorySeconds", metrics.getMemorySeconds())
      .add("vcoreSeconds", metrics.getVcoreSeconds())
      .add("gcoreSeconds", metrics.getGcoreSeconds())
      .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
      .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
      .add("preemptedResources", metrics.getResourcePreempted())
      .add("applicationType", app.getApplicationType());
  return summary;
}
 
Example #14
Source File: SystemMetricsPublisher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
  TimelineEntity entity =
      createApplicationEntity(event.getApplicationId());
  TimelineEvent tEvent = new TimelineEvent();
  tEvent.setEventType(
      ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  Map<String, Object> eventInfo = new HashMap<String, Object>();
  eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
      event.getDiagnosticsInfo());
  eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO,
      event.getFinalApplicationStatus().toString());
  eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
      event.getYarnApplicationState().toString());
  if (event.getLatestApplicationAttemptId() != null) {
    eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
        event.getLatestApplicationAttemptId().toString());
  }
  RMAppMetrics appMetrics = event.getAppMetrics();
  entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
      appMetrics.getVcoreSeconds());
  entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
      appMetrics.getMemorySeconds());
  entity.addOtherInfo(ApplicationMetricsConstants.APP_GPU_METRICS,
      appMetrics.getGcoreSeconds());

  tEvent.setEventInfo(eventInfo);
  entity.addEvent(tEvent);
  putEntity(entity);
}
 
Example #15
Source File: TestContainerResourceUsage.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test (timeout = 120000)
public void testUsageWithOneAttemptAndOneContainer() throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();
  
  MockNM nm =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm.registerNode();

  RMApp app0 = rm.submitApp(200);

  RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "Before app submittion, memory seconds should have been 0 but was "
                        + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() == 0);
  Assert.assertTrue(
      "Before app submission, vcore seconds should have been 0 but was "
                        + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() == 0);

  RMAppAttempt attempt0 = app0.getCurrentAppAttempt();

  nm.nodeHeartbeat(true);
  MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId());
  am0.registerAppAttempt();

  RMContainer rmContainer =
      rm.getResourceScheduler()
         .getRMContainer(attempt0.getMasterContainer().getId());

  // Allow metrics to accumulate.
  int sleepInterval = 1000;
  int cumulativeSleepTime = 0;
  while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
    Thread.sleep(sleepInterval);
    cumulativeSleepTime += sleepInterval;
  }

  rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "While app is running, memory seconds should be >0 but is "
          + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() > 0);
  Assert.assertTrue(
      "While app is running, vcore seconds should be >0 but is "
          + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() > 0);

  MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);

  AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
  rmAppMetrics = app0.getRMAppMetrics();

  Assert.assertEquals("Unexcpected MemorySeconds value",
      ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
  Assert.assertEquals("Unexpected VcoreSeconds value",
      ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());

  rm.stop();
}
 
Example #16
Source File: AppInfo.java    From big-c with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
  }
}
 
Example #17
Source File: TestAppPage.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception {
  final ApplicationId APP_ID = ApplicationId.newInstance(1234L, 0);
  Injector injector;
  
  // init app
  RMApp app = mock(RMApp.class);
  when(app.getTrackingUrl()).thenReturn("http://host:123");
  when(app.getState()).thenReturn(RMAppState.FAILED);
  when(app.getApplicationId()).thenReturn(APP_ID);
  when(app.getApplicationType()).thenReturn("Type");
  when(app.getUser()).thenReturn("user");
  when(app.getName()).thenReturn("Name");
  when(app.getQueue()).thenReturn("queue");
  when(app.getDiagnostics()).thenReturn(new StringBuilder());
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
  when(app.getStartTime()).thenReturn(0L);
  when(app.getFinishTime()).thenReturn(0L);
  when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
  
  RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
  when(app.getRMAppMetrics()).thenReturn(appMetrics);
  
  // initialize RM Context, and create RMApp, without creating RMAppAttempt
  final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8);
  rmContext.getRMApps().put(APP_ID, app);
  
  injector =
      WebAppTests.createMockInjector(RMContext.class, rmContext,
          new Module() {
            @Override
            public void configure(Binder binder) {
              try {
                ResourceManager rm = TestRMWebApp.mockRm(rmContext);
                binder.bind(ResourceManager.class).toInstance(rm);
                binder.bind(ApplicationBaseProtocol.class).toInstance(
                  rm.getClientRMService());
              } catch (IOException e) {
                throw new IllegalStateException(e);
              }
            }
          });
  
  AppBlock instance = injector.getInstance(AppBlock.class);
  instance.set(YarnWebParams.APPLICATION_ID, APP_ID.toString());
  instance.render();
}
 
Example #18
Source File: MockAsm.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public RMAppMetrics getRMAppMetrics() {
  return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
}
 
Example #19
Source File: MockRMApp.java    From incubator-myriad with Apache License 2.0 4 votes vote down vote up
@Override
public RMAppMetrics getRMAppMetrics() {
  return null;
}
 
Example #20
Source File: ApplicationFinishedEvent.java    From big-c with Apache License 2.0 4 votes vote down vote up
public RMAppMetrics getAppMetrics() {
  return appMetrics;
}
 
Example #21
Source File: RMAppBlock.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
protected void createApplicationMetricsTable(Block html){
  RMApp rmApp = this.rm.getRMContext().getRMApps().get(appID);
  RMAppMetrics appMetrics = rmApp == null ? null : rmApp.getRMAppMetrics();
  // Get attempt metrics and fields, it is possible currentAttempt of RMApp is
  // null. In that case, we will assume resource preempted and number of Non
  // AM container preempted on that attempt is 0
  RMAppAttemptMetrics attemptMetrics;
  if (rmApp == null || null == rmApp.getCurrentAppAttempt()) {
    attemptMetrics = null;
  } else {
    attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
  }
  Resource attemptResourcePreempted =
      attemptMetrics == null ? Resources.none() : attemptMetrics
        .getResourcePreempted();
  int attemptNumNonAMContainerPreempted =
      attemptMetrics == null ? 0 : attemptMetrics
        .getNumNonAMContainersPreempted();
  DIV<Hamlet> pdiv = html.
      _(InfoBlock.class).
      div(_INFO_WRAP);
  info("Application Overview").clear();
  info("Application Metrics")
      ._("Total Resource Preempted:",
        appMetrics == null ? "N/A" : appMetrics.getResourcePreempted())
      ._("Total Number of Non-AM Containers Preempted:",
        appMetrics == null ? "N/A"
            : appMetrics.getNumNonAMContainersPreempted())
      ._("Total Number of AM Containers Preempted:",
        appMetrics == null ? "N/A"
            : appMetrics.getNumAMContainersPreempted())
      ._("Resource Preempted from Current Attempt:",
        attemptResourcePreempted)
      ._("Number of Non-AM Containers Preempted from Current Attempt:",
        attemptNumNonAMContainerPreempted)
      ._("Aggregate Resource Allocation:",
        String.format("%d MB-seconds, %d vcore-seconds",
            appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(),
            appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()));
  pdiv._();
}
 
Example #22
Source File: RMContextImplEventRunnable.java    From garmadon with Apache License 2.0 4 votes vote down vote up
public void sendAppEvent(ApplicationId applicationId, RMApp rmApp) {
    if (cacheFinishedApp.getIfPresent(applicationId.toString()) == null) {
        Header.Builder headerBuilder = Header.newBuilder()
            .withId(applicationId.toString())
            .withApplicationID(applicationId.toString())
            .withUser(rmApp.getUser())
            .withApplicationName(rmApp.getName())
            .withFramework(rmApp.getApplicationType().toUpperCase());

        ApplicationEvent.Builder eventBuilder = ApplicationEvent.newBuilder()
            .setState(rmApp.getState().name())
            .setQueue(rmApp.getQueue());

        rmApp.getApplicationTags().stream()
            .filter(tag -> YARN_TAGS_TO_EXTRACT.stream().noneMatch(tag::startsWith) && !tag.contains(":"))
            .forEach(eventBuilder::addYarnTags);

        rmApp.getApplicationTags().stream()
            .filter(tag -> tag.contains(":") && YARN_TAGS_TO_EXTRACT.stream().anyMatch(tag::startsWith))
            .map(tag -> {
                int idx = tag.indexOf(':');
                String key = tag.substring(0, idx);
                String value = tag.substring(idx + 1);
                return new String[] {key, value};
            })
            .forEach(splitTag -> BUILDERS.get(splitTag[0]).accept(splitTag[1], eventBuilder));

        eventBuilder.setFinalStatus(rmApp.getFinalApplicationStatus().name());

        eventBuilder.setStartTime(rmApp.getStartTime());
        eventBuilder.setFinishTime(rmApp.getFinishTime());

        RMAppMetrics rmAppMetrics = rmApp.getRMAppMetrics();
        if (rmAppMetrics != null) {
            eventBuilder.setMemorySeconds(rmAppMetrics.getMemorySeconds());
            eventBuilder.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
        }

        RMAppAttempt rmAppAttempt = rmApp.getCurrentAppAttempt();
        if (rmAppAttempt != null) {
            headerBuilder.withAttemptID(rmAppAttempt.getAppAttemptId().toString());

            Container container = rmAppAttempt.getMasterContainer();
            if (container != null) {
                eventBuilder.setAmContainerId(container.getId().toString());
            }
        }

        if (rmApp.getTrackingUrl() != null) {
            eventBuilder.setTrackingUrl(normalizeTrackingUrl(rmApp.getTrackingUrl()));
        }

        if (rmApp.getOriginalTrackingUrl() != null && !"N/A".equals(rmApp.getOriginalTrackingUrl())) {
            eventBuilder.setOriginalTrackingUrl(normalizeTrackingUrl(rmApp.getOriginalTrackingUrl()));
        }

        eventHandler.accept(System.currentTimeMillis(), headerBuilder.build(), eventBuilder.build());

        if (rmApp.getState() == RMAppState.FINISHED || rmApp.getState() == RMAppState.KILLED || rmApp.getState() == RMAppState.FAILED) {
            cacheFinishedApp.put(applicationId.toString(), rmApp.getState().name());
        }
    }
}
 
Example #23
Source File: MockAsm.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public RMAppMetrics getRMAppMetrics() {
  return new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
}
 
Example #24
Source File: TestContainerResourceUsage.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test (timeout = 120000)
public void testUsageWithOneAttemptAndOneContainer() throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();
  
  MockNM nm =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm.registerNode();

  RMApp app0 = rm.submitApp(200);

  RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "Before app submittion, memory seconds should have been 0 but was "
                        + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() == 0);
  Assert.assertTrue(
      "Before app submission, vcore seconds should have been 0 but was "
                        + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() == 0);

  Assert.assertTrue(
      "Before app submission, gcore seconds should have been 0 but was "
                        + rmAppMetrics.getGcoreSeconds(),
      rmAppMetrics.getGcoreSeconds() == 0);

  RMAppAttempt attempt0 = app0.getCurrentAppAttempt();

  nm.nodeHeartbeat(true);
  MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId());
  am0.registerAppAttempt();

  RMContainer rmContainer =
      rm.getResourceScheduler()
         .getRMContainer(attempt0.getMasterContainer().getId());

  // Allow metrics to accumulate.
  int sleepInterval = 1000;
  int cumulativeSleepTime = 0;
  while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
    Thread.sleep(sleepInterval);
    cumulativeSleepTime += sleepInterval;
  }

  rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "While app is running, memory seconds should be >0 but is "
          + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() > 0);
  Assert.assertTrue(
      "While app is running, vcore seconds should be >0 but is "
          + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() > 0);

  MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);

  AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
  rmAppMetrics = app0.getRMAppMetrics();

  Assert.assertEquals("Unexcpected MemorySeconds value",
      ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
  Assert.assertEquals("Unexpected VcoreSeconds value",
      ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
  Assert.assertEquals("Unexpected GcoreSeconds value",
      ru.getGcoreSeconds(), rmAppMetrics.getGcoreSeconds());
  rm.stop();
}
 
Example #25
Source File: TestAppPage.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception {
  final ApplicationId APP_ID = ApplicationId.newInstance(1234L, 0);
  Injector injector;
  
  // init app
  RMApp app = mock(RMApp.class);
  when(app.getTrackingUrl()).thenReturn("http://host:123");
  when(app.getState()).thenReturn(RMAppState.FAILED);
  when(app.getApplicationId()).thenReturn(APP_ID);
  when(app.getApplicationType()).thenReturn("Type");
  when(app.getUser()).thenReturn("user");
  when(app.getName()).thenReturn("Name");
  when(app.getQueue()).thenReturn("queue");
  when(app.getDiagnostics()).thenReturn(new StringBuilder());
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
  when(app.getStartTime()).thenReturn(0L);
  when(app.getFinishTime()).thenReturn(0L);
  when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
  
  RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
  when(app.getRMAppMetrics()).thenReturn(appMetrics);
  
  // initialize RM Context, and create RMApp, without creating RMAppAttempt
  final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8);
  rmContext.getRMApps().put(APP_ID, app);
  
  injector =
      WebAppTests.createMockInjector(RMContext.class, rmContext,
          new Module() {
            @Override
            public void configure(Binder binder) {
              try {
                ResourceManager rm = TestRMWebApp.mockRm(rmContext);
                binder.bind(ResourceManager.class).toInstance(rm);
                binder.bind(ApplicationBaseProtocol.class).toInstance(
                  rm.getClientRMService());
              } catch (IOException e) {
                throw new IllegalStateException(e);
              }
            }
          });
  
  AppBlock instance = injector.getInstance(AppBlock.class);
  instance.set(YarnWebParams.APPLICATION_ID, APP_ID.toString());
  instance.render();
}
 
Example #26
Source File: ApplicationFinishedEvent.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public RMAppMetrics getAppMetrics() {
  return appMetrics;
}
 
Example #27
Source File: RMAppBlock.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
protected void createApplicationMetricsTable(Block html){
  RMApp rmApp = this.rm.getRMContext().getRMApps().get(appID);
  RMAppMetrics appMetrics = rmApp == null ? null : rmApp.getRMAppMetrics();
  // Get attempt metrics and fields, it is possible currentAttempt of RMApp is
  // null. In that case, we will assume resource preempted and number of Non
  // AM container preempted on that attempt is 0
  RMAppAttemptMetrics attemptMetrics;
  if (rmApp == null || null == rmApp.getCurrentAppAttempt()) {
    attemptMetrics = null;
  } else {
    attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
  }
  Resource attemptResourcePreempted =
      attemptMetrics == null ? Resources.none() : attemptMetrics
        .getResourcePreempted();
  int attemptNumNonAMContainerPreempted =
      attemptMetrics == null ? 0 : attemptMetrics
        .getNumNonAMContainersPreempted();
  DIV<Hamlet> pdiv = html.
      _(InfoBlock.class).
      div(_INFO_WRAP);
  info("Application Overview").clear();
  info("Application Metrics")
      ._("Total Resource Preempted:",
        appMetrics == null ? "N/A" : appMetrics.getResourcePreempted())
      ._("Total Number of Non-AM Containers Preempted:",
        appMetrics == null ? "N/A"
            : appMetrics.getNumNonAMContainersPreempted())
      ._("Total Number of AM Containers Preempted:",
        appMetrics == null ? "N/A"
            : appMetrics.getNumAMContainersPreempted())
      ._("Resource Preempted from Current Attempt:",
        attemptResourcePreempted)
      ._("Number of Non-AM Containers Preempted from Current Attempt:",
        attemptNumNonAMContainerPreempted)
      ._("Aggregate Resource Allocation:",
        String.format("%d MB-seconds, %d vcore-seconds, %d gcore-seconds",
            appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(),
            appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds(),
            appMetrics == null ? "N/A" : appMetrics.getGcoreSeconds()));
  pdiv._();
}
 
Example #28
Source File: AppInfo.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          allocatedGCores = usedResources.getGpuCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    preemptedResourceGCores =
        appMetrics.getResourcePreempted().getGpuCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
    gcoreSeconds = appMetrics.getGcoreSeconds();
  }
}