Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: ApplicationFinishedEvent.java    License: Apache License 2.0 6 votes vote down vote up
public ApplicationFinishedEvent(
    ApplicationId appId,
    String diagnosticsInfo,
    FinalApplicationStatus appStatus,
    YarnApplicationState state,
    ApplicationAttemptId latestAppAttemptId,
    long finishedTime,
    RMAppMetrics appMetrics) {
  super(SystemMetricsEventType.APP_FINISHED, finishedTime);
  this.appId = appId;
  this.diagnosticsInfo = diagnosticsInfo;
  this.appStatus = appStatus;
  this.latestAppAttemptId = latestAppAttemptId;
  this.state = state;
  this.appMetrics=appMetrics;
}
 
Example 2
Source Project: hadoop   Source File: TestSystemMetricsPublisher.java    License: Apache License 2.0 6 votes vote down vote up
private static RMApp createRMApp(ApplicationId appId) {
  RMApp app = mock(RMApp.class);
  when(app.getApplicationId()).thenReturn(appId);
  when(app.getName()).thenReturn("test app");
  when(app.getApplicationType()).thenReturn("test app type");
  when(app.getUser()).thenReturn("test user");
  when(app.getQueue()).thenReturn("test queue");
  when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
  when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
  when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
  when(app.getDiagnostics()).thenReturn(
      new StringBuilder("test diagnostics info"));
  RMAppAttempt appAttempt = mock(RMAppAttempt.class);
  when(appAttempt.getAppAttemptId()).thenReturn(
      ApplicationAttemptId.newInstance(appId, 1));
  when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
  when(app.getFinalApplicationStatus()).thenReturn(
      FinalApplicationStatus.UNDEFINED);
  when(app.getRMAppMetrics()).thenReturn(
      new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE));
  return app;
}
 
Example 3
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
private void waitForAppPreemptionInfo(RMApp app, Resource preempted,
    int numAMPreempted, int numTaskPreempted,
    Resource currentAttemptPreempted, boolean currentAttemptAMPreempted,
    int numLatestAttemptTaskPreempted) throws InterruptedException {
  while (true) {
    RMAppMetrics appPM = app.getRMAppMetrics();
    RMAppAttemptMetrics attemptPM =
        app.getCurrentAppAttempt().getRMAppAttemptMetrics();

    if (appPM.getResourcePreempted().equals(preempted)
        && appPM.getNumAMContainersPreempted() == numAMPreempted
        && appPM.getNumNonAMContainersPreempted() == numTaskPreempted
        && attemptPM.getResourcePreempted().equals(currentAttemptPreempted)
        && app.getCurrentAppAttempt().getRMAppAttemptMetrics()
          .getIsPreempted() == currentAttemptAMPreempted
        && attemptPM.getNumNonAMContainersPreempted() == 
           numLatestAttemptTaskPreempted) {
      return;
    }
    Thread.sleep(500);
  }
}
 
Example 4
Source Project: big-c   Source File: ApplicationFinishedEvent.java    License: Apache License 2.0 6 votes vote down vote up
public ApplicationFinishedEvent(
    ApplicationId appId,
    String diagnosticsInfo,
    FinalApplicationStatus appStatus,
    YarnApplicationState state,
    ApplicationAttemptId latestAppAttemptId,
    long finishedTime,
    RMAppMetrics appMetrics) {
  super(SystemMetricsEventType.APP_FINISHED, finishedTime);
  this.appId = appId;
  this.diagnosticsInfo = diagnosticsInfo;
  this.appStatus = appStatus;
  this.latestAppAttemptId = latestAppAttemptId;
  this.state = state;
  this.appMetrics=appMetrics;
}
 
Example 5
Source Project: big-c   Source File: TestSystemMetricsPublisher.java    License: Apache License 2.0 6 votes vote down vote up
private static RMApp createRMApp(ApplicationId appId) {
  RMApp app = mock(RMApp.class);
  when(app.getApplicationId()).thenReturn(appId);
  when(app.getName()).thenReturn("test app");
  when(app.getApplicationType()).thenReturn("test app type");
  when(app.getUser()).thenReturn("test user");
  when(app.getQueue()).thenReturn("test queue");
  when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
  when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
  when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
  when(app.getDiagnostics()).thenReturn(
      new StringBuilder("test diagnostics info"));
  RMAppAttempt appAttempt = mock(RMAppAttempt.class);
  when(appAttempt.getAppAttemptId()).thenReturn(
      ApplicationAttemptId.newInstance(appId, 1));
  when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
  when(app.getFinalApplicationStatus()).thenReturn(
      FinalApplicationStatus.UNDEFINED);
  when(app.getRMAppMetrics()).thenReturn(
      new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
  return app;
}
 
Example 6
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
private void waitForAppPreemptionInfo(RMApp app, Resource preempted,
    int numAMPreempted, int numTaskPreempted,
    Resource currentAttemptPreempted, boolean currentAttemptAMPreempted,
    int numLatestAttemptTaskPreempted) throws InterruptedException {
  while (true) {
    RMAppMetrics appPM = app.getRMAppMetrics();
    RMAppAttemptMetrics attemptPM =
        app.getCurrentAppAttempt().getRMAppAttemptMetrics();

    if (appPM.getResourcePreempted().equals(preempted)
        && appPM.getNumAMContainersPreempted() == numAMPreempted
        && appPM.getNumNonAMContainersPreempted() == numTaskPreempted
        && attemptPM.getResourcePreempted().equals(currentAttemptPreempted)
        && app.getCurrentAppAttempt().getRMAppAttemptMetrics()
          .getIsPreempted() == currentAttemptAMPreempted
        && attemptPM.getNumNonAMContainersPreempted() == 
           numLatestAttemptTaskPreempted) {
      return;
    }
    Thread.sleep(500);
  }
}
 
Example 7
Source Project: hadoop   Source File: SystemMetricsPublisher.java    License: Apache License 2.0 5 votes vote down vote up
private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
  TimelineEntity entity =
      createApplicationEntity(event.getApplicationId());
  TimelineEvent tEvent = new TimelineEvent();
  tEvent.setEventType(
      ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  Map<String, Object> eventInfo = new HashMap<String, Object>();
  eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
      event.getDiagnosticsInfo());
  eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO,
      event.getFinalApplicationStatus().toString());
  eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
      event.getYarnApplicationState().toString());
  if (event.getLatestApplicationAttemptId() != null) {
    eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
        event.getLatestApplicationAttemptId().toString());
  }
  RMAppMetrics appMetrics = event.getAppMetrics();
  entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
      appMetrics.getVcoreSeconds());
  entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
      appMetrics.getMemorySeconds());
  entity.addOtherInfo(ApplicationMetricsConstants.APP_GPU_METRICS,
      appMetrics.getGcoreSeconds());

  tEvent.setEventInfo(eventInfo);
  entity.addEvent(tEvent);
  putEntity(entity);
}
 
Example 8
Source Project: hadoop   Source File: RMAppManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * create a summary of the application's runtime.
 * 
 * @param app {@link RMApp} whose summary is to be created, cannot
 *            be <code>null</code>.
 */
public static SummaryBuilder createAppSummary(RMApp app) {
  String trackingUrl = "N/A";
  String host = "N/A";
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  if (attempt != null) {
    trackingUrl = attempt.getTrackingUrl();
    host = attempt.getHost();
  }
  RMAppMetrics metrics = app.getRMAppMetrics();
  SummaryBuilder summary = new SummaryBuilder()
      .add("appId", app.getApplicationId())
      .add("name", app.getName())
      .add("user", app.getUser())
      .add("queue", app.getQueue())
      .add("state", app.getState())
      .add("trackingUrl", trackingUrl)
      .add("appMasterHost", host)
      .add("startTime", app.getStartTime())
      .add("finishTime", app.getFinishTime())
      .add("finalStatus", app.getFinalApplicationStatus())
      .add("memorySeconds", metrics.getMemorySeconds())
      .add("vcoreSeconds", metrics.getVcoreSeconds())
      .add("gcoreSeconds", metrics.getGcoreSeconds())
      .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
      .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
      .add("preemptedResources", metrics.getResourcePreempted())
      .add("applicationType", app.getApplicationType());
  return summary;
}
 
Example 9
Source Project: hadoop   Source File: TestAppManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 30000)
 public void testEscapeApplicationSummary() {
   RMApp app = mock(RMAppImpl.class);
   when(app.getApplicationId()).thenReturn(
       ApplicationId.newInstance(100L, 1));
   when(app.getName()).thenReturn("Multiline\n\n\r\rAppName");
   when(app.getUser()).thenReturn("Multiline\n\n\r\rUserName");
   when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
   when(app.getState()).thenReturn(RMAppState.RUNNING);
   when(app.getApplicationType()).thenReturn("MAPREDUCE");
   RMAppMetrics metrics =
       new RMAppMetrics(Resource.newInstance(1234, 56, 56), 10, 1, 16384, 64, 64);
   when(app.getRMAppMetrics()).thenReturn(metrics);

   RMAppManager.ApplicationSummary.SummaryBuilder summary =
       new RMAppManager.ApplicationSummary().createAppSummary(app);
   String msg = summary.toString();
   LOG.info("summary: " + msg);
   Assert.assertFalse(msg.contains("\n"));
   Assert.assertFalse(msg.contains("\r"));

   String escaped = "\\n\\n\\r\\r";
   Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
   Assert.assertTrue(msg.contains("memorySeconds=16384"));
   Assert.assertTrue(msg.contains("vcoreSeconds=64"));
   Assert.assertTrue(msg.contains("gcoreSeconds=64"));
   Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
   Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
   Assert.assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56\\, gCores:56>"));
   Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
 
Example 10
Source Project: hadoop   Source File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}
 
Example 11
Source Project: big-c   Source File: SystemMetricsPublisher.java    License: Apache License 2.0 5 votes vote down vote up
private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
  TimelineEntity entity =
      createApplicationEntity(event.getApplicationId());
  TimelineEvent tEvent = new TimelineEvent();
  tEvent.setEventType(
      ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
  tEvent.setTimestamp(event.getTimestamp());
  Map<String, Object> eventInfo = new HashMap<String, Object>();
  eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
      event.getDiagnosticsInfo());
  eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO,
      event.getFinalApplicationStatus().toString());
  eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
      event.getYarnApplicationState().toString());
  if (event.getLatestApplicationAttemptId() != null) {
    eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
        event.getLatestApplicationAttemptId().toString());
  }
  RMAppMetrics appMetrics = event.getAppMetrics();
  entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
      appMetrics.getVcoreSeconds());
  entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
      appMetrics.getMemorySeconds());
  
  tEvent.setEventInfo(eventInfo);
  entity.addEvent(tEvent);
  putEntity(entity);
}
 
Example 12
Source Project: big-c   Source File: RMAppManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * create a summary of the application's runtime.
 * 
 * @param app {@link RMApp} whose summary is to be created, cannot
 *            be <code>null</code>.
 */
public static SummaryBuilder createAppSummary(RMApp app) {
  String trackingUrl = "N/A";
  String host = "N/A";
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  if (attempt != null) {
    trackingUrl = attempt.getTrackingUrl();
    host = attempt.getHost();
  }
  RMAppMetrics metrics = app.getRMAppMetrics();
  SummaryBuilder summary = new SummaryBuilder()
      .add("appId", app.getApplicationId())
      .add("name", app.getName())
      .add("user", app.getUser())
      .add("queue", app.getQueue())
      .add("state", app.getState())
      .add("trackingUrl", trackingUrl)
      .add("appMasterHost", host)
      .add("startTime", app.getStartTime())
      .add("finishTime", app.getFinishTime())
      .add("finalStatus", app.getFinalApplicationStatus())
      .add("memorySeconds", metrics.getMemorySeconds())
      .add("vcoreSeconds", metrics.getVcoreSeconds())
      .add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
      .add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
      .add("preemptedResources", metrics.getResourcePreempted())
      .add("applicationType", app.getApplicationType());
  return summary;
}
 
Example 13
Source Project: big-c   Source File: TestAppManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 30000)
 public void testEscapeApplicationSummary() {
   RMApp app = mock(RMAppImpl.class);
   when(app.getApplicationId()).thenReturn(
       ApplicationId.newInstance(100L, 1));
   when(app.getName()).thenReturn("Multiline\n\n\r\rAppName");
   when(app.getUser()).thenReturn("Multiline\n\n\r\rUserName");
   when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
   when(app.getState()).thenReturn(RMAppState.RUNNING);
   when(app.getApplicationType()).thenReturn("MAPREDUCE");
   RMAppMetrics metrics =
       new RMAppMetrics(Resource.newInstance(1234, 56), 10, 1, 16384, 64);
   when(app.getRMAppMetrics()).thenReturn(metrics);

   RMAppManager.ApplicationSummary.SummaryBuilder summary =
       new RMAppManager.ApplicationSummary().createAppSummary(app);
   String msg = summary.toString();
   LOG.info("summary: " + msg);
   Assert.assertFalse(msg.contains("\n"));
   Assert.assertFalse(msg.contains("\r"));

   String escaped = "\\n\\n\\r\\r";
   Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
   Assert.assertTrue(msg.contains("memorySeconds=16384"));
   Assert.assertTrue(msg.contains("vcoreSeconds=64"));
   Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
   Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
   Assert.assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
   Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
 
Example 14
Source Project: big-c   Source File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}
 
Example 15
Source Project: garmadon   Source File: RMContextImplEventRunnable.java    License: Apache License 2.0 4 votes vote down vote up
public void sendAppEvent(ApplicationId applicationId, RMApp rmApp) {
    if (cacheFinishedApp.getIfPresent(applicationId.toString()) == null) {
        Header.Builder headerBuilder = Header.newBuilder()
            .withId(applicationId.toString())
            .withApplicationID(applicationId.toString())
            .withUser(rmApp.getUser())
            .withApplicationName(rmApp.getName())
            .withFramework(rmApp.getApplicationType().toUpperCase());

        ApplicationEvent.Builder eventBuilder = ApplicationEvent.newBuilder()
            .setState(rmApp.getState().name())
            .setQueue(rmApp.getQueue());

        rmApp.getApplicationTags().stream()
            .filter(tag -> YARN_TAGS_TO_EXTRACT.stream().noneMatch(tag::startsWith) && !tag.contains(":"))
            .forEach(eventBuilder::addYarnTags);

        rmApp.getApplicationTags().stream()
            .filter(tag -> tag.contains(":") && YARN_TAGS_TO_EXTRACT.stream().anyMatch(tag::startsWith))
            .map(tag -> {
                int idx = tag.indexOf(':');
                String key = tag.substring(0, idx);
                String value = tag.substring(idx + 1);
                return new String[] {key, value};
            })
            .forEach(splitTag -> BUILDERS.get(splitTag[0]).accept(splitTag[1], eventBuilder));

        eventBuilder.setFinalStatus(rmApp.getFinalApplicationStatus().name());

        eventBuilder.setStartTime(rmApp.getStartTime());
        eventBuilder.setFinishTime(rmApp.getFinishTime());

        RMAppMetrics rmAppMetrics = rmApp.getRMAppMetrics();
        if (rmAppMetrics != null) {
            eventBuilder.setMemorySeconds(rmAppMetrics.getMemorySeconds());
            eventBuilder.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
        }

        RMAppAttempt rmAppAttempt = rmApp.getCurrentAppAttempt();
        if (rmAppAttempt != null) {
            headerBuilder.withAttemptID(rmAppAttempt.getAppAttemptId().toString());

            Container container = rmAppAttempt.getMasterContainer();
            if (container != null) {
                eventBuilder.setAmContainerId(container.getId().toString());
            }
        }

        if (rmApp.getTrackingUrl() != null) {
            eventBuilder.setTrackingUrl(normalizeTrackingUrl(rmApp.getTrackingUrl()));
        }

        if (rmApp.getOriginalTrackingUrl() != null && !"N/A".equals(rmApp.getOriginalTrackingUrl())) {
            eventBuilder.setOriginalTrackingUrl(normalizeTrackingUrl(rmApp.getOriginalTrackingUrl()));
        }

        eventHandler.accept(System.currentTimeMillis(), headerBuilder.build(), eventBuilder.build());

        if (rmApp.getState() == RMAppState.FINISHED || rmApp.getState() == RMAppState.KILLED || rmApp.getState() == RMAppState.FAILED) {
            cacheFinishedApp.put(applicationId.toString(), rmApp.getState().name());
        }
    }
}
 
Example 16
Source Project: hadoop   Source File: AppInfo.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          allocatedGCores = usedResources.getGpuCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    preemptedResourceGCores =
        appMetrics.getResourcePreempted().getGpuCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
    gcoreSeconds = appMetrics.getGcoreSeconds();
  }
}
 
Example 17
Source Project: hadoop   Source File: RMAppBlock.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void createApplicationMetricsTable(Block html){
  RMApp rmApp = this.rm.getRMContext().getRMApps().get(appID);
  RMAppMetrics appMetrics = rmApp == null ? null : rmApp.getRMAppMetrics();
  // Get attempt metrics and fields, it is possible currentAttempt of RMApp is
  // null. In that case, we will assume resource preempted and number of Non
  // AM container preempted on that attempt is 0
  RMAppAttemptMetrics attemptMetrics;
  if (rmApp == null || null == rmApp.getCurrentAppAttempt()) {
    attemptMetrics = null;
  } else {
    attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
  }
  Resource attemptResourcePreempted =
      attemptMetrics == null ? Resources.none() : attemptMetrics
        .getResourcePreempted();
  int attemptNumNonAMContainerPreempted =
      attemptMetrics == null ? 0 : attemptMetrics
        .getNumNonAMContainersPreempted();
  DIV<Hamlet> pdiv = html.
      _(InfoBlock.class).
      div(_INFO_WRAP);
  info("Application Overview").clear();
  info("Application Metrics")
      ._("Total Resource Preempted:",
        appMetrics == null ? "N/A" : appMetrics.getResourcePreempted())
      ._("Total Number of Non-AM Containers Preempted:",
        appMetrics == null ? "N/A"
            : appMetrics.getNumNonAMContainersPreempted())
      ._("Total Number of AM Containers Preempted:",
        appMetrics == null ? "N/A"
            : appMetrics.getNumAMContainersPreempted())
      ._("Resource Preempted from Current Attempt:",
        attemptResourcePreempted)
      ._("Number of Non-AM Containers Preempted from Current Attempt:",
        attemptNumNonAMContainerPreempted)
      ._("Aggregate Resource Allocation:",
        String.format("%d MB-seconds, %d vcore-seconds, %d gcore-seconds",
            appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(),
            appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds(),
            appMetrics == null ? "N/A" : appMetrics.getGcoreSeconds()));
  pdiv._();
}
 
Example 18
Source Project: hadoop   Source File: ApplicationFinishedEvent.java    License: Apache License 2.0 4 votes vote down vote up
public RMAppMetrics getAppMetrics() {
  return appMetrics;
}
 
Example 19
Source Project: hadoop   Source File: TestAppPage.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception {
  final ApplicationId APP_ID = ApplicationId.newInstance(1234L, 0);
  Injector injector;
  
  // init app
  RMApp app = mock(RMApp.class);
  when(app.getTrackingUrl()).thenReturn("http://host:123");
  when(app.getState()).thenReturn(RMAppState.FAILED);
  when(app.getApplicationId()).thenReturn(APP_ID);
  when(app.getApplicationType()).thenReturn("Type");
  when(app.getUser()).thenReturn("user");
  when(app.getName()).thenReturn("Name");
  when(app.getQueue()).thenReturn("queue");
  when(app.getDiagnostics()).thenReturn(new StringBuilder());
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
  when(app.getStartTime()).thenReturn(0L);
  when(app.getFinishTime()).thenReturn(0L);
  when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
  
  RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
  when(app.getRMAppMetrics()).thenReturn(appMetrics);
  
  // initialize RM Context, and create RMApp, without creating RMAppAttempt
  final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8);
  rmContext.getRMApps().put(APP_ID, app);
  
  injector =
      WebAppTests.createMockInjector(RMContext.class, rmContext,
          new Module() {
            @Override
            public void configure(Binder binder) {
              try {
                ResourceManager rm = TestRMWebApp.mockRm(rmContext);
                binder.bind(ResourceManager.class).toInstance(rm);
                binder.bind(ApplicationBaseProtocol.class).toInstance(
                  rm.getClientRMService());
              } catch (IOException e) {
                throw new IllegalStateException(e);
              }
            }
          });
  
  AppBlock instance = injector.getInstance(AppBlock.class);
  instance.set(YarnWebParams.APPLICATION_ID, APP_ID.toString());
  instance.render();
}
 
Example 20
Source Project: hadoop   Source File: TestContainerResourceUsage.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout = 120000)
public void testUsageWithOneAttemptAndOneContainer() throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();
  
  MockNM nm =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm.registerNode();

  RMApp app0 = rm.submitApp(200);

  RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "Before app submittion, memory seconds should have been 0 but was "
                        + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() == 0);
  Assert.assertTrue(
      "Before app submission, vcore seconds should have been 0 but was "
                        + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() == 0);

  Assert.assertTrue(
      "Before app submission, gcore seconds should have been 0 but was "
                        + rmAppMetrics.getGcoreSeconds(),
      rmAppMetrics.getGcoreSeconds() == 0);

  RMAppAttempt attempt0 = app0.getCurrentAppAttempt();

  nm.nodeHeartbeat(true);
  MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId());
  am0.registerAppAttempt();

  RMContainer rmContainer =
      rm.getResourceScheduler()
         .getRMContainer(attempt0.getMasterContainer().getId());

  // Allow metrics to accumulate.
  int sleepInterval = 1000;
  int cumulativeSleepTime = 0;
  while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
    Thread.sleep(sleepInterval);
    cumulativeSleepTime += sleepInterval;
  }

  rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "While app is running, memory seconds should be >0 but is "
          + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() > 0);
  Assert.assertTrue(
      "While app is running, vcore seconds should be >0 but is "
          + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() > 0);

  MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);

  AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
  rmAppMetrics = app0.getRMAppMetrics();

  Assert.assertEquals("Unexcpected MemorySeconds value",
      ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
  Assert.assertEquals("Unexpected VcoreSeconds value",
      ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
  Assert.assertEquals("Unexpected GcoreSeconds value",
      ru.getGcoreSeconds(), rmAppMetrics.getGcoreSeconds());
  rm.stop();
}
 
Example 21
Source Project: hadoop   Source File: MockAsm.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public RMAppMetrics getRMAppMetrics() {
  return new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
}
 
Example 22
Source Project: big-c   Source File: AppInfo.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
  }
}
 
Example 23
Source Project: big-c   Source File: RMAppBlock.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void createApplicationMetricsTable(Block html){
  RMApp rmApp = this.rm.getRMContext().getRMApps().get(appID);
  RMAppMetrics appMetrics = rmApp == null ? null : rmApp.getRMAppMetrics();
  // Get attempt metrics and fields, it is possible currentAttempt of RMApp is
  // null. In that case, we will assume resource preempted and number of Non
  // AM container preempted on that attempt is 0
  RMAppAttemptMetrics attemptMetrics;
  if (rmApp == null || null == rmApp.getCurrentAppAttempt()) {
    attemptMetrics = null;
  } else {
    attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
  }
  Resource attemptResourcePreempted =
      attemptMetrics == null ? Resources.none() : attemptMetrics
        .getResourcePreempted();
  int attemptNumNonAMContainerPreempted =
      attemptMetrics == null ? 0 : attemptMetrics
        .getNumNonAMContainersPreempted();
  DIV<Hamlet> pdiv = html.
      _(InfoBlock.class).
      div(_INFO_WRAP);
  info("Application Overview").clear();
  info("Application Metrics")
      ._("Total Resource Preempted:",
        appMetrics == null ? "N/A" : appMetrics.getResourcePreempted())
      ._("Total Number of Non-AM Containers Preempted:",
        appMetrics == null ? "N/A"
            : appMetrics.getNumNonAMContainersPreempted())
      ._("Total Number of AM Containers Preempted:",
        appMetrics == null ? "N/A"
            : appMetrics.getNumAMContainersPreempted())
      ._("Resource Preempted from Current Attempt:",
        attemptResourcePreempted)
      ._("Number of Non-AM Containers Preempted from Current Attempt:",
        attemptNumNonAMContainerPreempted)
      ._("Aggregate Resource Allocation:",
        String.format("%d MB-seconds, %d vcore-seconds",
            appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(),
            appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()));
  pdiv._();
}
 
Example 24
Source Project: big-c   Source File: ApplicationFinishedEvent.java    License: Apache License 2.0 4 votes vote down vote up
public RMAppMetrics getAppMetrics() {
  return appMetrics;
}
 
Example 25
Source Project: big-c   Source File: TestAppPage.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testAppBlockRenderWithNullCurrentAppAttempt() throws Exception {
  final ApplicationId APP_ID = ApplicationId.newInstance(1234L, 0);
  Injector injector;
  
  // init app
  RMApp app = mock(RMApp.class);
  when(app.getTrackingUrl()).thenReturn("http://host:123");
  when(app.getState()).thenReturn(RMAppState.FAILED);
  when(app.getApplicationId()).thenReturn(APP_ID);
  when(app.getApplicationType()).thenReturn("Type");
  when(app.getUser()).thenReturn("user");
  when(app.getName()).thenReturn("Name");
  when(app.getQueue()).thenReturn("queue");
  when(app.getDiagnostics()).thenReturn(new StringBuilder());
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.FAILED);
  when(app.getStartTime()).thenReturn(0L);
  when(app.getFinishTime()).thenReturn(0L);
  when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
  
  RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
  when(app.getRMAppMetrics()).thenReturn(appMetrics);
  
  // initialize RM Context, and create RMApp, without creating RMAppAttempt
  final RMContext rmContext = TestRMWebApp.mockRMContext(15, 1, 2, 8);
  rmContext.getRMApps().put(APP_ID, app);
  
  injector =
      WebAppTests.createMockInjector(RMContext.class, rmContext,
          new Module() {
            @Override
            public void configure(Binder binder) {
              try {
                ResourceManager rm = TestRMWebApp.mockRm(rmContext);
                binder.bind(ResourceManager.class).toInstance(rm);
                binder.bind(ApplicationBaseProtocol.class).toInstance(
                  rm.getClientRMService());
              } catch (IOException e) {
                throw new IllegalStateException(e);
              }
            }
          });
  
  AppBlock instance = injector.getInstance(AppBlock.class);
  instance.set(YarnWebParams.APPLICATION_ID, APP_ID.toString());
  instance.render();
}
 
Example 26
Source Project: big-c   Source File: TestContainerResourceUsage.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout = 120000)
public void testUsageWithOneAttemptAndOneContainer() throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();
  
  MockNM nm =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm.registerNode();

  RMApp app0 = rm.submitApp(200);

  RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "Before app submittion, memory seconds should have been 0 but was "
                        + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() == 0);
  Assert.assertTrue(
      "Before app submission, vcore seconds should have been 0 but was "
                        + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() == 0);

  RMAppAttempt attempt0 = app0.getCurrentAppAttempt();

  nm.nodeHeartbeat(true);
  MockAM am0 = rm.sendAMLaunched(attempt0.getAppAttemptId());
  am0.registerAppAttempt();

  RMContainer rmContainer =
      rm.getResourceScheduler()
         .getRMContainer(attempt0.getMasterContainer().getId());

  // Allow metrics to accumulate.
  int sleepInterval = 1000;
  int cumulativeSleepTime = 0;
  while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
    Thread.sleep(sleepInterval);
    cumulativeSleepTime += sleepInterval;
  }

  rmAppMetrics = app0.getRMAppMetrics();
  Assert.assertTrue(
      "While app is running, memory seconds should be >0 but is "
          + rmAppMetrics.getMemorySeconds(),
      rmAppMetrics.getMemorySeconds() > 0);
  Assert.assertTrue(
      "While app is running, vcore seconds should be >0 but is "
          + rmAppMetrics.getVcoreSeconds(),
      rmAppMetrics.getVcoreSeconds() > 0);

  MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);

  AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
  rmAppMetrics = app0.getRMAppMetrics();

  Assert.assertEquals("Unexcpected MemorySeconds value",
      ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
  Assert.assertEquals("Unexpected VcoreSeconds value",
      ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());

  rm.stop();
}
 
Example 27
Source Project: big-c   Source File: MockAsm.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public RMAppMetrics getRMAppMetrics() {
  return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
}
 
Example 28
Source Project: incubator-myriad   Source File: MockRMApp.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public RMAppMetrics getRMAppMetrics() {
  return null;
}