org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AppReportConverter.java    From cloudbreak with Apache License 2.0 6 votes vote down vote up
@Override
public AppReportJson convert(ApplicationReport source) {
    AppReportJson json = new AppReportJson();
    json.setAppId(source.getApplicationId().toString());
    json.setStart(source.getStartTime());
    json.setFinish(source.getFinishTime());
    json.setProgress(source.getProgress());
    json.setQueue(source.getQueue());
    json.setUrl(source.getTrackingUrl());
    json.setUser(source.getUser());
    json.setState(source.getYarnApplicationState().name());
    ApplicationResourceUsageReport usageReport = source.getApplicationResourceUsageReport();
    json.setReservedContainers(usageReport.getNumReservedContainers());
    json.setUsedContainers(usageReport.getNumUsedContainers());
    json.setUsedMemory(usageReport.getUsedResources().getMemory());
    json.setUsedVCores(usageReport.getUsedResources().getVirtualCores());
    return json;
}
 
Example #2
Source File: RMAppAttemptImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
  this.readLock.lock();
  try {
    ApplicationResourceUsageReport report =
        scheduler.getAppResourceUsageReport(this.getAppAttemptId());
    if (report == null) {
      report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
    }
    AggregateAppResourceUsage resUsage =
        this.attemptMetrics.getAggregateAppResourceUsage();
    report.setMemorySeconds(resUsage.getMemorySeconds());
    report.setVcoreSeconds(resUsage.getVcoreSeconds());
    return report;
  } finally {
    this.readLock.unlock();
  }
}
 
Example #3
Source File: TestApplicationACLs.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void verifyEnemyAppReport(ApplicationReport appReport) {
  Assert.assertEquals("Enemy should not see app host!",
      UNAVAILABLE, appReport.getHost());
  Assert.assertEquals("Enemy should not see app rpc port!",
      -1, appReport.getRpcPort());
  Assert.assertEquals("Enemy should not see app client token!",
      null, appReport.getClientToAMToken());
  Assert.assertEquals("Enemy should not see app diagnostics!",
      UNAVAILABLE, appReport.getDiagnostics());
  Assert.assertEquals("Enemy should not see app tracking url!",
      UNAVAILABLE, appReport.getTrackingUrl());
  Assert.assertEquals("Enemy should not see app original tracking url!",
      UNAVAILABLE, appReport.getOriginalTrackingUrl());
  ApplicationResourceUsageReport usageReport =
      appReport.getApplicationResourceUsageReport();
  Assert.assertEquals("Enemy should not see app used containers",
      -1, usageReport.getNumUsedContainers());
  Assert.assertEquals("Enemy should not see app reserved containers",
      -1, usageReport.getNumReservedContainers());
  Assert.assertEquals("Enemy should not see app used resources",
      -1, usageReport.getUsedResources().getMemory());
  Assert.assertEquals("Enemy should not see app reserved resources",
      -1, usageReport.getReservedResources().getMemory());
  Assert.assertEquals("Enemy should not see app needed resources",
      -1, usageReport.getNeededResources().getMemory());
}
 
Example #4
Source File: RMAppAttemptMetrics.java    From big-c with Apache License 2.0 6 votes vote down vote up
public AggregateAppResourceUsage getAggregateAppResourceUsage() {
  long memorySeconds = finishedMemorySeconds.get();
  long vcoreSeconds = finishedVcoreSeconds.get();

  // Only add in the running containers if this is the active attempt.
  RMAppAttempt currentAttempt = rmContext.getRMApps()
                 .get(attemptId.getApplicationId()).getCurrentAppAttempt();
  if (currentAttempt.getAppAttemptId().equals(attemptId)) {
    ApplicationResourceUsageReport appResUsageReport = rmContext
          .getScheduler().getAppResourceUsageReport(attemptId);
    if (appResUsageReport != null) {
      memorySeconds += appResUsageReport.getMemorySeconds();
      vcoreSeconds += appResUsageReport.getVcoreSeconds();
    }
  }
  return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds);
}
 
Example #5
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetApplicationResourceUsageReportDummy() throws YarnException,
    IOException {
  ApplicationAttemptId attemptId = getApplicationAttemptId(1);
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });
  ApplicationSubmissionContext asContext = 
      mock(ApplicationSubmissionContext.class);
  YarnConfiguration config = new YarnConfiguration();
  RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId,
      rmContext, yarnScheduler, null, asContext, config, false, null);
  ApplicationResourceUsageReport report = rmAppAttemptImpl
      .getApplicationResourceUsageReport();
  assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
}
 
Example #6
Source File: TestApplicationACLs.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void verifyEnemyAppReport(ApplicationReport appReport) {
  Assert.assertEquals("Enemy should not see app host!",
      UNAVAILABLE, appReport.getHost());
  Assert.assertEquals("Enemy should not see app rpc port!",
      -1, appReport.getRpcPort());
  Assert.assertEquals("Enemy should not see app client token!",
      null, appReport.getClientToAMToken());
  Assert.assertEquals("Enemy should not see app diagnostics!",
      UNAVAILABLE, appReport.getDiagnostics());
  Assert.assertEquals("Enemy should not see app tracking url!",
      UNAVAILABLE, appReport.getTrackingUrl());
  Assert.assertEquals("Enemy should not see app original tracking url!",
      UNAVAILABLE, appReport.getOriginalTrackingUrl());
  ApplicationResourceUsageReport usageReport =
      appReport.getApplicationResourceUsageReport();
  Assert.assertEquals("Enemy should not see app used containers",
      -1, usageReport.getNumUsedContainers());
  Assert.assertEquals("Enemy should not see app reserved containers",
      -1, usageReport.getNumReservedContainers());
  Assert.assertEquals("Enemy should not see app used resources",
      -1, usageReport.getUsedResources().getMemory());
  Assert.assertEquals("Enemy should not see app reserved resources",
      -1, usageReport.getReservedResources().getMemory());
  Assert.assertEquals("Enemy should not see app needed resources",
      -1, usageReport.getNeededResources().getMemory());
}
 
Example #7
Source File: RMAppAttemptMetrics.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public AggregateAppResourceUsage getAggregateAppResourceUsage() {
  long memorySeconds = finishedMemorySeconds.get();
  long vcoreSeconds = finishedVcoreSeconds.get();
  long gcoreSeconds = finishedGcoreSeconds.get();

  // Only add in the running containers if this is the active attempt.
  RMAppAttempt currentAttempt = rmContext.getRMApps()
                 .get(attemptId.getApplicationId()).getCurrentAppAttempt();
  if (currentAttempt.getAppAttemptId().equals(attemptId)) {
    ApplicationResourceUsageReport appResUsageReport = rmContext
          .getScheduler().getAppResourceUsageReport(attemptId);
    if (appResUsageReport != null) {
      memorySeconds += appResUsageReport.getMemorySeconds();
      vcoreSeconds += appResUsageReport.getVcoreSeconds();
      gcoreSeconds += appResUsageReport.getGcoreSeconds();
    }
  }
  return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds, gcoreSeconds);
}
 
Example #8
Source File: RMAppAttemptImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
  this.readLock.lock();
  try {
    ApplicationResourceUsageReport report =
        scheduler.getAppResourceUsageReport(this.getAppAttemptId());
    if (report == null) {
      report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
    }
    AggregateAppResourceUsage resUsage =
        this.attemptMetrics.getAggregateAppResourceUsage();
    report.setMemorySeconds(resUsage.getMemorySeconds());
    report.setVcoreSeconds(resUsage.getVcoreSeconds());
    report.setGcoreSeconds(resUsage.getGcoreSeconds());
    return report;
  } finally {
    this.readLock.unlock();
  }
}
 
Example #9
Source File: Utils.java    From AthenaX with Apache License 2.0 6 votes vote down vote up
static InstanceInfo extractInstanceInfo(String clusterName, ApplicationReport report) {
  InstanceMetadata md = getMetadata(report.getApplicationTags());
  if (md == null) {
    return null;
  }

  ApplicationResourceUsageReport usage = report.getApplicationResourceUsageReport();
  InstanceStatus stat = new InstanceStatus()
      .allocatedVCores((long) usage.getUsedResources().getVirtualCores())
      .allocatedMB((long) usage.getUsedResources().getMemory())
      .clusterId(clusterName)
      .applicationId(report.getApplicationId().toString())
      .startedTime(report.getStartTime())
      .runningContainers((long) usage.getNumUsedContainers())
      .trackingUrl(report.getTrackingUrl())
      .state(InstanceStatus.StateEnum.fromValue(report.getYarnApplicationState().toString()));
  return new InstanceInfo(clusterName, report.getApplicationId(), md, stat);
}
 
Example #10
Source File: BuilderUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
    int numUsedContainers, int numReservedContainers, Resource usedResources,
    Resource reservedResources, Resource neededResources, long memorySeconds, 
    long vcoreSeconds, long gcoreSeconds) {
  ApplicationResourceUsageReport report =
      recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
  report.setNumUsedContainers(numUsedContainers);
  report.setNumReservedContainers(numReservedContainers);
  report.setUsedResources(usedResources);
  report.setReservedResources(reservedResources);
  report.setNeededResources(neededResources);
  report.setMemorySeconds(memorySeconds);
  report.setVcoreSeconds(vcoreSeconds);
  report.setGcoreSeconds(gcoreSeconds);
  return report;
}
 
Example #11
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetApplicationResourceUsageReportDummy() throws YarnException,
    IOException {
  ApplicationAttemptId attemptId = getApplicationAttemptId(1);
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });
  ApplicationSubmissionContext asContext = 
      mock(ApplicationSubmissionContext.class);
  YarnConfiguration config = new YarnConfiguration();
  RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId,
      rmContext, yarnScheduler, null, asContext, config, false, null);
  ApplicationResourceUsageReport report = rmAppAttemptImpl
      .getApplicationResourceUsageReport();
  assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
}
 
Example #12
Source File: TestResourceMgrDelegate.java    From big-c with Apache License 2.0 6 votes vote down vote up
private ApplicationReport getApplicationReport(
    YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  ApplicationReport appReport = Mockito.mock(ApplicationReport.class);
  ApplicationResourceUsageReport appResources = Mockito
      .mock(ApplicationResourceUsageReport.class);
  Mockito.when(appReport.getApplicationId()).thenReturn(
      ApplicationId.newInstance(0, 0));
  Mockito.when(appResources.getNeededResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getReservedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getUsedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
      appResources);
  Mockito.when(appReport.getYarnApplicationState()).thenReturn(
      yarnApplicationState);
  Mockito.when(appReport.getFinalApplicationStatus()).thenReturn(
      finalApplicationStatus);

  return appReport;
}
 
Example #13
Source File: TestResourceMgrDelegate.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private ApplicationReport getApplicationReport(
    YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  ApplicationReport appReport = Mockito.mock(ApplicationReport.class);
  ApplicationResourceUsageReport appResources = Mockito
      .mock(ApplicationResourceUsageReport.class);
  Mockito.when(appReport.getApplicationId()).thenReturn(
      ApplicationId.newInstance(0, 0));
  Mockito.when(appResources.getNeededResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getReservedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getUsedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
      appResources);
  Mockito.when(appReport.getYarnApplicationState()).thenReturn(
      yarnApplicationState);
  Mockito.when(appReport.getFinalApplicationStatus()).thenReturn(
      finalApplicationStatus);

  return appReport;
}
 
Example #14
Source File: ClusterAnalysisService.java    From jumbune with GNU Lesser General Public License v3.0 6 votes vote down vote up
/**
 * It fetches used vcore and used containers for running applications
 *  from hadoop (resource manager) and persists into influxdb
 * @param cluster
 * @throws Exception
 */
private void persistRunningAppsDataToInfluxdb(Cluster cluster) throws Exception {
	
	List<ApplicationReport> list = sessionUtils.getRM(cluster, getSession()).getRunningApplications();
	
	InfluxDataWriter writer = new InfluxDataWriter(
			AdminConfigurationUtil.getInfluxdbConfiguration(cluster.getClusterName()));
	writer.setTableName(WebConstants.JOB_HISTORY_DETAILS_TABLE);
	writer.setTimeUnit(TimeUnit.SECONDS);
	
	ApplicationResourceUsageReport usage;
	
	for (ApplicationReport report: list) {
		usage = report.getApplicationResourceUsageReport();
		
		writer.addColumn(WebConstants.USED_V_CORES, usage.getUsedResources().getVirtualCores());
		
		writer.addColumn(WebConstants.USED_CONTAINERS, usage.getNumUsedContainers());
		
		writer.addTag(WebConstants.JOB_ID, report.getApplicationId()
				.toString().replace(WebConstants.APPLICATION, WebConstants.JOB));
		writer.writeData();
	}
}
 
Example #15
Source File: TestTypeConverter.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testFromYarn() throws Exception {
  int appStartTime = 612354;
  int appFinishTime = 612355;
  YarnApplicationState state = YarnApplicationState.RUNNING;
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationReport applicationReport = Records
      .newRecord(ApplicationReport.class);
  applicationReport.setApplicationId(applicationId);
  applicationReport.setYarnApplicationState(state);
  applicationReport.setStartTime(appStartTime);
  applicationReport.setFinishTime(appFinishTime);
  applicationReport.setUser("TestTypeConverter-user");
  ApplicationResourceUsageReport appUsageRpt = Records
      .newRecord(ApplicationResourceUsageReport.class);
  Resource r = Records.newRecord(Resource.class);
  r.setMemory(2048);
  appUsageRpt.setNeededResources(r);
  appUsageRpt.setNumReservedContainers(1);
  appUsageRpt.setNumUsedContainers(3);
  appUsageRpt.setReservedResources(r);
  appUsageRpt.setUsedResources(r);
  applicationReport.setApplicationResourceUsageReport(appUsageRpt);
  JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
  Assert.assertEquals(appStartTime, jobStatus.getStartTime());
  Assert.assertEquals(appFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
 
Example #16
Source File: TypeConverter.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static JobStatus fromYarn(ApplicationReport application,
    String jobFile) {
  String trackingUrl = application.getTrackingUrl();
  trackingUrl = trackingUrl == null ? "" : trackingUrl;
  JobStatus jobStatus =
    new JobStatus(
        TypeConverter.fromYarn(application.getApplicationId()),
        0.0f, 0.0f, 0.0f, 0.0f,
        TypeConverter.fromYarn(application.getYarnApplicationState(), application.getFinalApplicationStatus()),
        org.apache.hadoop.mapreduce.JobPriority.NORMAL,
        application.getUser(), application.getName(),
        application.getQueue(), jobFile, trackingUrl, false
    );
  jobStatus.setSchedulingInfo(trackingUrl); // Set AM tracking url
  jobStatus.setStartTime(application.getStartTime());
  jobStatus.setFinishTime(application.getFinishTime());
  jobStatus.setFailureInfo(application.getDiagnostics());
  ApplicationResourceUsageReport resourceUsageReport =
      application.getApplicationResourceUsageReport();
  if (resourceUsageReport != null) {
    jobStatus.setNeededMem(
        resourceUsageReport.getNeededResources().getMemory());
    jobStatus.setNumReservedSlots(
        resourceUsageReport.getNumReservedContainers());
    jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
    jobStatus.setReservedMem(
        resourceUsageReport.getReservedResources().getMemory());
    jobStatus.setUsedMem(resourceUsageReport.getUsedResources().getMemory());
  }
  return jobStatus;
}
 
Example #17
Source File: ApplicationReportPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void setApplicationResourceUsageReport(ApplicationResourceUsageReport appInfo) {
  maybeInitBuilder();
  if (appInfo == null) {
    builder.clearAppResourceUsage();
    return;
  }
  builder.setAppResourceUsage(convertToProtoFormat(appInfo));
}
 
Example #18
Source File: ApplicationReportPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
  ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasAppResourceUsage()) {
    return null;
  }
  return convertFromProtoFormat(p.getAppResourceUsage());
}
 
Example #19
Source File: TwillTester.java    From twill with Apache License 2.0 5 votes vote down vote up
public ApplicationResourceUsageReport getApplicationResourceReport(String appId) throws Exception {
  List<String> splits = Lists.newArrayList(Splitter.on('_').split(appId));
  Preconditions.checkArgument(splits.size() == 3, "Invalid application id - " + appId);
  ApplicationId applicationId = ApplicationId.newInstance(Long.parseLong(splits.get(1)),
                                                          Integer.parseInt(splits.get(2)));

  ClientRMService clientRMService = cluster.getResourceManager().getClientRMService();
  GetApplicationReportRequest request = Records.newRecord(GetApplicationReportRequest.class);
  request.setApplicationId(applicationId);
  return clientRMService.getApplicationReport(request)
    .getApplicationReport().getApplicationResourceUsageReport();
}
 
Example #20
Source File: SchedulerApplicationAttempt.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public synchronized ApplicationResourceUsageReport getResourceUsageReport() {
  AggregateAppResourceUsage runningResourceUsage =
      getRunningAggregateAppResourceUsage();
  Resource usedResourceClone =
      Resources.clone(attemptResourceUsage.getUsed());
  Resource reservedResourceClone =
      Resources.clone(attemptResourceUsage.getReserved());
  return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
      reservedContainers.size(), usedResourceClone, reservedResourceClone,
      Resources.add(usedResourceClone, reservedResourceClone),
      runningResourceUsage.getMemorySeconds(),
      runningResourceUsage.getVcoreSeconds(),
      runningResourceUsage.getGcoreSeconds());
}
 
Example #21
Source File: BuilderUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static ApplicationReport newApplicationReport(
    ApplicationId applicationId, ApplicationAttemptId applicationAttemptId,
    String user, String queue, String name, String host, int rpcPort,
    Token clientToAMToken, YarnApplicationState state, String diagnostics,
    String url, long startTime, long finishTime,
    FinalApplicationStatus finalStatus,
    ApplicationResourceUsageReport appResources, String origTrackingUrl,
    float progress, String appType, Token amRmToken, Set<String> tags) {
  ApplicationReport report = recordFactory
      .newRecordInstance(ApplicationReport.class);
  report.setApplicationId(applicationId);
  report.setCurrentApplicationAttemptId(applicationAttemptId);
  report.setUser(user);
  report.setQueue(queue);
  report.setName(name);
  report.setHost(host);
  report.setRpcPort(rpcPort);
  report.setClientToAMToken(clientToAMToken);
  report.setYarnApplicationState(state);
  report.setDiagnostics(diagnostics);
  report.setTrackingUrl(url);
  report.setStartTime(startTime);
  report.setFinishTime(finishTime);
  report.setFinalApplicationStatus(finalStatus);
  report.setApplicationResourceUsageReport(appResources);
  report.setOriginalTrackingUrl(origTrackingUrl);
  report.setProgress(progress);
  report.setApplicationType(appType);
  report.setAMRMToken(amRmToken);
  report.setApplicationTags(tags);
  return report;
}
 
Example #22
Source File: BuilderUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
    int numUsedContainers, int numReservedContainers, Resource usedResources,
    Resource reservedResources, Resource neededResources, long memorySeconds, 
    long vcoreSeconds) {
  ApplicationResourceUsageReport report =
      recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
  report.setNumUsedContainers(numUsedContainers);
  report.setNumReservedContainers(numReservedContainers);
  report.setUsedResources(usedResources);
  report.setReservedResources(reservedResources);
  report.setNeededResources(neededResources);
  report.setMemorySeconds(memorySeconds);
  report.setVcoreSeconds(vcoreSeconds);
  return report;
}
 
Example #23
Source File: SchedulerApplicationAttempt.java    From big-c with Apache License 2.0 5 votes vote down vote up
public synchronized ApplicationResourceUsageReport getResourceUsageReport() {
  AggregateAppResourceUsage resUsage = getRunningAggregateAppResourceUsage();
  return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
             reservedContainers.size(), Resources.clone(currentConsumption),
             Resources.clone(currentReservation),
             Resources.add(currentConsumption, currentReservation),
             resUsage.getMemorySeconds(), resUsage.getVcoreSeconds());
}
 
Example #24
Source File: AbstractYarnScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public ApplicationResourceUsageReport getAppResourceUsageReport(
    ApplicationAttemptId appAttemptId) {
  SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId);
  if (attempt == null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Request for appInfo of unknown attempt " + appAttemptId);
    }
    return null;
  }
  return attempt.getResourceUsageReport();
}
 
Example #25
Source File: TestRMWebApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  ClientRMService clientRMService = mock(ClientRMService.class);
  List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
  for (RMApp app : rmContext.getRMApps().values()) {
    ApplicationReport appReport =
        ApplicationReport.newInstance(
            app.getApplicationId(), (ApplicationAttemptId) null,
            app.getUser(), app.getQueue(),
            app.getName(), (String) null, 0, (Token) null,
            app.createApplicationState(),
            app.getDiagnostics().toString(), (String) null,
            app.getStartTime(), app.getFinishTime(),
            app.getFinalApplicationStatus(),
            (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
            app.getProgress(), app.getApplicationType(), (Token) null);
    appReports.add(appReport);
  }
  GetApplicationsResponse response = mock(GetApplicationsResponse.class);
  when(response.getApplicationList()).thenReturn(appReports);
  try {
    when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
        .thenReturn(response);
  } catch (YarnException e) {
    Assert.fail("Exception is not expteced.");
  }
  return clientRMService;
}
 
Example #26
Source File: TestRMAppAttemptTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testUsageReport() {
  // scheduler has info on running apps
  ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
  ApplicationResourceUsageReport appResUsgRpt =
          mock(ApplicationResourceUsageReport.class);
  when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
  when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class)))
  .thenReturn(appResUsgRpt);

  // start and finish the attempt
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(attemptId,
      "", FinalApplicationStatus.SUCCEEDED, ""));

  // expect usage stats to come from the scheduler report
  ApplicationResourceUsageReport report = 
      applicationAttempt.getApplicationResourceUsageReport();
  Assert.assertEquals(123456L, report.getMemorySeconds());
  Assert.assertEquals(55544L, report.getVcoreSeconds());

  // finish app attempt and remove it from scheduler 
  when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
  sendAttemptUpdateSavedEvent(applicationAttempt);
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
      attemptId, 
      ContainerStatus.newInstance(
          amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));

  when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);

  report = applicationAttempt.getApplicationResourceUsageReport();
  Assert.assertEquals(223456, report.getMemorySeconds());
  Assert.assertEquals(75544, report.getVcoreSeconds());
}
 
Example #27
Source File: TestClientRMService.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
  } finally {
    rmService.close();
  }
}
 
Example #28
Source File: TypeConverter.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static JobStatus fromYarn(ApplicationReport application,
    String jobFile) {
  String trackingUrl = application.getTrackingUrl();
  trackingUrl = trackingUrl == null ? "" : trackingUrl;
  JobStatus jobStatus =
    new JobStatus(
        TypeConverter.fromYarn(application.getApplicationId()),
        0.0f, 0.0f, 0.0f, 0.0f,
        TypeConverter.fromYarn(application.getYarnApplicationState(), application.getFinalApplicationStatus()),
        org.apache.hadoop.mapreduce.JobPriority.NORMAL,
        application.getUser(), application.getName(),
        application.getQueue(), jobFile, trackingUrl, false
    );
  jobStatus.setSchedulingInfo(trackingUrl); // Set AM tracking url
  jobStatus.setStartTime(application.getStartTime());
  jobStatus.setFinishTime(application.getFinishTime());
  jobStatus.setFailureInfo(application.getDiagnostics());
  ApplicationResourceUsageReport resourceUsageReport =
      application.getApplicationResourceUsageReport();
  if (resourceUsageReport != null) {
    jobStatus.setNeededMem(
        resourceUsageReport.getNeededResources().getMemory());
    jobStatus.setNumReservedSlots(
        resourceUsageReport.getNumReservedContainers());
    jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
    jobStatus.setReservedMem(
        resourceUsageReport.getReservedResources().getMemory());
    jobStatus.setUsedMem(resourceUsageReport.getUsedResources().getMemory());
  }
  return jobStatus;
}
 
Example #29
Source File: TestTypeConverter.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testFromYarn() throws Exception {
  int appStartTime = 612354;
  int appFinishTime = 612355;
  YarnApplicationState state = YarnApplicationState.RUNNING;
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationReport applicationReport = Records
      .newRecord(ApplicationReport.class);
  applicationReport.setApplicationId(applicationId);
  applicationReport.setYarnApplicationState(state);
  applicationReport.setStartTime(appStartTime);
  applicationReport.setFinishTime(appFinishTime);
  applicationReport.setUser("TestTypeConverter-user");
  ApplicationResourceUsageReport appUsageRpt = Records
      .newRecord(ApplicationResourceUsageReport.class);
  Resource r = Records.newRecord(Resource.class);
  r.setMemory(2048);
  appUsageRpt.setNeededResources(r);
  appUsageRpt.setNumReservedContainers(1);
  appUsageRpt.setNumUsedContainers(3);
  appUsageRpt.setReservedResources(r);
  appUsageRpt.setUsedResources(r);
  applicationReport.setApplicationResourceUsageReport(appUsageRpt);
  JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
  Assert.assertEquals(appStartTime, jobStatus.getStartTime());
  Assert.assertEquals(appFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
 
Example #30
Source File: GobblinYarnAppLauncher.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
private void sendEmailOnShutdown(Optional<ApplicationReport> applicationReport) {
  String subject = String.format("Gobblin Yarn application %s completed", this.applicationName);

  StringBuilder messageBuilder = new StringBuilder("Gobblin Yarn ApplicationReport:");
  if (applicationReport.isPresent()) {
    messageBuilder.append("\n");
    messageBuilder.append("\tApplication ID: ").append(applicationReport.get().getApplicationId()).append("\n");
    messageBuilder.append("\tApplication attempt ID: ")
        .append(applicationReport.get().getCurrentApplicationAttemptId()).append("\n");
    messageBuilder.append("\tFinal application status: ").append(applicationReport.get().getFinalApplicationStatus())
        .append("\n");
    messageBuilder.append("\tStart time: ").append(applicationReport.get().getStartTime()).append("\n");
    messageBuilder.append("\tFinish time: ").append(applicationReport.get().getFinishTime()).append("\n");

    if (!Strings.isNullOrEmpty(applicationReport.get().getDiagnostics())) {
      messageBuilder.append("\tDiagnostics: ").append(applicationReport.get().getDiagnostics()).append("\n");
    }

    ApplicationResourceUsageReport resourceUsageReport = applicationReport.get().getApplicationResourceUsageReport();
    if (resourceUsageReport != null) {
      messageBuilder.append("\tUsed containers: ").append(resourceUsageReport.getNumUsedContainers()).append("\n");
      Resource usedResource = resourceUsageReport.getUsedResources();
      if (usedResource != null) {
        messageBuilder.append("\tUsed memory (MBs): ").append(usedResource.getMemory()).append("\n");
        messageBuilder.append("\tUsed vcores: ").append(usedResource.getVirtualCores()).append("\n");
      }
    }
  } else {
    messageBuilder.append(' ').append("Not available");
  }

  try {
    EmailUtils.sendEmail(ConfigUtils.configToState(this.config), subject, messageBuilder.toString());
  } catch (EmailException ee) {
    LOGGER.error("Failed to send email notification on shutdown", ee);
  }
}