Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: AthenaX   Source File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
static InstanceInfo extractInstanceInfo(String clusterName, ApplicationReport report) {
  InstanceMetadata md = getMetadata(report.getApplicationTags());
  if (md == null) {
    return null;
  }

  ApplicationResourceUsageReport usage = report.getApplicationResourceUsageReport();
  InstanceStatus stat = new InstanceStatus()
      .allocatedVCores((long) usage.getUsedResources().getVirtualCores())
      .allocatedMB((long) usage.getUsedResources().getMemory())
      .clusterId(clusterName)
      .applicationId(report.getApplicationId().toString())
      .startedTime(report.getStartTime())
      .runningContainers((long) usage.getNumUsedContainers())
      .trackingUrl(report.getTrackingUrl())
      .state(InstanceStatus.StateEnum.fromValue(report.getYarnApplicationState().toString()));
  return new InstanceInfo(clusterName, report.getApplicationId(), md, stat);
}
 
Example 2
Source Project: hadoop   Source File: BuilderUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
    int numUsedContainers, int numReservedContainers, Resource usedResources,
    Resource reservedResources, Resource neededResources, long memorySeconds, 
    long vcoreSeconds, long gcoreSeconds) {
  ApplicationResourceUsageReport report =
      recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
  report.setNumUsedContainers(numUsedContainers);
  report.setNumReservedContainers(numReservedContainers);
  report.setUsedResources(usedResources);
  report.setReservedResources(reservedResources);
  report.setNeededResources(neededResources);
  report.setMemorySeconds(memorySeconds);
  report.setVcoreSeconds(vcoreSeconds);
  report.setGcoreSeconds(gcoreSeconds);
  return report;
}
 
Example 3
Source Project: hadoop   Source File: RMAppAttemptImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
  this.readLock.lock();
  try {
    ApplicationResourceUsageReport report =
        scheduler.getAppResourceUsageReport(this.getAppAttemptId());
    if (report == null) {
      report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
    }
    AggregateAppResourceUsage resUsage =
        this.attemptMetrics.getAggregateAppResourceUsage();
    report.setMemorySeconds(resUsage.getMemorySeconds());
    report.setVcoreSeconds(resUsage.getVcoreSeconds());
    report.setGcoreSeconds(resUsage.getGcoreSeconds());
    return report;
  } finally {
    this.readLock.unlock();
  }
}
 
Example 4
Source Project: hadoop   Source File: RMAppAttemptMetrics.java    License: Apache License 2.0 6 votes vote down vote up
public AggregateAppResourceUsage getAggregateAppResourceUsage() {
  long memorySeconds = finishedMemorySeconds.get();
  long vcoreSeconds = finishedVcoreSeconds.get();
  long gcoreSeconds = finishedGcoreSeconds.get();

  // Only add in the running containers if this is the active attempt.
  RMAppAttempt currentAttempt = rmContext.getRMApps()
                 .get(attemptId.getApplicationId()).getCurrentAppAttempt();
  if (currentAttempt.getAppAttemptId().equals(attemptId)) {
    ApplicationResourceUsageReport appResUsageReport = rmContext
          .getScheduler().getAppResourceUsageReport(attemptId);
    if (appResUsageReport != null) {
      memorySeconds += appResUsageReport.getMemorySeconds();
      vcoreSeconds += appResUsageReport.getVcoreSeconds();
      gcoreSeconds += appResUsageReport.getGcoreSeconds();
    }
  }
  return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds, gcoreSeconds);
}
 
Example 5
Source Project: hadoop   Source File: TestApplicationACLs.java    License: Apache License 2.0 6 votes vote down vote up
private void verifyEnemyAppReport(ApplicationReport appReport) {
  Assert.assertEquals("Enemy should not see app host!",
      UNAVAILABLE, appReport.getHost());
  Assert.assertEquals("Enemy should not see app rpc port!",
      -1, appReport.getRpcPort());
  Assert.assertEquals("Enemy should not see app client token!",
      null, appReport.getClientToAMToken());
  Assert.assertEquals("Enemy should not see app diagnostics!",
      UNAVAILABLE, appReport.getDiagnostics());
  Assert.assertEquals("Enemy should not see app tracking url!",
      UNAVAILABLE, appReport.getTrackingUrl());
  Assert.assertEquals("Enemy should not see app original tracking url!",
      UNAVAILABLE, appReport.getOriginalTrackingUrl());
  ApplicationResourceUsageReport usageReport =
      appReport.getApplicationResourceUsageReport();
  Assert.assertEquals("Enemy should not see app used containers",
      -1, usageReport.getNumUsedContainers());
  Assert.assertEquals("Enemy should not see app reserved containers",
      -1, usageReport.getNumReservedContainers());
  Assert.assertEquals("Enemy should not see app used resources",
      -1, usageReport.getUsedResources().getMemory());
  Assert.assertEquals("Enemy should not see app reserved resources",
      -1, usageReport.getReservedResources().getMemory());
  Assert.assertEquals("Enemy should not see app needed resources",
      -1, usageReport.getNeededResources().getMemory());
}
 
Example 6
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetApplicationResourceUsageReportDummy() throws YarnException,
    IOException {
  ApplicationAttemptId attemptId = getApplicationAttemptId(1);
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });
  ApplicationSubmissionContext asContext = 
      mock(ApplicationSubmissionContext.class);
  YarnConfiguration config = new YarnConfiguration();
  RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId,
      rmContext, yarnScheduler, null, asContext, config, false, null);
  ApplicationResourceUsageReport report = rmAppAttemptImpl
      .getApplicationResourceUsageReport();
  assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
}
 
Example 7
Source Project: hadoop   Source File: TestResourceMgrDelegate.java    License: Apache License 2.0 6 votes vote down vote up
private ApplicationReport getApplicationReport(
    YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  ApplicationReport appReport = Mockito.mock(ApplicationReport.class);
  ApplicationResourceUsageReport appResources = Mockito
      .mock(ApplicationResourceUsageReport.class);
  Mockito.when(appReport.getApplicationId()).thenReturn(
      ApplicationId.newInstance(0, 0));
  Mockito.when(appResources.getNeededResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getReservedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getUsedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
      appResources);
  Mockito.when(appReport.getYarnApplicationState()).thenReturn(
      yarnApplicationState);
  Mockito.when(appReport.getFinalApplicationStatus()).thenReturn(
      finalApplicationStatus);

  return appReport;
}
 
Example 8
Source Project: big-c   Source File: RMAppAttemptImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
  this.readLock.lock();
  try {
    ApplicationResourceUsageReport report =
        scheduler.getAppResourceUsageReport(this.getAppAttemptId());
    if (report == null) {
      report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
    }
    AggregateAppResourceUsage resUsage =
        this.attemptMetrics.getAggregateAppResourceUsage();
    report.setMemorySeconds(resUsage.getMemorySeconds());
    report.setVcoreSeconds(resUsage.getVcoreSeconds());
    return report;
  } finally {
    this.readLock.unlock();
  }
}
 
Example 9
Source Project: big-c   Source File: RMAppAttemptMetrics.java    License: Apache License 2.0 6 votes vote down vote up
public AggregateAppResourceUsage getAggregateAppResourceUsage() {
  long memorySeconds = finishedMemorySeconds.get();
  long vcoreSeconds = finishedVcoreSeconds.get();

  // Only add in the running containers if this is the active attempt.
  RMAppAttempt currentAttempt = rmContext.getRMApps()
                 .get(attemptId.getApplicationId()).getCurrentAppAttempt();
  if (currentAttempt.getAppAttemptId().equals(attemptId)) {
    ApplicationResourceUsageReport appResUsageReport = rmContext
          .getScheduler().getAppResourceUsageReport(attemptId);
    if (appResUsageReport != null) {
      memorySeconds += appResUsageReport.getMemorySeconds();
      vcoreSeconds += appResUsageReport.getVcoreSeconds();
    }
  }
  return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds);
}
 
Example 10
Source Project: big-c   Source File: TestApplicationACLs.java    License: Apache License 2.0 6 votes vote down vote up
private void verifyEnemyAppReport(ApplicationReport appReport) {
  Assert.assertEquals("Enemy should not see app host!",
      UNAVAILABLE, appReport.getHost());
  Assert.assertEquals("Enemy should not see app rpc port!",
      -1, appReport.getRpcPort());
  Assert.assertEquals("Enemy should not see app client token!",
      null, appReport.getClientToAMToken());
  Assert.assertEquals("Enemy should not see app diagnostics!",
      UNAVAILABLE, appReport.getDiagnostics());
  Assert.assertEquals("Enemy should not see app tracking url!",
      UNAVAILABLE, appReport.getTrackingUrl());
  Assert.assertEquals("Enemy should not see app original tracking url!",
      UNAVAILABLE, appReport.getOriginalTrackingUrl());
  ApplicationResourceUsageReport usageReport =
      appReport.getApplicationResourceUsageReport();
  Assert.assertEquals("Enemy should not see app used containers",
      -1, usageReport.getNumUsedContainers());
  Assert.assertEquals("Enemy should not see app reserved containers",
      -1, usageReport.getNumReservedContainers());
  Assert.assertEquals("Enemy should not see app used resources",
      -1, usageReport.getUsedResources().getMemory());
  Assert.assertEquals("Enemy should not see app reserved resources",
      -1, usageReport.getReservedResources().getMemory());
  Assert.assertEquals("Enemy should not see app needed resources",
      -1, usageReport.getNeededResources().getMemory());
}
 
Example 11
Source Project: big-c   Source File: TestClientRMService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGetApplicationResourceUsageReportDummy() throws YarnException,
    IOException {
  ApplicationAttemptId attemptId = getApplicationAttemptId(1);
  YarnScheduler yarnScheduler = mockYarnScheduler();
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);
  when(rmContext.getDispatcher().getEventHandler()).thenReturn(
      new EventHandler<Event>() {
        public void handle(Event event) {
        }
      });
  ApplicationSubmissionContext asContext = 
      mock(ApplicationSubmissionContext.class);
  YarnConfiguration config = new YarnConfiguration();
  RMAppAttemptImpl rmAppAttemptImpl = new RMAppAttemptImpl(attemptId,
      rmContext, yarnScheduler, null, asContext, config, false, null);
  ApplicationResourceUsageReport report = rmAppAttemptImpl
      .getApplicationResourceUsageReport();
  assertEquals(report, RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
}
 
Example 12
Source Project: big-c   Source File: TestResourceMgrDelegate.java    License: Apache License 2.0 6 votes vote down vote up
private ApplicationReport getApplicationReport(
    YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  ApplicationReport appReport = Mockito.mock(ApplicationReport.class);
  ApplicationResourceUsageReport appResources = Mockito
      .mock(ApplicationResourceUsageReport.class);
  Mockito.when(appReport.getApplicationId()).thenReturn(
      ApplicationId.newInstance(0, 0));
  Mockito.when(appResources.getNeededResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getReservedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appResources.getUsedResources()).thenReturn(
      Records.newRecord(Resource.class));
  Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
      appResources);
  Mockito.when(appReport.getYarnApplicationState()).thenReturn(
      yarnApplicationState);
  Mockito.when(appReport.getFinalApplicationStatus()).thenReturn(
      finalApplicationStatus);

  return appReport;
}
 
Example 13
/**
 * It fetches used vcore and used containers for running applications
 *  from hadoop (resource manager) and persists into influxdb
 * @param cluster
 * @throws Exception
 */
private void persistRunningAppsDataToInfluxdb(Cluster cluster) throws Exception {
	
	List<ApplicationReport> list = sessionUtils.getRM(cluster, getSession()).getRunningApplications();
	
	InfluxDataWriter writer = new InfluxDataWriter(
			AdminConfigurationUtil.getInfluxdbConfiguration(cluster.getClusterName()));
	writer.setTableName(WebConstants.JOB_HISTORY_DETAILS_TABLE);
	writer.setTimeUnit(TimeUnit.SECONDS);
	
	ApplicationResourceUsageReport usage;
	
	for (ApplicationReport report: list) {
		usage = report.getApplicationResourceUsageReport();
		
		writer.addColumn(WebConstants.USED_V_CORES, usage.getUsedResources().getVirtualCores());
		
		writer.addColumn(WebConstants.USED_CONTAINERS, usage.getNumUsedContainers());
		
		writer.addTag(WebConstants.JOB_ID, report.getApplicationId()
				.toString().replace(WebConstants.APPLICATION, WebConstants.JOB));
		writer.writeData();
	}
}
 
Example 14
Source Project: cloudbreak   Source File: AppReportConverter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public AppReportJson convert(ApplicationReport source) {
    AppReportJson json = new AppReportJson();
    json.setAppId(source.getApplicationId().toString());
    json.setStart(source.getStartTime());
    json.setFinish(source.getFinishTime());
    json.setProgress(source.getProgress());
    json.setQueue(source.getQueue());
    json.setUrl(source.getTrackingUrl());
    json.setUser(source.getUser());
    json.setState(source.getYarnApplicationState().name());
    ApplicationResourceUsageReport usageReport = source.getApplicationResourceUsageReport();
    json.setReservedContainers(usageReport.getNumReservedContainers());
    json.setUsedContainers(usageReport.getNumUsedContainers());
    json.setUsedMemory(usageReport.getUsedResources().getMemory());
    json.setUsedVCores(usageReport.getUsedResources().getVirtualCores());
    return json;
}
 
Example 15
Source Project: hadoop   Source File: ApplicationReportPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
public void setApplicationResourceUsageReport(ApplicationResourceUsageReport appInfo) {
  maybeInitBuilder();
  if (appInfo == null) {
    builder.clearAppResourceUsage();
    return;
  }
  builder.setAppResourceUsage(convertToProtoFormat(appInfo));
}
 
Example 16
Source Project: hadoop   Source File: ApplicationReportPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
  ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasAppResourceUsage()) {
    return null;
  }
  return convertFromProtoFormat(p.getAppResourceUsage());
}
 
Example 17
Source Project: hadoop   Source File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static ApplicationReport newApplicationReport(
    ApplicationId applicationId, ApplicationAttemptId applicationAttemptId,
    String user, String queue, String name, String host, int rpcPort,
    Token clientToAMToken, YarnApplicationState state, String diagnostics,
    String url, long startTime, long finishTime,
    FinalApplicationStatus finalStatus,
    ApplicationResourceUsageReport appResources, String origTrackingUrl,
    float progress, String appType, Token amRmToken, Set<String> tags) {
  ApplicationReport report = recordFactory
      .newRecordInstance(ApplicationReport.class);
  report.setApplicationId(applicationId);
  report.setCurrentApplicationAttemptId(applicationAttemptId);
  report.setUser(user);
  report.setQueue(queue);
  report.setName(name);
  report.setHost(host);
  report.setRpcPort(rpcPort);
  report.setClientToAMToken(clientToAMToken);
  report.setYarnApplicationState(state);
  report.setDiagnostics(diagnostics);
  report.setTrackingUrl(url);
  report.setStartTime(startTime);
  report.setFinishTime(finishTime);
  report.setFinalApplicationStatus(finalStatus);
  report.setApplicationResourceUsageReport(appResources);
  report.setOriginalTrackingUrl(origTrackingUrl);
  report.setProgress(progress);
  report.setApplicationType(appType);
  report.setAMRMToken(amRmToken);
  report.setApplicationTags(tags);
  return report;
}
 
Example 18
Source Project: hadoop   Source File: SchedulerApplicationAttempt.java    License: Apache License 2.0 5 votes vote down vote up
public synchronized ApplicationResourceUsageReport getResourceUsageReport() {
  AggregateAppResourceUsage runningResourceUsage =
      getRunningAggregateAppResourceUsage();
  Resource usedResourceClone =
      Resources.clone(attemptResourceUsage.getUsed());
  Resource reservedResourceClone =
      Resources.clone(attemptResourceUsage.getReserved());
  return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
      reservedContainers.size(), usedResourceClone, reservedResourceClone,
      Resources.add(usedResourceClone, reservedResourceClone),
      runningResourceUsage.getMemorySeconds(),
      runningResourceUsage.getVcoreSeconds(),
      runningResourceUsage.getGcoreSeconds());
}
 
Example 19
Source Project: hadoop   Source File: AbstractYarnScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ApplicationResourceUsageReport getAppResourceUsageReport(
    ApplicationAttemptId appAttemptId) {
  SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId);
  if (attempt == null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Request for appInfo of unknown attempt " + appAttemptId);
    }
    return null;
  }
  return attempt.getResourceUsageReport();
}
 
Example 20
Source Project: hadoop   Source File: TestRMWebApp.java    License: Apache License 2.0 5 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  ClientRMService clientRMService = mock(ClientRMService.class);
  List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
  for (RMApp app : rmContext.getRMApps().values()) {
    ApplicationReport appReport =
        ApplicationReport.newInstance(
            app.getApplicationId(), (ApplicationAttemptId) null,
            app.getUser(), app.getQueue(),
            app.getName(), (String) null, 0, (Token) null,
            app.createApplicationState(),
            app.getDiagnostics().toString(), (String) null,
            app.getStartTime(), app.getFinishTime(),
            app.getFinalApplicationStatus(),
            (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
            app.getProgress(), app.getApplicationType(), (Token) null);
    appReports.add(appReport);
  }
  GetApplicationsResponse response = mock(GetApplicationsResponse.class);
  when(response.getApplicationList()).thenReturn(appReports);
  try {
    when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
        .thenReturn(response);
  } catch (YarnException e) {
    Assert.fail("Exception is not expteced.");
  }
  return clientRMService;
}
 
Example 21
Source Project: hadoop   Source File: TestClientRMService.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetApplicationReport() throws Exception {
  YarnScheduler yarnScheduler = mock(YarnScheduler.class);
  RMContext rmContext = mock(RMContext.class);
  mockRMContext(yarnScheduler, rmContext);

  ApplicationId appId1 = getApplicationId(1);

  ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
  when(
      mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(),
          ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);

  ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler,
      null, mockAclsManager, null, null);
  try {
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
    GetApplicationReportRequest request = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    request.setApplicationId(appId1);
    GetApplicationReportResponse response = 
        rmService.getApplicationReport(request);
    ApplicationReport report = response.getApplicationReport();
    ApplicationResourceUsageReport usageReport = 
        report.getApplicationResourceUsageReport();
    Assert.assertEquals(10, usageReport.getMemorySeconds());
    Assert.assertEquals(3, usageReport.getVcoreSeconds());
    Assert.assertEquals(3, usageReport.getGcoreSeconds());
  } finally {
    rmService.close();
  }
}
 
Example 22
Source Project: hadoop   Source File: TypeConverter.java    License: Apache License 2.0 5 votes vote down vote up
public static JobStatus fromYarn(ApplicationReport application,
    String jobFile) {
  String trackingUrl = application.getTrackingUrl();
  trackingUrl = trackingUrl == null ? "" : trackingUrl;
  JobStatus jobStatus =
    new JobStatus(
        TypeConverter.fromYarn(application.getApplicationId()),
        0.0f, 0.0f, 0.0f, 0.0f,
        TypeConverter.fromYarn(application.getYarnApplicationState(), application.getFinalApplicationStatus()),
        org.apache.hadoop.mapreduce.JobPriority.NORMAL,
        application.getUser(), application.getName(),
        application.getQueue(), jobFile, trackingUrl, false
    );
  jobStatus.setSchedulingInfo(trackingUrl); // Set AM tracking url
  jobStatus.setStartTime(application.getStartTime());
  jobStatus.setFinishTime(application.getFinishTime());
  jobStatus.setFailureInfo(application.getDiagnostics());
  ApplicationResourceUsageReport resourceUsageReport =
      application.getApplicationResourceUsageReport();
  if (resourceUsageReport != null) {
    jobStatus.setNeededMem(
        resourceUsageReport.getNeededResources().getMemory());
    jobStatus.setNumReservedSlots(
        resourceUsageReport.getNumReservedContainers());
    jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
    jobStatus.setReservedMem(
        resourceUsageReport.getReservedResources().getMemory());
    jobStatus.setUsedMem(resourceUsageReport.getUsedResources().getMemory());
  }
  return jobStatus;
}
 
Example 23
Source Project: hadoop   Source File: TestTypeConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFromYarn() throws Exception {
  int appStartTime = 612354;
  int appFinishTime = 612355;
  YarnApplicationState state = YarnApplicationState.RUNNING;
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationReport applicationReport = Records
      .newRecord(ApplicationReport.class);
  applicationReport.setApplicationId(applicationId);
  applicationReport.setYarnApplicationState(state);
  applicationReport.setStartTime(appStartTime);
  applicationReport.setFinishTime(appFinishTime);
  applicationReport.setUser("TestTypeConverter-user");
  ApplicationResourceUsageReport appUsageRpt = Records
      .newRecord(ApplicationResourceUsageReport.class);
  Resource r = Records.newRecord(Resource.class);
  r.setMemory(2048);
  appUsageRpt.setNeededResources(r);
  appUsageRpt.setNumReservedContainers(1);
  appUsageRpt.setNumUsedContainers(3);
  appUsageRpt.setReservedResources(r);
  appUsageRpt.setUsedResources(r);
  applicationReport.setApplicationResourceUsageReport(appUsageRpt);
  JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
  Assert.assertEquals(appStartTime, jobStatus.getStartTime());
  Assert.assertEquals(appFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
 
Example 24
Source Project: twill   Source File: TwillTester.java    License: Apache License 2.0 5 votes vote down vote up
public ApplicationResourceUsageReport getApplicationResourceReport(String appId) throws Exception {
  List<String> splits = Lists.newArrayList(Splitter.on('_').split(appId));
  Preconditions.checkArgument(splits.size() == 3, "Invalid application id - " + appId);
  ApplicationId applicationId = ApplicationId.newInstance(Long.parseLong(splits.get(1)),
                                                          Integer.parseInt(splits.get(2)));

  ClientRMService clientRMService = cluster.getResourceManager().getClientRMService();
  GetApplicationReportRequest request = Records.newRecord(GetApplicationReportRequest.class);
  request.setApplicationId(applicationId);
  return clientRMService.getApplicationReport(request)
    .getApplicationReport().getApplicationResourceUsageReport();
}
 
Example 25
Source Project: big-c   Source File: ApplicationReportPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
public void setApplicationResourceUsageReport(ApplicationResourceUsageReport appInfo) {
  maybeInitBuilder();
  if (appInfo == null) {
    builder.clearAppResourceUsage();
    return;
  }
  builder.setAppResourceUsage(convertToProtoFormat(appInfo));
}
 
Example 26
Source Project: big-c   Source File: ApplicationReportPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
  ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasAppResourceUsage()) {
    return null;
  }
  return convertFromProtoFormat(p.getAppResourceUsage());
}
 
Example 27
Source Project: big-c   Source File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static ApplicationReport newApplicationReport(
    ApplicationId applicationId, ApplicationAttemptId applicationAttemptId,
    String user, String queue, String name, String host, int rpcPort,
    Token clientToAMToken, YarnApplicationState state, String diagnostics,
    String url, long startTime, long finishTime,
    FinalApplicationStatus finalStatus,
    ApplicationResourceUsageReport appResources, String origTrackingUrl,
    float progress, String appType, Token amRmToken, Set<String> tags) {
  ApplicationReport report = recordFactory
      .newRecordInstance(ApplicationReport.class);
  report.setApplicationId(applicationId);
  report.setCurrentApplicationAttemptId(applicationAttemptId);
  report.setUser(user);
  report.setQueue(queue);
  report.setName(name);
  report.setHost(host);
  report.setRpcPort(rpcPort);
  report.setClientToAMToken(clientToAMToken);
  report.setYarnApplicationState(state);
  report.setDiagnostics(diagnostics);
  report.setTrackingUrl(url);
  report.setStartTime(startTime);
  report.setFinishTime(finishTime);
  report.setFinalApplicationStatus(finalStatus);
  report.setApplicationResourceUsageReport(appResources);
  report.setOriginalTrackingUrl(origTrackingUrl);
  report.setProgress(progress);
  report.setApplicationType(appType);
  report.setAMRMToken(amRmToken);
  report.setApplicationTags(tags);
  return report;
}
 
Example 28
Source Project: big-c   Source File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
    int numUsedContainers, int numReservedContainers, Resource usedResources,
    Resource reservedResources, Resource neededResources, long memorySeconds, 
    long vcoreSeconds) {
  ApplicationResourceUsageReport report =
      recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
  report.setNumUsedContainers(numUsedContainers);
  report.setNumReservedContainers(numReservedContainers);
  report.setUsedResources(usedResources);
  report.setReservedResources(reservedResources);
  report.setNeededResources(neededResources);
  report.setMemorySeconds(memorySeconds);
  report.setVcoreSeconds(vcoreSeconds);
  return report;
}
 
Example 29
Source Project: big-c   Source File: SchedulerApplicationAttempt.java    License: Apache License 2.0 5 votes vote down vote up
public synchronized ApplicationResourceUsageReport getResourceUsageReport() {
  AggregateAppResourceUsage resUsage = getRunningAggregateAppResourceUsage();
  return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
             reservedContainers.size(), Resources.clone(currentConsumption),
             Resources.clone(currentReservation),
             Resources.add(currentConsumption, currentReservation),
             resUsage.getMemorySeconds(), resUsage.getVcoreSeconds());
}
 
Example 30
Source Project: big-c   Source File: AbstractYarnScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ApplicationResourceUsageReport getAppResourceUsageReport(
    ApplicationAttemptId appAttemptId) {
  SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId);
  if (attempt == null) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Request for appInfo of unknown attempt " + appAttemptId);
    }
    return null;
  }
  return attempt.getResourceUsageReport();
}