Java Code Examples for org.apache.hadoop.yarn.api.records.ContainerReport

The following examples show how to use org.apache.hadoop.yarn.api.records.ContainerReport. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
public Map<ContainerId, ContainerReport> getContainers(
    ApplicationAttemptId appAttemptId) throws YarnException, IOException {
  ApplicationReportExt app = getApplication(
      appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
  checkAccess(app);
  TimelineEntities entities = timelineDataManager.getEntities(
      ContainerMetricsConstants.ENTITY_TYPE,
      new NameValuePair(
          ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
          appAttemptId.toString()), null, null, null,
      null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
      UserGroupInformation.getLoginUser());
  Map<ContainerId, ContainerReport> containers =
      new LinkedHashMap<ContainerId, ContainerReport>();
  if (entities != null && entities.getEntities() != null) {
    for (TimelineEntity entity : entities.getEntities()) {
      ContainerReport container = convertToContainerReport(
          entity, serverHttpAddress, app.appReport.getUser());
      containers.put(container.getContainerId(), container);
    }
  }
  return containers;
}
 
Example 2
Source Project: hadoop   Source File: ContainerInfo.java    License: Apache License 2.0 6 votes vote down vote up
public ContainerInfo(ContainerReport container) {
  containerId = container.getContainerId().toString();
  if (container.getAllocatedResource() != null) {
    allocatedMB = container.getAllocatedResource().getMemory();
    allocatedVCores = container.getAllocatedResource().getVirtualCores();
    allocatedGCores = container.getAllocatedResource().getGpuCores();
  }
  if (container.getAssignedNode() != null) {
    assignedNodeId = container.getAssignedNode().toString();
  }
  priority = container.getPriority().getPriority();
  startedTime = container.getCreationTime();
  finishedTime = container.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  diagnosticsInfo = container.getDiagnosticsInfo();
  logUrl = container.getLogUrl();
  containerExitStatus = container.getContainerExitStatus();
  containerState = container.getContainerState();
  nodeHttpAddress = container.getNodeHttpAddress();
}
 
Example 3
@Override
public Map<ContainerId, ContainerReport> getContainers(
    ApplicationAttemptId appAttemptId) throws YarnException, IOException {
  ApplicationReportExt app = getApplication(
      appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
  checkAccess(app);
  TimelineEntities entities = timelineDataManager.getEntities(
      ContainerMetricsConstants.ENTITY_TYPE,
      new NameValuePair(
          ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
          appAttemptId.toString()), null, null, null,
      null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
      UserGroupInformation.getLoginUser());
  Map<ContainerId, ContainerReport> containers =
      new LinkedHashMap<ContainerId, ContainerReport>();
  if (entities != null && entities.getEntities() != null) {
    for (TimelineEntity entity : entities.getEntities()) {
      ContainerReport container = convertToContainerReport(
          entity, serverHttpAddress, app.appReport.getUser());
      containers.put(container.getContainerId(), container);
    }
  }
  return containers;
}
 
Example 4
Source Project: big-c   Source File: ApplicationCLI.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Lists the containers matching the given application attempts
 * 
 * @param appAttemptId
 * @throws YarnException
 * @throws IOException
 */
private void listContainers(String appAttemptId) throws YarnException,
    IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));

  List<ContainerReport> appsReport = client
      .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
  writer.println("Total number of containers " + ":" + appsReport.size());
  writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
      "Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
  for (ContainerReport containerReport : appsReport) {
    writer.printf(
        CONTAINER_PATTERN,
        containerReport.getContainerId(),
        Times.format(containerReport.getCreationTime()),
        Times.format(containerReport.getFinishTime()),      
        containerReport.getContainerState(), containerReport
            .getAssignedNode(), containerReport.getNodeHttpAddress() == null
                ? "N/A" : containerReport.getNodeHttpAddress(),
        containerReport.getLogUrl());
  }
  writer.flush();
}
 
Example 5
@Test
public void testContainers() throws IOException, YarnException {
  ApplicationId appId = ApplicationId.newInstance(0, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 2);
  GetContainersRequest request =
      GetContainersRequest.newInstance(appAttemptId);
  GetContainersResponse response =
      clientService.getContainers(request);
  List<ContainerReport> containers = response.getContainerList();
  Assert.assertNotNull(containers);
  Assert.assertEquals(containerId, containers.get(0).getContainerId());
  Assert.assertEquals(containerId1, containers.get(1).getContainerId());
}
 
Example 6
Source Project: big-c   Source File: TestApplicationHistoryClientService.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testContainerReport() throws IOException, YarnException {
  ApplicationId appId = ApplicationId.newInstance(0, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  GetContainerReportRequest request =
      GetContainerReportRequest.newInstance(containerId);
  GetContainerReportResponse response =
      clientService.getContainerReport(request);
  ContainerReport container = response.getContainerReport();
  Assert.assertNotNull(container);
  Assert.assertEquals(containerId, container.getContainerId());
  Assert.assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" +
      "test host:100/container_0_0001_01_000001/" +
      "container_0_0001_01_000001/user1", container.getLogUrl());
}
 
Example 7
Source Project: big-c   Source File: TestAHSClient.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 10000)
public void testGetContainers() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final AHSClient client = new MockAHSClient();
  client.init(conf);
  client.start();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 1);
  List<ContainerReport> reports = client.getContainers(appAttemptId);
  Assert.assertNotNull(reports);
  Assert.assertEquals(reports.get(0).getContainerId(),
    (ContainerId.newContainerId(appAttemptId, 1)));
  Assert.assertEquals(reports.get(1).getContainerId(),
    (ContainerId.newContainerId(appAttemptId, 2)));
  client.stop();
}
 
Example 8
Source Project: hadoop   Source File: YarnClientImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ContainerReport getContainerReport(ContainerId containerId)
    throws YarnException, IOException {
  try {
    GetContainerReportRequest request = Records
        .newRecord(GetContainerReportRequest.class);
    request.setContainerId(containerId);
    GetContainerReportResponse response = rmClient
        .getContainerReport(request);
    return response.getContainerReport();
  } catch (YarnException e) {
    if (!historyServiceEnabled) {
      // Just throw it as usual if historyService is not enabled.
      throw e;
    }
    // Even if history-service is enabled, treat all exceptions still the same
    // except the following
    if (e.getClass() != ApplicationNotFoundException.class
        && e.getClass() != ContainerNotFoundException.class) {
      throw e;
    }
    return historyClient.getContainerReport(containerId);
  }
}
 
Example 9
Source Project: hadoop   Source File: ApplicationCLI.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Lists the containers matching the given application attempts
 * 
 * @param appAttemptId
 * @throws YarnException
 * @throws IOException
 */
private void listContainers(String appAttemptId) throws YarnException,
    IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));

  List<ContainerReport> appsReport = client
      .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
  writer.println("Total number of containers " + ":" + appsReport.size());
  writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
      "Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
  for (ContainerReport containerReport : appsReport) {
    writer.printf(
        CONTAINER_PATTERN,
        containerReport.getContainerId(),
        Times.format(containerReport.getCreationTime()),
        Times.format(containerReport.getFinishTime()),      
        containerReport.getContainerState(), containerReport
            .getAssignedNode(), containerReport.getNodeHttpAddress() == null
                ? "N/A" : containerReport.getNodeHttpAddress(),
        containerReport.getLogUrl());
  }
  writer.flush();
}
 
Example 10
Source Project: hadoop   Source File: TestAHSClient.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 10000)
public void testGetContainerReport() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final AHSClient client = new MockAHSClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports =
      ((MockAHSClient) client).getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  ContainerReport report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(), (ContainerId
    .newContainerId(expectedReports.get(0).getCurrentApplicationAttemptId(), 1))
    .toString());
  client.stop();
}
 
Example 11
Source Project: samza   Source File: YarnJobValidationTool.java    License: Apache License 2.0 6 votes vote down vote up
public int validateContainerCount(ApplicationAttemptId attemptId) throws Exception {
  int runningContainerCount = 0;
  for (ContainerReport containerReport : this.client.getContainers(attemptId)) {
    if (containerReport.getContainerState() == ContainerState.RUNNING) {
      ++runningContainerCount;
    }
  }
  // expected containers to be the configured job containers plus the AppMaster container
  int containerExpected = this.config.getContainerCount() + 1;

  if (runningContainerCount == containerExpected) {
    log.info("Container count matches. " + runningContainerCount + " containers are running.");
    return runningContainerCount;
  } else {
    throw new SamzaException("Container count does not match. " + runningContainerCount + " containers are running, while " + containerExpected + " is expected.");
  }
}
 
Example 12
Source Project: big-c   Source File: YarnClientImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ContainerReport getContainerReport(ContainerId containerId)
    throws YarnException, IOException {
  try {
    GetContainerReportRequest request = Records
        .newRecord(GetContainerReportRequest.class);
    request.setContainerId(containerId);
    GetContainerReportResponse response = rmClient
        .getContainerReport(request);
    return response.getContainerReport();
  } catch (YarnException e) {
    if (!historyServiceEnabled) {
      // Just throw it as usual if historyService is not enabled.
      throw e;
    }
    // Even if history-service is enabled, treat all exceptions still the same
    // except the following
    if (e.getClass() != ApplicationNotFoundException.class
        && e.getClass() != ContainerNotFoundException.class) {
      throw e;
    }
    return historyClient.getContainerReport(containerId);
  }
}
 
Example 13
Source Project: big-c   Source File: TestYarnClient.java    License: Apache License 2.0 6 votes vote down vote up
private ContainerReport getContainer(
    ContainerId containerId,
    HashMap<ApplicationAttemptId, List<ContainerReport>> containersToAppAttemptMapping)
    throws YarnException, IOException {
  List<ContainerReport> containersForAppAttempt =
      containersToAppAttemptMapping.get(containerId
          .getApplicationAttemptId());
  if (containersForAppAttempt == null) {
    throw new ApplicationNotFoundException(containerId
        .getApplicationAttemptId().getApplicationId() + " is not found ");
  }
  Iterator<ContainerReport> iterator = containersForAppAttempt.iterator();
  while (iterator.hasNext()) {
    ContainerReport next = iterator.next();
    if (next.getContainerId().equals(containerId)) {
      return next;
    }
  }
  throw new ContainerNotFoundException(containerId + " is not found ");
}
 
Example 14
Source Project: big-c   Source File: ContainerInfo.java    License: Apache License 2.0 6 votes vote down vote up
public ContainerInfo(ContainerReport container) {
  containerId = container.getContainerId().toString();
  if (container.getAllocatedResource() != null) {
    allocatedMB = container.getAllocatedResource().getMemory();
    allocatedVCores = container.getAllocatedResource().getVirtualCores();
  }
  if (container.getAssignedNode() != null) {
    assignedNodeId = container.getAssignedNode().toString();
  }
  priority = container.getPriority().getPriority();
  startedTime = container.getCreationTime();
  finishedTime = container.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  diagnosticsInfo = container.getDiagnosticsInfo();
  logUrl = container.getLogUrl();
  containerExitStatus = container.getContainerExitStatus();
  containerState = container.getContainerState();
  nodeHttpAddress = container.getNodeHttpAddress();
}
 
Example 15
Source Project: big-c   Source File: TestYarnClient.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 10000)
public void testGetContainers() throws YarnException, IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
      true);
  
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  List<ContainerReport> reports = client.getContainers(appAttemptId);
  Assert.assertNotNull(reports);
  Assert.assertEquals(reports.get(0).getContainerId(),
      (ContainerId.newContainerId(appAttemptId, 1)));
  Assert.assertEquals(reports.get(1).getContainerId(),
      (ContainerId.newContainerId(appAttemptId, 2)));
  Assert.assertEquals(reports.get(2).getContainerId(),
      (ContainerId.newContainerId(appAttemptId, 3)));
  
  //First2 containers should come from RM with updated state information and 
  // 3rd container is not there in RM and should
  Assert.assertEquals(ContainerState.RUNNING,
      (reports.get(0).getContainerState()));
  Assert.assertEquals(ContainerState.RUNNING,
      (reports.get(1).getContainerState()));
  Assert.assertEquals(ContainerState.COMPLETE,
      (reports.get(2).getContainerState()));
  client.stop();
}
 
Example 16
Source Project: hadoop   Source File: GetContainerReportResponsePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setContainerReport(ContainerReport containerReport) {
  maybeInitBuilder();
  if (containerReport == null) {
    builder.clearContainerReport();
  }
  this.containerReport = containerReport;
}
 
Example 17
Source Project: hadoop   Source File: GetContainersResponsePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void initLocalContainerList() {
  if (this.containerList != null) {
    return;
  }
  GetContainersResponseProtoOrBuilder p = viaProto ? proto : builder;
  List<ContainerReportProto> list = p.getContainersList();
  containerList = new ArrayList<ContainerReport>();

  for (ContainerReportProto c : list) {
    containerList.add(convertFromProtoFormat(c));
  }
}
 
Example 18
@Override
public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
    throws YarnException, IOException {
  ApplicationAttemptReport appAttempt =
      getApplicationAttempt(appAttemptId, false);
  return getContainer(appAttempt.getAMContainerId());
}
 
Example 19
Source Project: big-c   Source File: GetContainersResponse.java    License: Apache License 2.0 5 votes vote down vote up
@Public
@Unstable
public static GetContainersResponse newInstance(
    List<ContainerReport> containers) {
  GetContainersResponse response =
      Records.newRecord(GetContainersResponse.class);
  response.setContainerList(containers);
  return response;
}
 
Example 20
Source Project: hadoop   Source File: AppAttemptBlock.java    License: Apache License 2.0 5 votes vote down vote up
private boolean hasAMContainer(ContainerId containerId,
    Collection<ContainerReport> containers) {
  for (ContainerReport container : containers) {
    if (containerId.equals(container.getContainerId())) {
      return true;
    }
  }
  return false;
}
 
Example 21
Source Project: big-c   Source File: TestApplicationClientProtocolOnHA.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 15000)
public void testGetContainerReportOnHA() throws Exception {
  ContainerReport report =
      client.getContainerReport(cluster.createFakeContainerId());
  Assert.assertTrue(report != null);
  Assert.assertEquals(cluster.createFakeContainerReport(), report);
}
 
Example 22
Source Project: hadoop   Source File: ApplicationHistoryManagerImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
    throws IOException {
  ApplicationReport app =
      getApplication(appAttemptId.getApplicationId());
  return convertToContainerReport(historyStore.getAMContainer(appAttemptId),
      app == null ? null : app.getUser());
}
 
Example 23
Source Project: hadoop   Source File: ApplicationHistoryManagerImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ContainerReport getContainer(ContainerId containerId)
    throws IOException {
  ApplicationReport app =
      getApplication(containerId.getApplicationAttemptId().getApplicationId());
  return convertToContainerReport(historyStore.getContainer(containerId),
      app == null ? null: app.getUser());
}
 
Example 24
Source Project: big-c   Source File: AHSClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ContainerReport getContainerReport(ContainerId containerId)
    throws YarnException, IOException {
  GetContainerReportRequest request = GetContainerReportRequest
      .newInstance(containerId);
  GetContainerReportResponse response = ahsClient.getContainerReport(request);
  return response.getContainerReport();
}
 
Example 25
@Test
public void testGetContainers() throws Exception {
  final ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
  Collection<ContainerReport> containers;
  if (callerUGI == null) {
    containers = historyManager.getContainers(appAttemptId).values();
  } else {
    try {
      containers = callerUGI.doAs(
          new PrivilegedExceptionAction<Collection<ContainerReport>> () {
        @Override
        public Collection<ContainerReport> run() throws Exception {
          return historyManager.getContainers(appAttemptId).values();
        }
      });
      if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
        // The exception is expected
        Assert.fail();
      }
    } catch (AuthorizationException e) {
      if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
        // The exception is expected
        return;
      }
      throw e;
    }
  }
  Assert.assertNotNull(containers);
  Assert.assertEquals(SCALE, containers.size());
}
 
Example 26
Source Project: hadoop   Source File: ApplicationHistoryClientService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public GetContainersResponse getContainers(GetContainersRequest request)
    throws YarnException, IOException {
  GetContainersResponse response =
      GetContainersResponse.newInstance(new ArrayList<ContainerReport>(
        history.getContainers(request.getApplicationAttemptId()).values()));
  return response;
}
 
Example 27
@Override
public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
    throws YarnException, IOException {
  ApplicationAttemptReport appAttempt =
      getApplicationAttempt(appAttemptId, false);
  return getContainer(appAttempt.getAMContainerId());
}
 
Example 28
@Test
public void testGetContainers() throws Exception {
  final ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
  Collection<ContainerReport> containers;
  if (callerUGI == null) {
    containers = historyManager.getContainers(appAttemptId).values();
  } else {
    try {
      containers = callerUGI.doAs(
          new PrivilegedExceptionAction<Collection<ContainerReport>> () {
        @Override
        public Collection<ContainerReport> run() throws Exception {
          return historyManager.getContainers(appAttemptId).values();
        }
      });
      if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
        // The exception is expected
        Assert.fail();
      }
    } catch (AuthorizationException e) {
      if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
        // The exception is expected
        return;
      }
      throw e;
    }
  }
  Assert.assertNotNull(containers);
  Assert.assertEquals(SCALE, containers.size());
}
 
Example 29
Source Project: big-c   Source File: TestYarnClient.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<ContainerReport>
    getContainers(ApplicationAttemptId appAttemptId) throws YarnException,
        IOException {
  when(mockContainersResponse.getContainerList()).thenReturn(
    getContainersReport(appAttemptId));
  when(historyClient.getContainers(any(ApplicationAttemptId.class)))
  .thenReturn(getContainersFromAHS(appAttemptId));
  return super.getContainers(appAttemptId);
}
 
Example 30
Source Project: hadoop   Source File: AHSClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ContainerReport getContainerReport(ContainerId containerId)
    throws YarnException, IOException {
  GetContainerReportRequest request = GetContainerReportRequest
      .newInstance(containerId);
  GetContainerReportResponse response = ahsClient.getContainerReport(request);
  return response.getContainerReport();
}