org.apache.hadoop.yarn.util.Times Java Examples

The following examples show how to use org.apache.hadoop.yarn.util.Times. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ReduceTaskAttemptInfo.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public ReduceTaskAttemptInfo(TaskAttempt ta, TaskType type) {
  super(ta, type, false);

  this.shuffleFinishTime = ta.getShuffleFinishTime();
  this.mergeFinishTime = ta.getSortFinishTime();
  this.elapsedShuffleTime = Times.elapsed(this.startTime,
      this.shuffleFinishTime, false);
  if (this.elapsedShuffleTime == -1) {
    this.elapsedShuffleTime = 0;
  }
  this.elapsedMergeTime = Times.elapsed(this.shuffleFinishTime,
      this.mergeFinishTime, false);
  if (this.elapsedMergeTime == -1) {
    this.elapsedMergeTime = 0;
  }
  this.elapsedReduceTime = Times.elapsed(this.mergeFinishTime,
      this.finishTime, false);
  if (this.elapsedReduceTime == -1) {
    this.elapsedReduceTime = 0;
  }
}
 
Example #2
Source File: AboutBlock.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected void render(Block html) {
  html._(MetricsOverviewTable.class);
  ResourceManager rm = getInstance(ResourceManager.class);
  ClusterInfo cinfo = new ClusterInfo(rm);
  info("Cluster overview").
    _("Cluster ID:", cinfo.getClusterId()).
    _("ResourceManager state:", cinfo.getState()).
    _("ResourceManager HA state:", cinfo.getHAState()).
    _("ResourceManager RMStateStore:", cinfo.getRMStateStore()).
    _("ResourceManager started on:", Times.format(cinfo.getStartedOn())).
    _("ResourceManager version:", cinfo.getRMBuildVersion() +
        " on " + cinfo.getRMVersionBuiltOn()).
    _("Hadoop version:", cinfo.getHadoopBuildVersion() +
        " on " + cinfo.getHadoopVersionBuiltOn());
  html._(InfoBlock.class);
}
 
Example #3
Source File: SCMOverviewPage.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
protected void render(Block html) {
  SCMMetricsInfo metricsInfo = new SCMMetricsInfo(
      CleanerMetrics.getInstance(), ClientSCMMetrics.getInstance(),
          SharedCacheUploaderMetrics.getInstance());
  info("Shared Cache Manager overview").
      _("Started on:", Times.format(scm.getStartTime())).
      _("Cache hits: ", metricsInfo.getCacheHits()).
      _("Cache misses: ", metricsInfo.getCacheMisses()).
      _("Cache releases: ", metricsInfo.getCacheReleases()).
      _("Accepted uploads: ", metricsInfo.getAcceptedUploads()).
      _("Rejected uploads: ", metricsInfo.getRejectUploads()).
      _("Deleted files by the cleaner: ", metricsInfo.getTotalDeletedFiles()).
      _("Processed files by the cleaner: ", metricsInfo.getTotalProcessedFiles());
  html._(InfoBlock.class);
}
 
Example #4
Source File: AppInfo.java    From big-c with Apache License 2.0 6 votes vote down vote up
public AppInfo(ApplicationReport app) {
  appId = app.getApplicationId().toString();
  if (app.getCurrentApplicationAttemptId() != null) {
    currentAppAttemptId = app.getCurrentApplicationAttemptId().toString();
  }
  user = app.getUser();
  queue = app.getQueue();
  name = app.getName();
  type = app.getApplicationType();
  host = app.getHost();
  rpcPort = app.getRpcPort();
  appState = app.getYarnApplicationState();
  diagnosticsInfo = app.getDiagnostics();
  trackingUrl = app.getTrackingUrl();
  originalTrackingUrl = app.getOriginalTrackingUrl();
  submittedTime = app.getStartTime();
  startedTime = app.getStartTime();
  finishedTime = app.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  finalAppStatus = app.getFinalApplicationStatus();
  progress = app.getProgress() * 100; // in percent
  if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
    this.applicationTags = CSV_JOINER.join(app.getApplicationTags());
  }
}
 
Example #5
Source File: ContainerInfo.java    From big-c with Apache License 2.0 6 votes vote down vote up
public ContainerInfo(ContainerReport container) {
  containerId = container.getContainerId().toString();
  if (container.getAllocatedResource() != null) {
    allocatedMB = container.getAllocatedResource().getMemory();
    allocatedVCores = container.getAllocatedResource().getVirtualCores();
  }
  if (container.getAssignedNode() != null) {
    assignedNodeId = container.getAssignedNode().toString();
  }
  priority = container.getPriority().getPriority();
  startedTime = container.getCreationTime();
  finishedTime = container.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  diagnosticsInfo = container.getDiagnosticsInfo();
  logUrl = container.getLogUrl();
  containerExitStatus = container.getContainerExitStatus();
  containerState = container.getContainerState();
  nodeHttpAddress = container.getNodeHttpAddress();
}
 
Example #6
Source File: ApplicationCLI.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Lists the containers matching the given application attempts
 * 
 * @param appAttemptId
 * @throws YarnException
 * @throws IOException
 */
private void listContainers(String appAttemptId) throws YarnException,
    IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));

  List<ContainerReport> appsReport = client
      .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
  writer.println("Total number of containers " + ":" + appsReport.size());
  writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
      "Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
  for (ContainerReport containerReport : appsReport) {
    writer.printf(
        CONTAINER_PATTERN,
        containerReport.getContainerId(),
        Times.format(containerReport.getCreationTime()),
        Times.format(containerReport.getFinishTime()),      
        containerReport.getContainerState(), containerReport
            .getAssignedNode(), containerReport.getNodeHttpAddress() == null
                ? "N/A" : containerReport.getNodeHttpAddress(),
        containerReport.getLogUrl());
  }
  writer.flush();
}
 
Example #7
Source File: AppInfo.java    From Bats with Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param context
 */
public AppInfo(StramAppContext context)
{
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName();
  this.docLink = context.getApplicationDocLink();
  this.user = context.getUser().toString();
  this.startTime = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startTime, 0);
  this.appPath = context.getApplicationPath();
  this.appMasterTrackingUrl = context.getAppMasterTrackingUrl();
  this.stats = context.getStats();
  this.gatewayAddress = context.getGatewayAddress();
  this.version = VersionInfo.APEX_VERSION.getBuildVersion();
  this.attributes = new TreeMap<>();
  for (Map.Entry<Attribute<Object>, Object> entry : AttributeMap.AttributeInitializer.getAllAttributes(context, DAGContext.class).entrySet()) {
    this.attributes.put(entry.getKey().getSimpleName(), entry.getKey().codec.toString(entry.getValue()));
  }
  this.gatewayConnected = context.isGatewayConnected();
  this.appDataSources = context.getAppDataSources();
  this.metrics = context.getMetrics();
}
 
Example #8
Source File: TaskAttemptInfo.java    From big-c with Apache License 2.0 6 votes vote down vote up
public TaskAttemptInfo(TaskAttempt ta, TaskType type, Boolean isRunning) {
  final TaskAttemptReport report = ta.getReport();
  this.type = type.toString();
  this.id = MRApps.toString(ta.getID());
  this.nodeHttpAddress = ta.getNodeHttpAddress();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.assignedContainerId = ConverterUtils.toString(report.getContainerId());
  this.assignedContainer = report.getContainerId();
  this.progress = report.getProgress() * 100;
  this.status = report.getStateString();
  this.state = report.getTaskAttemptState();
  this.elapsedTime = Times
      .elapsed(this.startTime, this.finishTime, isRunning);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.diagnostics = report.getDiagnosticInfo();
  this.rack = ta.getNodeRackName();
}
 
Example #9
Source File: ReduceTaskAttemptInfo.java    From big-c with Apache License 2.0 6 votes vote down vote up
public ReduceTaskAttemptInfo(TaskAttempt ta, TaskType type) {
  super(ta, type, false);

  this.shuffleFinishTime = ta.getShuffleFinishTime();
  this.mergeFinishTime = ta.getSortFinishTime();
  this.elapsedShuffleTime = Times.elapsed(this.startTime,
      this.shuffleFinishTime, false);
  if (this.elapsedShuffleTime == -1) {
    this.elapsedShuffleTime = 0;
  }
  this.elapsedMergeTime = Times.elapsed(this.shuffleFinishTime,
      this.mergeFinishTime, false);
  if (this.elapsedMergeTime == -1) {
    this.elapsedMergeTime = 0;
  }
  this.elapsedReduceTime = Times.elapsed(this.mergeFinishTime,
      this.finishTime, false);
  if (this.elapsedReduceTime == -1) {
    this.elapsedReduceTime = 0;
  }
}
 
Example #10
Source File: TaskInfo.java    From big-c with Apache License 2.0 6 votes vote down vote up
public TaskInfo(Task task) {
  TaskType ttype = task.getType();
  this.type = ttype.toString();
  TaskReport report = task.getReport();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.state = report.getTaskState();
  this.elapsedTime = Times.elapsed(this.startTime, this.finishTime,
    this.state == TaskState.RUNNING);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.progress = report.getProgress() * 100;
  this.status =  report.getStatus();
  this.id = MRApps.toString(task.getID());
  this.taskNum = task.getID().getId();
  this.successful = getSuccessfulAttempt(task);
  if (successful != null) {
    this.successfulAttempt = MRApps.toString(successful.getID());
  } else {
    this.successfulAttempt = "";
  }
}
 
Example #11
Source File: TaskInfo.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public TaskInfo(Task task) {
  TaskType ttype = task.getType();
  this.type = ttype.toString();
  TaskReport report = task.getReport();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.state = report.getTaskState();
  this.elapsedTime = Times.elapsed(this.startTime, this.finishTime,
    this.state == TaskState.RUNNING);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.progress = report.getProgress() * 100;
  this.status =  report.getStatus();
  this.id = MRApps.toString(task.getID());
  this.taskNum = task.getID().getId();
  this.successful = getSuccessfulAttempt(task);
  if (successful != null) {
    this.successfulAttempt = MRApps.toString(successful.getID());
  } else {
    this.successfulAttempt = "";
  }
}
 
Example #12
Source File: TaskAttemptInfo.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public TaskAttemptInfo(TaskAttempt ta, TaskType type, Boolean isRunning) {
  final TaskAttemptReport report = ta.getReport();
  this.type = type.toString();
  this.id = MRApps.toString(ta.getID());
  this.nodeHttpAddress = ta.getNodeHttpAddress();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.assignedContainerId = ConverterUtils.toString(report.getContainerId());
  this.assignedContainer = report.getContainerId();
  this.progress = report.getProgress() * 100;
  this.status = report.getStateString();
  this.state = report.getTaskAttemptState();
  this.elapsedTime = Times
      .elapsed(this.startTime, this.finishTime, isRunning);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.diagnostics = report.getDiagnosticInfo();
  this.rack = ta.getNodeRackName();
}
 
Example #13
Source File: ApplicationCLI.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Lists the containers matching the given application attempts
 * 
 * @param appAttemptId
 * @throws YarnException
 * @throws IOException
 */
private void listContainers(String appAttemptId) throws YarnException,
    IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));

  List<ContainerReport> appsReport = client
      .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
  writer.println("Total number of containers " + ":" + appsReport.size());
  writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
      "Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
  for (ContainerReport containerReport : appsReport) {
    writer.printf(
        CONTAINER_PATTERN,
        containerReport.getContainerId(),
        Times.format(containerReport.getCreationTime()),
        Times.format(containerReport.getFinishTime()),      
        containerReport.getContainerState(), containerReport
            .getAssignedNode(), containerReport.getNodeHttpAddress() == null
                ? "N/A" : containerReport.getNodeHttpAddress(),
        containerReport.getLogUrl());
  }
  writer.flush();
}
 
Example #14
Source File: AboutBlock.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
protected void render(Block html) {
  html._(MetricsOverviewTable.class);
  ResourceManager rm = getInstance(ResourceManager.class);
  ClusterInfo cinfo = new ClusterInfo(rm);
  info("Cluster overview").
    _("Cluster ID:", cinfo.getClusterId()).
    _("ResourceManager state:", cinfo.getState()).
    _("ResourceManager HA state:", cinfo.getHAState()).
    _("ResourceManager RMStateStore:", cinfo.getRMStateStore()).
    _("ResourceManager started on:", Times.format(cinfo.getStartedOn())).
    _("ResourceManager version:", cinfo.getRMBuildVersion() +
        " on " + cinfo.getRMVersionBuiltOn()).
    _("Hadoop version:", cinfo.getHadoopBuildVersion() +
        " on " + cinfo.getHadoopVersionBuiltOn());
  html._(InfoBlock.class);
}
 
Example #15
Source File: SCMOverviewPage.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
protected void render(Block html) {
  SCMMetricsInfo metricsInfo = new SCMMetricsInfo(
      CleanerMetrics.getInstance(), ClientSCMMetrics.getInstance(),
          SharedCacheUploaderMetrics.getInstance());
  info("Shared Cache Manager overview").
      _("Started on:", Times.format(scm.getStartTime())).
      _("Cache hits: ", metricsInfo.getCacheHits()).
      _("Cache misses: ", metricsInfo.getCacheMisses()).
      _("Cache releases: ", metricsInfo.getCacheReleases()).
      _("Accepted uploads: ", metricsInfo.getAcceptedUploads()).
      _("Rejected uploads: ", metricsInfo.getRejectUploads()).
      _("Deleted files by the cleaner: ", metricsInfo.getTotalDeletedFiles()).
      _("Processed files by the cleaner: ", metricsInfo.getTotalProcessedFiles());
  html._(InfoBlock.class);
}
 
Example #16
Source File: AppInfo.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public AppInfo(ApplicationReport app) {
  appId = app.getApplicationId().toString();
  if (app.getCurrentApplicationAttemptId() != null) {
    currentAppAttemptId = app.getCurrentApplicationAttemptId().toString();
  }
  user = app.getUser();
  queue = app.getQueue();
  name = app.getName();
  type = app.getApplicationType();
  host = app.getHost();
  rpcPort = app.getRpcPort();
  appState = app.getYarnApplicationState();
  diagnosticsInfo = app.getDiagnostics();
  trackingUrl = app.getTrackingUrl();
  originalTrackingUrl = app.getOriginalTrackingUrl();
  submittedTime = app.getStartTime();
  startedTime = app.getStartTime();
  finishedTime = app.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  finalAppStatus = app.getFinalApplicationStatus();
  progress = app.getProgress() * 100; // in percent
  if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
    this.applicationTags = CSV_JOINER.join(app.getApplicationTags());
  }
}
 
Example #17
Source File: ContainerInfo.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public ContainerInfo(ContainerReport container) {
  containerId = container.getContainerId().toString();
  if (container.getAllocatedResource() != null) {
    allocatedMB = container.getAllocatedResource().getMemory();
    allocatedVCores = container.getAllocatedResource().getVirtualCores();
    allocatedGCores = container.getAllocatedResource().getGpuCores();
  }
  if (container.getAssignedNode() != null) {
    assignedNodeId = container.getAssignedNode().toString();
  }
  priority = container.getPriority().getPriority();
  startedTime = container.getCreationTime();
  finishedTime = container.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  diagnosticsInfo = container.getDiagnosticsInfo();
  logUrl = container.getLogUrl();
  containerExitStatus = container.getContainerExitStatus();
  containerState = container.getContainerState();
  nodeHttpAddress = container.getNodeHttpAddress();
}
 
Example #18
Source File: AppInfo.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param context
 */
public AppInfo(StramAppContext context)
{
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName();
  this.docLink = context.getApplicationDocLink();
  this.user = context.getUser().toString();
  this.startTime = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startTime, 0);
  this.appPath = context.getApplicationPath();
  this.appMasterTrackingUrl = context.getAppMasterTrackingUrl();
  this.stats = context.getStats();
  this.gatewayAddress = context.getGatewayAddress();
  this.version = VersionInfo.APEX_VERSION.getBuildVersion();
  this.attributes = new TreeMap<>();
  for (Map.Entry<Attribute<Object>, Object> entry : AttributeMap.AttributeInitializer.getAllAttributes(context, DAGContext.class).entrySet()) {
    this.attributes.put(entry.getKey().getSimpleName(), entry.getKey().codec.toString(entry.getValue()));
  }
  this.gatewayConnected = context.isGatewayConnected();
  this.appDataSources = context.getAppDataSources();
  this.metrics = context.getMetrics();
}
 
Example #19
Source File: AggregatedLogFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static void readContainerLogs(DataInputStream valueStream,
    PrintStream out, long logUploadedTime) throws IOException {
  byte[] buf = new byte[65535];

  String fileType = valueStream.readUTF();
  String fileLengthStr = valueStream.readUTF();
  long fileLength = Long.parseLong(fileLengthStr);
  out.print("LogType:");
  out.println(fileType);
  if (logUploadedTime != -1) {
    out.print("Log Upload Time:");
    out.println(Times.format(logUploadedTime));
  }
  out.print("LogLength:");
  out.println(fileLengthStr);
  out.println("Log Contents:");

  long curRead = 0;
  long pendingRead = fileLength - curRead;
  int toRead =
            pendingRead > buf.length ? buf.length : (int) pendingRead;
  int len = valueStream.read(buf, 0, toRead);
  while (len != -1 && curRead < fileLength) {
    out.write(buf, 0, len);
    curRead += len;

    pendingRead = fileLength - curRead;
    toRead =
              pendingRead > buf.length ? buf.length : (int) pendingRead;
    len = valueStream.read(buf, 0, toRead);
  }
  out.println("End of LogType:" + fileType);
  out.println("");
}
 
Example #20
Source File: AppInfo.java    From big-c with Apache License 2.0 5 votes vote down vote up
public AppInfo(App app, AppContext context) {
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName().toString();
  this.user = context.getUser().toString();
  this.startedOn = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startedOn, 0);
}
 
Example #21
Source File: TestAMWebServicesJobs.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void verifyAMJobGeneric(Job job, String id, String user, String name,
    String state, long startTime, long finishTime, long elapsedTime,
    int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted,
    float reduceProgress, float mapProgress) {
  JobReport report = job.getReport();

  WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
      id);
  WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
      user);
  WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
  WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
      state);

  assertEquals("startTime incorrect", report.getStartTime(), startTime);
  assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
  assertEquals("elapsedTime incorrect",
      Times.elapsed(report.getStartTime(), report.getFinishTime()),
      elapsedTime);
  assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
  assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
      mapsCompleted);
  assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
  assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
      reducesCompleted);
  assertEquals("mapProgress incorrect", report.getMapProgress() * 100,
      mapProgress, 0);
  assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100,
      reduceProgress, 0);
}
 
Example #22
Source File: HsAboutPage.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * The content of this page is the attempts block
 * @return AttemptsBlock.class
 */
@Override protected Class<? extends SubView> content() {
  HistoryInfo info = new HistoryInfo();
  info("History Server").
    _("BuildVersion", info.getHadoopBuildVersion()
      + " on " + info.getHadoopVersionBuiltOn()).
    _("History Server started on", Times.format(info.getStartedOn()));
  return InfoBlock.class;
}
 
Example #23
Source File: HsAboutPage.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * The content of this page is the attempts block
 * @return AttemptsBlock.class
 */
@Override protected Class<? extends SubView> content() {
  HistoryInfo info = new HistoryInfo();
  info("History Server").
    _("BuildVersion", info.getHadoopBuildVersion()
      + " on " + info.getHadoopVersionBuiltOn()).
    _("History Server started on", Times.format(info.getStartedOn()));
  return InfoBlock.class;
}
 
Example #24
Source File: TestAMWebServicesJobs.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void verifyAMJobGeneric(Job job, String id, String user, String name,
    String state, long startTime, long finishTime, long elapsedTime,
    int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted,
    float reduceProgress, float mapProgress) {
  JobReport report = job.getReport();

  WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
      id);
  WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
      user);
  WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
  WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
      state);

  assertEquals("startTime incorrect", report.getStartTime(), startTime);
  assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
  assertEquals("elapsedTime incorrect",
      Times.elapsed(report.getStartTime(), report.getFinishTime()),
      elapsedTime);
  assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
  assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
      mapsCompleted);
  assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
  assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
      reducesCompleted);
  assertEquals("mapProgress incorrect", report.getMapProgress() * 100,
      mapProgress, 0);
  assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100,
      reduceProgress, 0);
}
 
Example #25
Source File: AppInfo.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public AppInfo(App app, AppContext context) {
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName().toString();
  this.user = context.getUser().toString();
  this.startedOn = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startedOn, 0);
}
 
Example #26
Source File: AggregatedLogFormat.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static void readContainerLogs(DataInputStream valueStream,
    PrintStream out, long logUploadedTime) throws IOException {
  byte[] buf = new byte[65535];

  String fileType = valueStream.readUTF();
  String fileLengthStr = valueStream.readUTF();
  long fileLength = Long.parseLong(fileLengthStr);
  out.print("LogType:");
  out.println(fileType);
  if (logUploadedTime != -1) {
    out.print("Log Upload Time:");
    out.println(Times.format(logUploadedTime));
  }
  out.print("LogLength:");
  out.println(fileLengthStr);
  out.println("Log Contents:");

  long curRead = 0;
  long pendingRead = fileLength - curRead;
  int toRead =
            pendingRead > buf.length ? buf.length : (int) pendingRead;
  int len = valueStream.read(buf, 0, toRead);
  while (len != -1 && curRead < fileLength) {
    out.write(buf, 0, len);
    curRead += len;

    pendingRead = fileLength - curRead;
    toRead =
              pendingRead > buf.length ? buf.length : (int) pendingRead;
    len = valueStream.read(buf, 0, toRead);
  }
  out.println("End of LogType:" + fileType);
  out.println("");
}
 
Example #27
Source File: NodesPage.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
protected void render(Block html) {
  html._(MetricsOverviewTable.class);

  ResourceScheduler sched = rm.getResourceScheduler();
  String type = $(NODE_STATE);
  String labelFilter = $(NODE_LABEL, CommonNodeLabelsManager.ANY).trim();
  TBODY<TABLE<Hamlet>> tbody =
      html.table("#nodes").thead().tr()
          .th(".nodelabels", "Node Labels")
          .th(".rack", "Rack")
          .th(".state", "Node State")
          .th(".nodeaddress", "Node Address")
          .th(".nodehttpaddress", "Node HTTP Address")
          .th(".lastHealthUpdate", "Last health-update")
          .th(".healthReport", "Health-report")
          .th(".containers", "Containers")
          .th(".mem", "Mem Used")
          .th(".mem", "Mem Avail")
          .th(".vcores", "VCores Used")
          .th(".vcores", "VCores Avail")
          .th(".nodeManagerVersion", "Version")._()._().tbody();
  NodeState stateFilter = null;
  if (type != null && !type.isEmpty()) {
    stateFilter = NodeState.valueOf(StringUtils.toUpperCase(type));
  }
  Collection<RMNode> rmNodes = this.rm.getRMContext().getRMNodes().values();
  boolean isInactive = false;
  if (stateFilter != null) {
    switch (stateFilter) {
    case DECOMMISSIONED:
    case LOST:
    case REBOOTED:
      rmNodes = this.rm.getRMContext().getInactiveRMNodes().values();
      isInactive = true;
      break;
    default:
      LOG.debug("Unexpected state filter for inactive RM node");
    }
  }
  for (RMNode ni : rmNodes) {
    if (stateFilter != null) {
      NodeState state = ni.getState();
      if (!stateFilter.equals(state)) {
        continue;
      }
    } else {
      // No filter. User is asking for all nodes. Make sure you skip the
      // unhealthy nodes.
      if (ni.getState() == NodeState.UNHEALTHY) {
        continue;
      }
    }
    // Besides state, we need to filter label as well.
    if (!labelFilter.equals(RMNodeLabelsManager.ANY)) {
      if (labelFilter.isEmpty()) {
        // Empty label filter means only shows nodes without label
        if (!ni.getNodeLabels().isEmpty()) {
          continue;
        }
      } else if (!ni.getNodeLabels().contains(labelFilter)) {
        // Only nodes have given label can show on web page.
        continue;
      }
    }
    NodeInfo info = new NodeInfo(ni, sched);
    int usedMemory = (int) info.getUsedMemory();
    int availableMemory = (int) info.getAvailableMemory();
    TR<TBODY<TABLE<Hamlet>>> row =
        tbody.tr().td(StringUtils.join(",", info.getNodeLabels()))
            .td(info.getRack()).td(info.getState()).td(info.getNodeId());
    if (isInactive) {
      row.td()._("N/A")._();
    } else {
      String httpAddress = info.getNodeHTTPAddress();
      row.td().a("//" + httpAddress, httpAddress)._();
    }
    row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._()
        ._(Times.format(info.getLastHealthUpdate()))._()
        .td(info.getHealthReport())
        .td(String.valueOf(info.getNumContainers())).td().br()
        .$title(String.valueOf(usedMemory))._()
        ._(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().td().br()
        .$title(String.valueOf(availableMemory))._()
        ._(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._()
        .td(String.valueOf(info.getUsedVirtualCores()))
        .td(String.valueOf(info.getAvailableVirtualCores()))
        .td(ni.getNodeManagerVersion())._();
  }
  tbody._()._();
}
 
Example #28
Source File: AppInfo.java    From big-c with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
  }
}
 
Example #29
Source File: TestAggregatedLogFormat.java    From big-c with Apache License 2.0 4 votes vote down vote up
private void testReadAcontainerLog(boolean logUploadedTime) throws Exception {
  Configuration conf = new Configuration();
  File workDir = new File(testWorkDir, "testReadAcontainerLogs1");
  Path remoteAppLogFile =
      new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
  Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");
  ContainerId testContainerId = TestContainerId.newContainerId(1, 1, 1, 1);
  Path t =
      new Path(srcFileRoot, testContainerId.getApplicationAttemptId()
          .getApplicationId().toString());
  Path srcFilePath = new Path(t, testContainerId.toString());

  int numChars = 80000;

  // create a sub-folder under srcFilePath
  // and create file logs in this sub-folder.
  // We only aggregate top level files.
  // So, this log file should be ignored.
  Path subDir = new Path(srcFilePath, "subDir");
  fs.mkdirs(subDir);
  writeSrcFile(subDir, "logs", numChars);

  // create file stderr and stdout in containerLogDir
  writeSrcFile(srcFilePath, "stderr", numChars);
  writeSrcFile(srcFilePath, "stdout", numChars);

  UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
  LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);

  LogKey logKey = new LogKey(testContainerId);
  LogValue logValue =
      new LogValue(Collections.singletonList(srcFileRoot.toString()),
          testContainerId, ugi.getShortUserName());

  // When we try to open FileInputStream for stderr, it will throw out an IOException.
  // Skip the log aggregation for stderr.
  LogValue spyLogValue = spy(logValue);
  File errorFile = new File((new Path(srcFilePath, "stderr")).toString());
  doThrow(new IOException("Mock can not open FileInputStream")).when(
    spyLogValue).secureOpenFile(errorFile);

  logWriter.append(logKey, spyLogValue);
  logWriter.close();

  // make sure permission are correct on the file
  FileStatus fsStatus =  fs.getFileStatus(remoteAppLogFile);
  Assert.assertEquals("permissions on log aggregation file are wrong",  
    FsPermission.createImmutable((short) 0640), fsStatus.getPermission()); 

  LogReader logReader = new LogReader(conf, remoteAppLogFile);
  LogKey rLogKey = new LogKey();
  DataInputStream dis = logReader.next(rLogKey);
  Writer writer = new StringWriter();

  if (logUploadedTime) {
    LogReader.readAcontainerLogs(dis, writer, System.currentTimeMillis());
  } else {
    LogReader.readAcontainerLogs(dis, writer);
  }

  // We should only do the log aggregation for stdout.
  // Since we could not open the fileInputStream for stderr, this file is not
  // aggregated.
  String s = writer.toString();
  int expectedLength =
      "LogType:stdout".length()
          + (logUploadedTime ? ("\nLog Upload Time:" + Times.format(System
            .currentTimeMillis())).length() : 0)
          + ("\nLogLength:" + numChars).length()
          + "\nLog Contents:\n".length() + numChars + "\n".length()
          + "End of LogType:stdout\n".length();
  Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
  Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
  Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
  Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
  Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));
  
  StringBuilder sb = new StringBuilder();
  for (int i = 0 ; i < numChars ; i++) {
    sb.append(filler);
  }
  String expectedContent = sb.toString();
  Assert.assertTrue("Log content incorrect", s.contains(expectedContent));
  
  Assert.assertEquals(expectedLength, s.length());
}
 
Example #30
Source File: AppInfo.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          allocatedGCores = usedResources.getGpuCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    preemptedResourceGCores =
        appMetrics.getResourcePreempted().getGpuCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
    gcoreSeconds = appMetrics.getGcoreSeconds();
  }
}