Java Code Examples for org.apache.hadoop.yarn.util.Times

The following examples show how to use org.apache.hadoop.yarn.util.Times. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Bats   Source File: AppInfo.java    License: Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param context
 */
public AppInfo(StramAppContext context)
{
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName();
  this.docLink = context.getApplicationDocLink();
  this.user = context.getUser().toString();
  this.startTime = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startTime, 0);
  this.appPath = context.getApplicationPath();
  this.appMasterTrackingUrl = context.getAppMasterTrackingUrl();
  this.stats = context.getStats();
  this.gatewayAddress = context.getGatewayAddress();
  this.version = VersionInfo.APEX_VERSION.getBuildVersion();
  this.attributes = new TreeMap<>();
  for (Map.Entry<Attribute<Object>, Object> entry : AttributeMap.AttributeInitializer.getAllAttributes(context, DAGContext.class).entrySet()) {
    this.attributes.put(entry.getKey().getSimpleName(), entry.getKey().codec.toString(entry.getValue()));
  }
  this.gatewayConnected = context.isGatewayConnected();
  this.appDataSources = context.getAppDataSources();
  this.metrics = context.getMetrics();
}
 
Example 2
Source Project: hadoop   Source File: ContainerInfo.java    License: Apache License 2.0 6 votes vote down vote up
public ContainerInfo(ContainerReport container) {
  containerId = container.getContainerId().toString();
  if (container.getAllocatedResource() != null) {
    allocatedMB = container.getAllocatedResource().getMemory();
    allocatedVCores = container.getAllocatedResource().getVirtualCores();
    allocatedGCores = container.getAllocatedResource().getGpuCores();
  }
  if (container.getAssignedNode() != null) {
    assignedNodeId = container.getAssignedNode().toString();
  }
  priority = container.getPriority().getPriority();
  startedTime = container.getCreationTime();
  finishedTime = container.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  diagnosticsInfo = container.getDiagnosticsInfo();
  logUrl = container.getLogUrl();
  containerExitStatus = container.getContainerExitStatus();
  containerState = container.getContainerState();
  nodeHttpAddress = container.getNodeHttpAddress();
}
 
Example 3
Source Project: hadoop   Source File: AppInfo.java    License: Apache License 2.0 6 votes vote down vote up
public AppInfo(ApplicationReport app) {
  appId = app.getApplicationId().toString();
  if (app.getCurrentApplicationAttemptId() != null) {
    currentAppAttemptId = app.getCurrentApplicationAttemptId().toString();
  }
  user = app.getUser();
  queue = app.getQueue();
  name = app.getName();
  type = app.getApplicationType();
  host = app.getHost();
  rpcPort = app.getRpcPort();
  appState = app.getYarnApplicationState();
  diagnosticsInfo = app.getDiagnostics();
  trackingUrl = app.getTrackingUrl();
  originalTrackingUrl = app.getOriginalTrackingUrl();
  submittedTime = app.getStartTime();
  startedTime = app.getStartTime();
  finishedTime = app.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  finalAppStatus = app.getFinalApplicationStatus();
  progress = app.getProgress() * 100; // in percent
  if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
    this.applicationTags = CSV_JOINER.join(app.getApplicationTags());
  }
}
 
Example 4
Source Project: hadoop   Source File: SCMOverviewPage.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void render(Block html) {
  SCMMetricsInfo metricsInfo = new SCMMetricsInfo(
      CleanerMetrics.getInstance(), ClientSCMMetrics.getInstance(),
          SharedCacheUploaderMetrics.getInstance());
  info("Shared Cache Manager overview").
      _("Started on:", Times.format(scm.getStartTime())).
      _("Cache hits: ", metricsInfo.getCacheHits()).
      _("Cache misses: ", metricsInfo.getCacheMisses()).
      _("Cache releases: ", metricsInfo.getCacheReleases()).
      _("Accepted uploads: ", metricsInfo.getAcceptedUploads()).
      _("Rejected uploads: ", metricsInfo.getRejectUploads()).
      _("Deleted files by the cleaner: ", metricsInfo.getTotalDeletedFiles()).
      _("Processed files by the cleaner: ", metricsInfo.getTotalProcessedFiles());
  html._(InfoBlock.class);
}
 
Example 5
Source Project: hadoop   Source File: AboutBlock.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void render(Block html) {
  html._(MetricsOverviewTable.class);
  ResourceManager rm = getInstance(ResourceManager.class);
  ClusterInfo cinfo = new ClusterInfo(rm);
  info("Cluster overview").
    _("Cluster ID:", cinfo.getClusterId()).
    _("ResourceManager state:", cinfo.getState()).
    _("ResourceManager HA state:", cinfo.getHAState()).
    _("ResourceManager RMStateStore:", cinfo.getRMStateStore()).
    _("ResourceManager started on:", Times.format(cinfo.getStartedOn())).
    _("ResourceManager version:", cinfo.getRMBuildVersion() +
        " on " + cinfo.getRMVersionBuiltOn()).
    _("Hadoop version:", cinfo.getHadoopBuildVersion() +
        " on " + cinfo.getHadoopVersionBuiltOn());
  html._(InfoBlock.class);
}
 
Example 6
Source Project: hadoop   Source File: ApplicationCLI.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Lists the containers matching the given application attempts
 * 
 * @param appAttemptId
 * @throws YarnException
 * @throws IOException
 */
private void listContainers(String appAttemptId) throws YarnException,
    IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));

  List<ContainerReport> appsReport = client
      .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
  writer.println("Total number of containers " + ":" + appsReport.size());
  writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
      "Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
  for (ContainerReport containerReport : appsReport) {
    writer.printf(
        CONTAINER_PATTERN,
        containerReport.getContainerId(),
        Times.format(containerReport.getCreationTime()),
        Times.format(containerReport.getFinishTime()),      
        containerReport.getContainerState(), containerReport
            .getAssignedNode(), containerReport.getNodeHttpAddress() == null
                ? "N/A" : containerReport.getNodeHttpAddress(),
        containerReport.getLogUrl());
  }
  writer.flush();
}
 
Example 7
Source Project: hadoop   Source File: TaskAttemptInfo.java    License: Apache License 2.0 6 votes vote down vote up
public TaskAttemptInfo(TaskAttempt ta, TaskType type, Boolean isRunning) {
  final TaskAttemptReport report = ta.getReport();
  this.type = type.toString();
  this.id = MRApps.toString(ta.getID());
  this.nodeHttpAddress = ta.getNodeHttpAddress();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.assignedContainerId = ConverterUtils.toString(report.getContainerId());
  this.assignedContainer = report.getContainerId();
  this.progress = report.getProgress() * 100;
  this.status = report.getStateString();
  this.state = report.getTaskAttemptState();
  this.elapsedTime = Times
      .elapsed(this.startTime, this.finishTime, isRunning);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.diagnostics = report.getDiagnosticInfo();
  this.rack = ta.getNodeRackName();
}
 
Example 8
Source Project: hadoop   Source File: ReduceTaskAttemptInfo.java    License: Apache License 2.0 6 votes vote down vote up
public ReduceTaskAttemptInfo(TaskAttempt ta, TaskType type) {
  super(ta, type, false);

  this.shuffleFinishTime = ta.getShuffleFinishTime();
  this.mergeFinishTime = ta.getSortFinishTime();
  this.elapsedShuffleTime = Times.elapsed(this.startTime,
      this.shuffleFinishTime, false);
  if (this.elapsedShuffleTime == -1) {
    this.elapsedShuffleTime = 0;
  }
  this.elapsedMergeTime = Times.elapsed(this.shuffleFinishTime,
      this.mergeFinishTime, false);
  if (this.elapsedMergeTime == -1) {
    this.elapsedMergeTime = 0;
  }
  this.elapsedReduceTime = Times.elapsed(this.mergeFinishTime,
      this.finishTime, false);
  if (this.elapsedReduceTime == -1) {
    this.elapsedReduceTime = 0;
  }
}
 
Example 9
Source Project: hadoop   Source File: TaskInfo.java    License: Apache License 2.0 6 votes vote down vote up
public TaskInfo(Task task) {
  TaskType ttype = task.getType();
  this.type = ttype.toString();
  TaskReport report = task.getReport();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.state = report.getTaskState();
  this.elapsedTime = Times.elapsed(this.startTime, this.finishTime,
    this.state == TaskState.RUNNING);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.progress = report.getProgress() * 100;
  this.status =  report.getStatus();
  this.id = MRApps.toString(task.getID());
  this.taskNum = task.getID().getId();
  this.successful = getSuccessfulAttempt(task);
  if (successful != null) {
    this.successfulAttempt = MRApps.toString(successful.getID());
  } else {
    this.successfulAttempt = "";
  }
}
 
Example 10
Source Project: big-c   Source File: ContainerInfo.java    License: Apache License 2.0 6 votes vote down vote up
public ContainerInfo(ContainerReport container) {
  containerId = container.getContainerId().toString();
  if (container.getAllocatedResource() != null) {
    allocatedMB = container.getAllocatedResource().getMemory();
    allocatedVCores = container.getAllocatedResource().getVirtualCores();
  }
  if (container.getAssignedNode() != null) {
    assignedNodeId = container.getAssignedNode().toString();
  }
  priority = container.getPriority().getPriority();
  startedTime = container.getCreationTime();
  finishedTime = container.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  diagnosticsInfo = container.getDiagnosticsInfo();
  logUrl = container.getLogUrl();
  containerExitStatus = container.getContainerExitStatus();
  containerState = container.getContainerState();
  nodeHttpAddress = container.getNodeHttpAddress();
}
 
Example 11
Source Project: big-c   Source File: AppInfo.java    License: Apache License 2.0 6 votes vote down vote up
public AppInfo(ApplicationReport app) {
  appId = app.getApplicationId().toString();
  if (app.getCurrentApplicationAttemptId() != null) {
    currentAppAttemptId = app.getCurrentApplicationAttemptId().toString();
  }
  user = app.getUser();
  queue = app.getQueue();
  name = app.getName();
  type = app.getApplicationType();
  host = app.getHost();
  rpcPort = app.getRpcPort();
  appState = app.getYarnApplicationState();
  diagnosticsInfo = app.getDiagnostics();
  trackingUrl = app.getTrackingUrl();
  originalTrackingUrl = app.getOriginalTrackingUrl();
  submittedTime = app.getStartTime();
  startedTime = app.getStartTime();
  finishedTime = app.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  finalAppStatus = app.getFinalApplicationStatus();
  progress = app.getProgress() * 100; // in percent
  if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
    this.applicationTags = CSV_JOINER.join(app.getApplicationTags());
  }
}
 
Example 12
Source Project: big-c   Source File: SCMOverviewPage.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void render(Block html) {
  SCMMetricsInfo metricsInfo = new SCMMetricsInfo(
      CleanerMetrics.getInstance(), ClientSCMMetrics.getInstance(),
          SharedCacheUploaderMetrics.getInstance());
  info("Shared Cache Manager overview").
      _("Started on:", Times.format(scm.getStartTime())).
      _("Cache hits: ", metricsInfo.getCacheHits()).
      _("Cache misses: ", metricsInfo.getCacheMisses()).
      _("Cache releases: ", metricsInfo.getCacheReleases()).
      _("Accepted uploads: ", metricsInfo.getAcceptedUploads()).
      _("Rejected uploads: ", metricsInfo.getRejectUploads()).
      _("Deleted files by the cleaner: ", metricsInfo.getTotalDeletedFiles()).
      _("Processed files by the cleaner: ", metricsInfo.getTotalProcessedFiles());
  html._(InfoBlock.class);
}
 
Example 13
Source Project: big-c   Source File: AboutBlock.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void render(Block html) {
  html._(MetricsOverviewTable.class);
  ResourceManager rm = getInstance(ResourceManager.class);
  ClusterInfo cinfo = new ClusterInfo(rm);
  info("Cluster overview").
    _("Cluster ID:", cinfo.getClusterId()).
    _("ResourceManager state:", cinfo.getState()).
    _("ResourceManager HA state:", cinfo.getHAState()).
    _("ResourceManager RMStateStore:", cinfo.getRMStateStore()).
    _("ResourceManager started on:", Times.format(cinfo.getStartedOn())).
    _("ResourceManager version:", cinfo.getRMBuildVersion() +
        " on " + cinfo.getRMVersionBuiltOn()).
    _("Hadoop version:", cinfo.getHadoopBuildVersion() +
        " on " + cinfo.getHadoopVersionBuiltOn());
  html._(InfoBlock.class);
}
 
Example 14
Source Project: big-c   Source File: ApplicationCLI.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Lists the containers matching the given application attempts
 * 
 * @param appAttemptId
 * @throws YarnException
 * @throws IOException
 */
private void listContainers(String appAttemptId) throws YarnException,
    IOException {
  PrintWriter writer = new PrintWriter(
      new OutputStreamWriter(sysout, Charset.forName("UTF-8")));

  List<ContainerReport> appsReport = client
      .getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
  writer.println("Total number of containers " + ":" + appsReport.size());
  writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
      "Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
  for (ContainerReport containerReport : appsReport) {
    writer.printf(
        CONTAINER_PATTERN,
        containerReport.getContainerId(),
        Times.format(containerReport.getCreationTime()),
        Times.format(containerReport.getFinishTime()),      
        containerReport.getContainerState(), containerReport
            .getAssignedNode(), containerReport.getNodeHttpAddress() == null
                ? "N/A" : containerReport.getNodeHttpAddress(),
        containerReport.getLogUrl());
  }
  writer.flush();
}
 
Example 15
Source Project: big-c   Source File: TaskAttemptInfo.java    License: Apache License 2.0 6 votes vote down vote up
public TaskAttemptInfo(TaskAttempt ta, TaskType type, Boolean isRunning) {
  final TaskAttemptReport report = ta.getReport();
  this.type = type.toString();
  this.id = MRApps.toString(ta.getID());
  this.nodeHttpAddress = ta.getNodeHttpAddress();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.assignedContainerId = ConverterUtils.toString(report.getContainerId());
  this.assignedContainer = report.getContainerId();
  this.progress = report.getProgress() * 100;
  this.status = report.getStateString();
  this.state = report.getTaskAttemptState();
  this.elapsedTime = Times
      .elapsed(this.startTime, this.finishTime, isRunning);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.diagnostics = report.getDiagnosticInfo();
  this.rack = ta.getNodeRackName();
}
 
Example 16
Source Project: big-c   Source File: ReduceTaskAttemptInfo.java    License: Apache License 2.0 6 votes vote down vote up
public ReduceTaskAttemptInfo(TaskAttempt ta, TaskType type) {
  super(ta, type, false);

  this.shuffleFinishTime = ta.getShuffleFinishTime();
  this.mergeFinishTime = ta.getSortFinishTime();
  this.elapsedShuffleTime = Times.elapsed(this.startTime,
      this.shuffleFinishTime, false);
  if (this.elapsedShuffleTime == -1) {
    this.elapsedShuffleTime = 0;
  }
  this.elapsedMergeTime = Times.elapsed(this.shuffleFinishTime,
      this.mergeFinishTime, false);
  if (this.elapsedMergeTime == -1) {
    this.elapsedMergeTime = 0;
  }
  this.elapsedReduceTime = Times.elapsed(this.mergeFinishTime,
      this.finishTime, false);
  if (this.elapsedReduceTime == -1) {
    this.elapsedReduceTime = 0;
  }
}
 
Example 17
Source Project: big-c   Source File: TaskInfo.java    License: Apache License 2.0 6 votes vote down vote up
public TaskInfo(Task task) {
  TaskType ttype = task.getType();
  this.type = ttype.toString();
  TaskReport report = task.getReport();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.state = report.getTaskState();
  this.elapsedTime = Times.elapsed(this.startTime, this.finishTime,
    this.state == TaskState.RUNNING);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.progress = report.getProgress() * 100;
  this.status =  report.getStatus();
  this.id = MRApps.toString(task.getID());
  this.taskNum = task.getID().getId();
  this.successful = getSuccessfulAttempt(task);
  if (successful != null) {
    this.successfulAttempt = MRApps.toString(successful.getID());
  } else {
    this.successfulAttempt = "";
  }
}
 
Example 18
Source Project: attic-apex-core   Source File: AppInfo.java    License: Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param context
 */
public AppInfo(StramAppContext context)
{
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName();
  this.docLink = context.getApplicationDocLink();
  this.user = context.getUser().toString();
  this.startTime = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startTime, 0);
  this.appPath = context.getApplicationPath();
  this.appMasterTrackingUrl = context.getAppMasterTrackingUrl();
  this.stats = context.getStats();
  this.gatewayAddress = context.getGatewayAddress();
  this.version = VersionInfo.APEX_VERSION.getBuildVersion();
  this.attributes = new TreeMap<>();
  for (Map.Entry<Attribute<Object>, Object> entry : AttributeMap.AttributeInitializer.getAllAttributes(context, DAGContext.class).entrySet()) {
    this.attributes.put(entry.getKey().getSimpleName(), entry.getKey().codec.toString(entry.getValue()));
  }
  this.gatewayConnected = context.isGatewayConnected();
  this.appDataSources = context.getAppDataSources();
  this.metrics = context.getMetrics();
}
 
Example 19
Source Project: hadoop   Source File: AggregatedLogFormat.java    License: Apache License 2.0 5 votes vote down vote up
private static void readContainerLogs(DataInputStream valueStream,
    PrintStream out, long logUploadedTime) throws IOException {
  byte[] buf = new byte[65535];

  String fileType = valueStream.readUTF();
  String fileLengthStr = valueStream.readUTF();
  long fileLength = Long.parseLong(fileLengthStr);
  out.print("LogType:");
  out.println(fileType);
  if (logUploadedTime != -1) {
    out.print("Log Upload Time:");
    out.println(Times.format(logUploadedTime));
  }
  out.print("LogLength:");
  out.println(fileLengthStr);
  out.println("Log Contents:");

  long curRead = 0;
  long pendingRead = fileLength - curRead;
  int toRead =
            pendingRead > buf.length ? buf.length : (int) pendingRead;
  int len = valueStream.read(buf, 0, toRead);
  while (len != -1 && curRead < fileLength) {
    out.write(buf, 0, len);
    curRead += len;

    pendingRead = fileLength - curRead;
    toRead =
              pendingRead > buf.length ? buf.length : (int) pendingRead;
    len = valueStream.read(buf, 0, toRead);
  }
  out.println("End of LogType:" + fileType);
  out.println("");
}
 
Example 20
Source Project: hadoop   Source File: AppInfo.java    License: Apache License 2.0 5 votes vote down vote up
public AppInfo(App app, AppContext context) {
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName().toString();
  this.user = context.getUser().toString();
  this.startedOn = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startedOn, 0);
}
 
Example 21
Source Project: hadoop   Source File: TestAMWebServicesJobs.java    License: Apache License 2.0 5 votes vote down vote up
public void verifyAMJobGeneric(Job job, String id, String user, String name,
    String state, long startTime, long finishTime, long elapsedTime,
    int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted,
    float reduceProgress, float mapProgress) {
  JobReport report = job.getReport();

  WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
      id);
  WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
      user);
  WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
  WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
      state);

  assertEquals("startTime incorrect", report.getStartTime(), startTime);
  assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
  assertEquals("elapsedTime incorrect",
      Times.elapsed(report.getStartTime(), report.getFinishTime()),
      elapsedTime);
  assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
  assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
      mapsCompleted);
  assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
  assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
      reducesCompleted);
  assertEquals("mapProgress incorrect", report.getMapProgress() * 100,
      mapProgress, 0);
  assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100,
      reduceProgress, 0);
}
 
Example 22
Source Project: hadoop   Source File: HsAboutPage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * The content of this page is the attempts block
 * @return AttemptsBlock.class
 */
@Override protected Class<? extends SubView> content() {
  HistoryInfo info = new HistoryInfo();
  info("History Server").
    _("BuildVersion", info.getHadoopBuildVersion()
      + " on " + info.getHadoopVersionBuiltOn()).
    _("History Server started on", Times.format(info.getStartedOn()));
  return InfoBlock.class;
}
 
Example 23
Source Project: big-c   Source File: AggregatedLogFormat.java    License: Apache License 2.0 5 votes vote down vote up
private static void readContainerLogs(DataInputStream valueStream,
    PrintStream out, long logUploadedTime) throws IOException {
  byte[] buf = new byte[65535];

  String fileType = valueStream.readUTF();
  String fileLengthStr = valueStream.readUTF();
  long fileLength = Long.parseLong(fileLengthStr);
  out.print("LogType:");
  out.println(fileType);
  if (logUploadedTime != -1) {
    out.print("Log Upload Time:");
    out.println(Times.format(logUploadedTime));
  }
  out.print("LogLength:");
  out.println(fileLengthStr);
  out.println("Log Contents:");

  long curRead = 0;
  long pendingRead = fileLength - curRead;
  int toRead =
            pendingRead > buf.length ? buf.length : (int) pendingRead;
  int len = valueStream.read(buf, 0, toRead);
  while (len != -1 && curRead < fileLength) {
    out.write(buf, 0, len);
    curRead += len;

    pendingRead = fileLength - curRead;
    toRead =
              pendingRead > buf.length ? buf.length : (int) pendingRead;
    len = valueStream.read(buf, 0, toRead);
  }
  out.println("End of LogType:" + fileType);
  out.println("");
}
 
Example 24
Source Project: big-c   Source File: AppInfo.java    License: Apache License 2.0 5 votes vote down vote up
public AppInfo(App app, AppContext context) {
  this.appId = context.getApplicationID().toString();
  this.name = context.getApplicationName().toString();
  this.user = context.getUser().toString();
  this.startedOn = context.getStartTime();
  this.elapsedTime = Times.elapsed(this.startedOn, 0);
}
 
Example 25
Source Project: big-c   Source File: TestAMWebServicesJobs.java    License: Apache License 2.0 5 votes vote down vote up
public void verifyAMJobGeneric(Job job, String id, String user, String name,
    String state, long startTime, long finishTime, long elapsedTime,
    int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted,
    float reduceProgress, float mapProgress) {
  JobReport report = job.getReport();

  WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
      id);
  WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
      user);
  WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
  WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
      state);

  assertEquals("startTime incorrect", report.getStartTime(), startTime);
  assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
  assertEquals("elapsedTime incorrect",
      Times.elapsed(report.getStartTime(), report.getFinishTime()),
      elapsedTime);
  assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
  assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
      mapsCompleted);
  assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
  assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
      reducesCompleted);
  assertEquals("mapProgress incorrect", report.getMapProgress() * 100,
      mapProgress, 0);
  assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100,
      reduceProgress, 0);
}
 
Example 26
Source Project: big-c   Source File: HsAboutPage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * The content of this page is the attempts block
 * @return AttemptsBlock.class
 */
@Override protected Class<? extends SubView> content() {
  HistoryInfo info = new HistoryInfo();
  info("History Server").
    _("BuildVersion", info.getHadoopBuildVersion()
      + " on " + info.getHadoopVersionBuiltOn()).
    _("History Server started on", Times.format(info.getStartedOn()));
  return InfoBlock.class;
}
 
Example 27
Source Project: hadoop   Source File: TestAggregatedLogFormat.java    License: Apache License 2.0 4 votes vote down vote up
private void testReadAcontainerLog(boolean logUploadedTime) throws Exception {
  Configuration conf = new Configuration();
  File workDir = new File(testWorkDir, "testReadAcontainerLogs1");
  Path remoteAppLogFile =
      new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
  Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");
  ContainerId testContainerId = TestContainerId.newContainerId(1, 1, 1, 1);
  Path t =
      new Path(srcFileRoot, testContainerId.getApplicationAttemptId()
          .getApplicationId().toString());
  Path srcFilePath = new Path(t, testContainerId.toString());

  int numChars = 80000;

  // create a sub-folder under srcFilePath
  // and create file logs in this sub-folder.
  // We only aggregate top level files.
  // So, this log file should be ignored.
  Path subDir = new Path(srcFilePath, "subDir");
  fs.mkdirs(subDir);
  writeSrcFile(subDir, "logs", numChars);

  // create file stderr and stdout in containerLogDir
  writeSrcFile(srcFilePath, "stderr", numChars);
  writeSrcFile(srcFilePath, "stdout", numChars);

  UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
  LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);

  LogKey logKey = new LogKey(testContainerId);
  LogValue logValue =
      new LogValue(Collections.singletonList(srcFileRoot.toString()),
          testContainerId, ugi.getShortUserName());

  // When we try to open FileInputStream for stderr, it will throw out an IOException.
  // Skip the log aggregation for stderr.
  LogValue spyLogValue = spy(logValue);
  File errorFile = new File((new Path(srcFilePath, "stderr")).toString());
  doThrow(new IOException("Mock can not open FileInputStream")).when(
    spyLogValue).secureOpenFile(errorFile);

  logWriter.append(logKey, spyLogValue);
  logWriter.close();

  // make sure permission are correct on the file
  FileStatus fsStatus =  fs.getFileStatus(remoteAppLogFile);
  Assert.assertEquals("permissions on log aggregation file are wrong",  
    FsPermission.createImmutable((short) 0640), fsStatus.getPermission()); 

  LogReader logReader = new LogReader(conf, remoteAppLogFile);
  LogKey rLogKey = new LogKey();
  DataInputStream dis = logReader.next(rLogKey);
  Writer writer = new StringWriter();

  if (logUploadedTime) {
    LogReader.readAcontainerLogs(dis, writer, System.currentTimeMillis());
  } else {
    LogReader.readAcontainerLogs(dis, writer);
  }

  // We should only do the log aggregation for stdout.
  // Since we could not open the fileInputStream for stderr, this file is not
  // aggregated.
  String s = writer.toString();
  int expectedLength =
      "LogType:stdout".length()
          + (logUploadedTime ? ("\nLog Upload Time:" + Times.format(System
            .currentTimeMillis())).length() : 0)
          + ("\nLogLength:" + numChars).length()
          + "\nLog Contents:\n".length() + numChars + "\n".length()
          + "End of LogType:stdout\n".length();
  Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
  Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
  Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
  Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
  Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));
  
  StringBuilder sb = new StringBuilder();
  for (int i = 0 ; i < numChars ; i++) {
    sb.append(filler);
  }
  String expectedContent = sb.toString();
  Assert.assertTrue("Log content incorrect", s.contains(expectedContent));
  
  Assert.assertEquals(expectedLength, s.length());
}
 
Example 28
Source Project: hadoop   Source File: NodesPage.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void render(Block html) {
  html._(MetricsOverviewTable.class);

  ResourceScheduler sched = rm.getResourceScheduler();
  String type = $(NODE_STATE);
  String labelFilter = $(NODE_LABEL, CommonNodeLabelsManager.ANY).trim();
  TBODY<TABLE<Hamlet>> tbody =
      html.table("#nodes").thead().tr()
          .th(".nodelabels", "Node Labels")
          .th(".rack", "Rack")
          .th(".state", "Node State")
          .th(".nodeaddress", "Node Address")
          .th(".nodehttpaddress", "Node HTTP Address")
          .th(".lastHealthUpdate", "Last health-update")
          .th(".healthReport", "Health-report")
          .th(".containers", "Containers")
          .th(".mem", "Mem Used")
          .th(".mem", "Mem Avail")
          .th(".vcores", "VCores Used")
          .th(".vcores", "VCores Avail")
          .th(".gcores", "GCores Used")
          .th(".gcores", "GCores Avail")
          .th(".nodeManagerVersion", "Version")._()._().tbody();
  NodeState stateFilter = null;
  if (type != null && !type.isEmpty()) {
    stateFilter = NodeState.valueOf(StringUtils.toUpperCase(type));
  }
  Collection<RMNode> rmNodes = this.rm.getRMContext().getRMNodes().values();
  boolean isInactive = false;
  if (stateFilter != null) {
    switch (stateFilter) {
    case DECOMMISSIONED:
    case LOST:
    case REBOOTED:
      rmNodes = this.rm.getRMContext().getInactiveRMNodes().values();
      isInactive = true;
      break;
    default:
      LOG.debug("Unexpected state filter for inactive RM node");
    }
  }
  for (RMNode ni : rmNodes) {
    if (stateFilter != null) {
      NodeState state = ni.getState();
      if (!stateFilter.equals(state)) {
        continue;
      }
    } else {
      // No filter. User is asking for all nodes. Make sure you skip the
      // unhealthy nodes.
      if (ni.getState() == NodeState.UNHEALTHY) {
        continue;
      }
    }
    // Besides state, we need to filter label as well.
    if (!labelFilter.equals(RMNodeLabelsManager.ANY)) {
      if (labelFilter.isEmpty()) {
        // Empty label filter means only shows nodes without label
        if (!ni.getNodeLabels().isEmpty()) {
          continue;
        }
      } else if (!ni.getNodeLabels().contains(labelFilter)) {
        // Only nodes have given label can show on web page.
        continue;
      }
    }
    NodeInfo info = new NodeInfo(ni, sched);
    int usedMemory = (int) info.getUsedMemory();
    int availableMemory = (int) info.getAvailableMemory();
    TR<TBODY<TABLE<Hamlet>>> row =
        tbody.tr().td(StringUtils.join(",", info.getNodeLabels()))
            .td(info.getRack()).td(info.getState()).td(info.getNodeId());
    if (isInactive) {
      row.td()._("N/A")._();
    } else {
      String httpAddress = info.getNodeHTTPAddress();
      row.td().a("//" + httpAddress, httpAddress)._();
    }
    row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._()
        ._(Times.format(info.getLastHealthUpdate()))._()
        .td(info.getHealthReport())
        .td(String.valueOf(info.getNumContainers())).td().br()
        .$title(String.valueOf(usedMemory))._()
        ._(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().td().br()
        .$title(String.valueOf(availableMemory))._()
        ._(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._()
        .td(String.valueOf(info.getUsedVirtualCores()))
        .td(String.valueOf(info.getAvailableVirtualCores()))
        .td(String.valueOf(info.getUsedGpuCores()))
        .td(String.valueOf(info.getAvailableGpuCores()))
        .td(ni.getNodeManagerVersion())._();
  }
  tbody._()._();
}
 
Example 29
Source Project: hadoop   Source File: AppInfo.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          allocatedGCores = usedResources.getGpuCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    preemptedResourceGCores =
        appMetrics.getResourcePreempted().getGpuCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
    gcoreSeconds = appMetrics.getGcoreSeconds();
  }
}
 
Example 30
Source Project: big-c   Source File: TestAggregatedLogFormat.java    License: Apache License 2.0 4 votes vote down vote up
private void testReadAcontainerLog(boolean logUploadedTime) throws Exception {
  Configuration conf = new Configuration();
  File workDir = new File(testWorkDir, "testReadAcontainerLogs1");
  Path remoteAppLogFile =
      new Path(workDir.getAbsolutePath(), "aggregatedLogFile");
  Path srcFileRoot = new Path(workDir.getAbsolutePath(), "srcFiles");
  ContainerId testContainerId = TestContainerId.newContainerId(1, 1, 1, 1);
  Path t =
      new Path(srcFileRoot, testContainerId.getApplicationAttemptId()
          .getApplicationId().toString());
  Path srcFilePath = new Path(t, testContainerId.toString());

  int numChars = 80000;

  // create a sub-folder under srcFilePath
  // and create file logs in this sub-folder.
  // We only aggregate top level files.
  // So, this log file should be ignored.
  Path subDir = new Path(srcFilePath, "subDir");
  fs.mkdirs(subDir);
  writeSrcFile(subDir, "logs", numChars);

  // create file stderr and stdout in containerLogDir
  writeSrcFile(srcFilePath, "stderr", numChars);
  writeSrcFile(srcFilePath, "stdout", numChars);

  UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
  LogWriter logWriter = new LogWriter(conf, remoteAppLogFile, ugi);

  LogKey logKey = new LogKey(testContainerId);
  LogValue logValue =
      new LogValue(Collections.singletonList(srcFileRoot.toString()),
          testContainerId, ugi.getShortUserName());

  // When we try to open FileInputStream for stderr, it will throw out an IOException.
  // Skip the log aggregation for stderr.
  LogValue spyLogValue = spy(logValue);
  File errorFile = new File((new Path(srcFilePath, "stderr")).toString());
  doThrow(new IOException("Mock can not open FileInputStream")).when(
    spyLogValue).secureOpenFile(errorFile);

  logWriter.append(logKey, spyLogValue);
  logWriter.close();

  // make sure permission are correct on the file
  FileStatus fsStatus =  fs.getFileStatus(remoteAppLogFile);
  Assert.assertEquals("permissions on log aggregation file are wrong",  
    FsPermission.createImmutable((short) 0640), fsStatus.getPermission()); 

  LogReader logReader = new LogReader(conf, remoteAppLogFile);
  LogKey rLogKey = new LogKey();
  DataInputStream dis = logReader.next(rLogKey);
  Writer writer = new StringWriter();

  if (logUploadedTime) {
    LogReader.readAcontainerLogs(dis, writer, System.currentTimeMillis());
  } else {
    LogReader.readAcontainerLogs(dis, writer);
  }

  // We should only do the log aggregation for stdout.
  // Since we could not open the fileInputStream for stderr, this file is not
  // aggregated.
  String s = writer.toString();
  int expectedLength =
      "LogType:stdout".length()
          + (logUploadedTime ? ("\nLog Upload Time:" + Times.format(System
            .currentTimeMillis())).length() : 0)
          + ("\nLogLength:" + numChars).length()
          + "\nLog Contents:\n".length() + numChars + "\n".length()
          + "End of LogType:stdout\n".length();
  Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
  Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
  Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
  Assert.assertTrue("LogLength not matched", s.contains("LogLength:" + numChars));
  Assert.assertTrue("Log Contents not matched", s.contains("Log Contents"));
  
  StringBuilder sb = new StringBuilder();
  for (int i = 0 ; i < numChars ; i++) {
    sb.append(filler);
  }
  String expectedContent = sb.toString();
  Assert.assertTrue("Log content incorrect", s.contains(expectedContent));
  
  Assert.assertEquals(expectedLength, s.length());
}