Java Code Examples for org.apache.hadoop.mapreduce.v2.app.job.Job#getReport()

The following examples show how to use org.apache.hadoop.mapreduce.v2.app.job.Job#getReport() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MRApp.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public void waitForState(Job job, JobState finalState) throws Exception {
  int timeoutSecs = 0;
  JobReport report = job.getReport();
  while (!finalState.equals(report.getJobState()) &&
      timeoutSecs++ < 20) {
    System.out.println("Job State is : " + report.getJobState() +
        " Waiting for state : " + finalState +
        "   map progress : " + report.getMapProgress() + 
        "   reduce progress : " + report.getReduceProgress());
    report = job.getReport();
    Thread.sleep(500);
  }
  System.out.println("Job State is : " + report.getJobState());
  Assert.assertEquals("Job state is not correct (timedout)", finalState, 
      job.getState());
}
Example 2
Source File: MRApp.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public void verifyCompleted() {
  for (Job job : getContext().getAllJobs().values()) {
    JobReport jobReport = job.getReport();
    System.out.println("Job start time :" + jobReport.getStartTime());
    System.out.println("Job finish time :" + jobReport.getFinishTime());
    Assert.assertTrue("Job start time is not less than finish time",
        jobReport.getStartTime() <= jobReport.getFinishTime());
    Assert.assertTrue("Job finish time is in future",
        jobReport.getFinishTime() <= System.currentTimeMillis());
    for (Task task : job.getTasks().values()) {
      TaskReport taskReport = task.getReport();
      System.out.println("Task start time : " + taskReport.getStartTime());
      System.out.println("Task finish time : " + taskReport.getFinishTime());
      Assert.assertTrue("Task start time is not less than finish time",
          taskReport.getStartTime() <= taskReport.getFinishTime());
      for (TaskAttempt attempt : task.getAttempts().values()) {
        TaskAttemptReport attemptReport = attempt.getReport();
        Assert.assertTrue("Attempt start time is not less than finish time",
            attemptReport.getStartTime() <= attemptReport.getFinishTime());
      }
    }
  }
}
Example 3
Source File: VerifyJobsUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static void verifyHsJobGeneric(Job job, String id, String user,
    String name, String state, String queue, long startTime, long finishTime,
    int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted) {
  JobReport report = job.getReport();

  WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
      id);
  WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
      user);
  WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
  WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
      state);
  WebServicesTestUtils.checkStringMatch("queue", job.getQueueName(), queue);

  assertEquals("startTime incorrect", report.getStartTime(), startTime);
  assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);

  assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
  assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
      mapsCompleted);
  assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
  assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
      reducesCompleted);
}
Example 4
Source File: MRApp.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public void waitForState(Job job, JobState finalState) throws Exception {
  int timeoutSecs = 0;
  JobReport report = job.getReport();
  while (!finalState.equals(report.getJobState()) &&
      timeoutSecs++ < 20) {
    System.out.println("Job State is : " + report.getJobState() +
        " Waiting for state : " + finalState +
        "   map progress : " + report.getMapProgress() + 
        "   reduce progress : " + report.getReduceProgress());
    report = job.getReport();
    Thread.sleep(500);
  }
  System.out.println("Job State is : " + report.getJobState());
  Assert.assertEquals("Job state is not correct (timedout)", finalState, 
      job.getState());
}
Example 5
Source File: MRApp.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public void verifyCompleted() {
  for (Job job : getContext().getAllJobs().values()) {
    JobReport jobReport = job.getReport();
    System.out.println("Job start time :" + jobReport.getStartTime());
    System.out.println("Job finish time :" + jobReport.getFinishTime());
    Assert.assertTrue("Job start time is not less than finish time",
        jobReport.getStartTime() <= jobReport.getFinishTime());
    Assert.assertTrue("Job finish time is in future",
        jobReport.getFinishTime() <= System.currentTimeMillis());
    for (Task task : job.getTasks().values()) {
      TaskReport taskReport = task.getReport();
      System.out.println("Task start time : " + taskReport.getStartTime());
      System.out.println("Task finish time : " + taskReport.getFinishTime());
      Assert.assertTrue("Task start time is not less than finish time",
          taskReport.getStartTime() <= taskReport.getFinishTime());
      for (TaskAttempt attempt : task.getAttempts().values()) {
        TaskAttemptReport attemptReport = attempt.getReport();
        Assert.assertTrue("Attempt start time is not less than finish time",
            attemptReport.getStartTime() <= attemptReport.getFinishTime());
      }
    }
  }
}
Example 6
Source File: VerifyJobsUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static void verifyHsJobGeneric(Job job, String id, String user,
    String name, String state, String queue, long startTime, long finishTime,
    int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted) {
  JobReport report = job.getReport();

  WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
      id);
  WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
      user);
  WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
  WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
      state);
  WebServicesTestUtils.checkStringMatch("queue", job.getQueueName(), queue);

  assertEquals("startTime incorrect", report.getStartTime(), startTime);
  assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);

  assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
  assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
      mapsCompleted);
  assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
  assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
      reducesCompleted);
}
Example 7
Source File: TestAMWebServicesJobs.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public void verifyAMJobGeneric(Job job, String id, String user, String name,
    String state, long startTime, long finishTime, long elapsedTime,
    int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted,
    float reduceProgress, float mapProgress) {
  JobReport report = job.getReport();

  WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
      id);
  WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
      user);
  WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
  WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
      state);

  assertEquals("startTime incorrect", report.getStartTime(), startTime);
  assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
  assertEquals("elapsedTime incorrect",
      Times.elapsed(report.getStartTime(), report.getFinishTime()),
      elapsedTime);
  assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
  assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
      mapsCompleted);
  assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
  assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
      reducesCompleted);
  assertEquals("mapProgress incorrect", report.getMapProgress() * 100,
      mapProgress, 0);
  assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100,
      reduceProgress, 0);
}
Example 8
Source File: TestAMWebServicesJobs.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public void verifyAMJobGeneric(Job job, String id, String user, String name,
    String state, long startTime, long finishTime, long elapsedTime,
    int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted,
    float reduceProgress, float mapProgress) {
  JobReport report = job.getReport();

  WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()),
      id);
  WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(),
      user);
  WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
  WebServicesTestUtils.checkStringMatch("state", job.getState().toString(),
      state);

  assertEquals("startTime incorrect", report.getStartTime(), startTime);
  assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
  assertEquals("elapsedTime incorrect",
      Times.elapsed(report.getStartTime(), report.getFinishTime()),
      elapsedTime);
  assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
  assertEquals("mapsCompleted incorrect", job.getCompletedMaps(),
      mapsCompleted);
  assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
  assertEquals("reducesCompleted incorrect", job.getCompletedReduces(),
      reducesCompleted);
  assertEquals("mapProgress incorrect", report.getMapProgress() * 100,
      mapProgress, 0);
  assertEquals("reduceProgress incorrect", report.getReduceProgress() * 100,
      reduceProgress, 0);
}
Example 9
Source File: CachedHistoryStorage.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
public static JobsInfo getPartialJobs(Collection<Job> jobs, Long offset,
    Long count, String user, String queue, Long sBegin, Long sEnd,
    Long fBegin, Long fEnd, JobState jobState) {
  JobsInfo allJobs = new JobsInfo();

  if (sBegin == null || sBegin < 0)
    sBegin = 0l;
  if (sEnd == null)
    sEnd = Long.MAX_VALUE;
  if (fBegin == null || fBegin < 0)
    fBegin = 0l;
  if (fEnd == null)
    fEnd = Long.MAX_VALUE;
  if (offset == null || offset < 0)
    offset = 0l;
  if (count == null)
    count = Long.MAX_VALUE;

  if (offset > jobs.size()) {
    return allJobs;
  }

  long at = 0;
  long end = offset + count - 1;
  if (end < 0) { // due to overflow
    end = Long.MAX_VALUE;
  }

  for (Job job : jobs) {
    if (at > end) {
      break;
    }

    // can't really validate queue is a valid one since queues could change
    if (queue != null && !queue.isEmpty()) {
      if (!job.getQueueName().equals(queue)) {
        continue;
      }
    }

    if (user != null && !user.isEmpty()) {
      if (!job.getUserName().equals(user)) {
        continue;
      }
    }

    JobReport report = job.getReport();

    if (report.getStartTime() < sBegin || report.getStartTime() > sEnd) {
      continue;
    }
    if (report.getFinishTime() < fBegin || report.getFinishTime() > fEnd) {
      continue;
    }
    if (jobState != null && jobState != report.getJobState()) {
      continue;
    }

    at++;
    if ((at - 1) < offset) {
      continue;
    }

    JobInfo jobInfo = new JobInfo(job);

    allJobs.add(jobInfo);
  }
  return allJobs;
}
Example 10
Source File: CachedHistoryStorage.java    From big-c with Apache License 2.0 4 votes vote down vote up 
public static JobsInfo getPartialJobs(Collection<Job> jobs, Long offset,
    Long count, String user, String queue, Long sBegin, Long sEnd,
    Long fBegin, Long fEnd, JobState jobState) {
  JobsInfo allJobs = new JobsInfo();

  if (sBegin == null || sBegin < 0)
    sBegin = 0l;
  if (sEnd == null)
    sEnd = Long.MAX_VALUE;
  if (fBegin == null || fBegin < 0)
    fBegin = 0l;
  if (fEnd == null)
    fEnd = Long.MAX_VALUE;
  if (offset == null || offset < 0)
    offset = 0l;
  if (count == null)
    count = Long.MAX_VALUE;

  if (offset > jobs.size()) {
    return allJobs;
  }

  long at = 0;
  long end = offset + count - 1;
  if (end < 0) { // due to overflow
    end = Long.MAX_VALUE;
  }

  for (Job job : jobs) {
    if (at > end) {
      break;
    }

    // can't really validate queue is a valid one since queues could change
    if (queue != null && !queue.isEmpty()) {
      if (!job.getQueueName().equals(queue)) {
        continue;
      }
    }

    if (user != null && !user.isEmpty()) {
      if (!job.getUserName().equals(user)) {
        continue;
      }
    }

    JobReport report = job.getReport();

    if (report.getStartTime() < sBegin || report.getStartTime() > sEnd) {
      continue;
    }
    if (report.getFinishTime() < fBegin || report.getFinishTime() > fEnd) {
      continue;
    }
    if (jobState != null && jobState != report.getJobState()) {
      continue;
    }

    at++;
    if ((at - 1) < offset) {
      continue;
    }

    JobInfo jobInfo = new JobInfo(job);

    allJobs.add(jobInfo);
  }
  return allJobs;
}