Java Code Examples for org.apache.hadoop.mapreduce.v2.app.job.Task#getType()

public void verifyHsTask(JSONArray arr, Job job, String type)
    throws JSONException {
  for (Task task : job.getTasks().values()) {
    TaskId id = task.getID();
    String tid = MRApps.toString(id);
    Boolean found = false;
    if (type != null && task.getType() == MRApps.taskType(type)) {

      for (int i = 0; i < arr.length(); i++) {
        JSONObject info = arr.getJSONObject(i);
        if (tid.matches(info.getString("id"))) {
          found = true;
          verifyHsSingleTask(info, task);
        }
      }
      assertTrue("task with id: " + tid + " not in web service output", found);
    }
  }
}
 

protected DataStatistics dataStatisticsForTask(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  if (job == null) {
    return null;
  }

  Task task = job.getTask(taskID);

  if (task == null) {
    return null;
  }

  return task.getType() == TaskType.MAP
          ? mapperStatistics.get(job)
          : task.getType() == TaskType.REDUCE
              ? reducerStatistics.get(job)
              : null;
}
 

public TaskInfo(Task task) {
  TaskType ttype = task.getType();
  this.type = ttype.toString();
  TaskReport report = task.getReport();
  this.startTime = report.getStartTime();
  this.finishTime = report.getFinishTime();
  this.state = report.getTaskState();
  this.elapsedTime = Times.elapsed(this.startTime, this.finishTime,
    this.state == TaskState.RUNNING);
  if (this.elapsedTime == -1) {
    this.elapsedTime = 0;
  }
  this.progress = report.getProgress() * 100;
  this.status =  report.getStatus();
  this.id = MRApps.toString(task.getID());
  this.taskNum = task.getID().getId();
  this.successful = getSuccessfulAttempt(task);
  if (successful != null) {
    this.successfulAttempt = MRApps.toString(successful.getID());
  } else {
    this.successfulAttempt = "";
  }
}
 

@GET
@Path("/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptsInfo getJobTaskAttempts(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid) {

  init();
  TaskAttemptsInfo attempts = new TaskAttemptsInfo();
  Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
  checkAccess(job, hsr);
  Task task = AMWebServices.getTaskFromTaskIdString(tid, job);
  for (TaskAttempt ta : task.getAttempts().values()) {
    if (ta != null) {
      if (task.getType() == TaskType.REDUCE) {
        attempts.add(new ReduceTaskAttemptInfo(ta, task.getType()));
      } else {
        attempts.add(new TaskAttemptInfo(ta, task.getType(), false));
      }
    }
  }
  return attempts;
}
 

public void verifyAMTask(JSONArray arr, Job job, String type)
    throws JSONException {
  for (Task task : job.getTasks().values()) {
    TaskId id = task.getID();
    String tid = MRApps.toString(id);
    Boolean found = false;
    if (type != null && task.getType() == MRApps.taskType(type)) {

      for (int i = 0; i < arr.length(); i++) {
        JSONObject info = arr.getJSONObject(i);
        if (tid.matches(info.getString("id"))) {
          found = true;
          verifyAMSingleTask(info, task);
        }
      }
      assertTrue("task with id: " + tid + " not in web service output", found);
    }
  }
}
 

public void verifyAMTask(JSONArray arr, Job job, String type)
    throws JSONException {
  for (Task task : job.getTasks().values()) {
    TaskId id = task.getID();
    String tid = MRApps.toString(id);
    Boolean found = false;
    if (type != null && task.getType() == MRApps.taskType(type)) {

      for (int i = 0; i < arr.length(); i++) {
        JSONObject info = arr.getJSONObject(i);
        if (tid.matches(info.getString("id"))) {
          found = true;
          verifyAMSingleTask(info, task);
        }
      }
      assertTrue("task with id: " + tid + " not in web service output", found);
    }
  }
}
 

@Private
public void constructFinalFullcounters() {
  this.fullCounters = new Counters();
  this.finalMapCounters = new Counters();
  this.finalReduceCounters = new Counters();
  this.fullCounters.incrAllCounters(jobCounters);
  for (Task t : this.tasks.values()) {
    Counters counters = t.getCounters();
    switch (t.getType()) {
    case MAP:
      this.finalMapCounters.incrAllCounters(counters);
      break;
    case REDUCE:
      this.finalReduceCounters.incrAllCounters(counters);
      break;
    default:
      throw new IllegalStateException("Task type neither map nor reduce: " + 
          t.getType());
    }
    this.fullCounters.incrAllCounters(counters);
  }
}
 

@GET
@Path("/jobs/{jobid}/tasks/{taskid}/attempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptsInfo getJobTaskAttempts(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid) {

  init();
  TaskAttemptsInfo attempts = new TaskAttemptsInfo();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  Task task = getTaskFromTaskIdString(tid, job);

  for (TaskAttempt ta : task.getAttempts().values()) {
    if (ta != null) {
      if (task.getType() == TaskType.REDUCE) {
        attempts.add(new ReduceTaskAttemptInfo(ta, task.getType()));
      } else {
        attempts.add(new TaskAttemptInfo(ta, task.getType(), true));
      }
    }
  }
  return attempts;
}
 

private void computeProgress() {
  this.readLock.lock();
  try {
    float mapProgress = 0f;
    float reduceProgress = 0f;
    for (Task task : this.tasks.values()) {
      if (task.getType() == TaskType.MAP) {
        mapProgress += (task.isFinished() ? 1f : task.getProgress());
      } else {
        reduceProgress += (task.isFinished() ? 1f : task.getProgress());
      }
    }
    if (this.numMapTasks != 0) {
      mapProgress = mapProgress / this.numMapTasks;
    }
    if (this.numReduceTasks != 0) {
      reduceProgress = reduceProgress / this.numReduceTasks;
    }
    this.mapProgress = mapProgress;
    this.reduceProgress = reduceProgress;
  } finally {
    this.readLock.unlock();
  }
}
 

public void waitingTask(Task task) {
  switch (task.getType()) {
    case MAP:
      mapsWaiting.incr();
      break;
    case REDUCE:
      reducesWaiting.incr();
  }
}
 

public void runningTask(Task task) {
  switch (task.getType()) {
    case MAP:
      mapsRunning.incr();
      break;
    case REDUCE:
      reducesRunning.incr();
      break;
  }
}
 

void incr(Task task) {
  TaskType type = task.getType();
  boolean finished = task.isFinished();
  if (type == TaskType.MAP) {
    if (finished) {
      ++completedMaps;
    }
    ++maps;
  } else if (type == TaskType.REDUCE) {
    if (finished) {
      ++completedReduces;
    }
    ++reduces;
  }
}
 

public void failedTask(Task task) {
  switch (task.getType()) {
    case MAP:
      mapsFailed.incr();
      break;
    case REDUCE:
      reducesFailed.incr();
      break;
  }
}
 

public void completedTask(Task task) {
  switch (task.getType()) {
    case MAP:
      mapsCompleted.incr();
      break;
    case REDUCE:
      reducesCompleted.incr();
      break;
  }
}
 

private void taskFailed(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.failedMapTaskCount++;
  } else if (task.getType() == TaskType.REDUCE) {
    job.failedReduceTaskCount++;
  }
  job.addDiagnostic("Task failed " + task.getID());
  job.metrics.failedTask(task);
}
 

public void killedTask(Task task) {
  switch (task.getType()) {
    case MAP:
      mapsKilled.incr();
      break;
    case REDUCE:
      reducesKilled.incr();
      break;
  }
}
 

private void taskSucceeded(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.succeededMapTaskCount++;
  } else {
    job.succeededReduceTaskCount++;
  }
  job.metrics.completedTask(task);
}
 

public void failedTask(Task task) {
  switch (task.getType()) {
    case MAP:
      mapsFailed.incr();
      break;
    case REDUCE:
      reducesFailed.incr();
      break;
  }
}
 

public void launchedTask(Task task) {
  switch (task.getType()) {
    case MAP:
      mapsLaunched.incr();
      break;
    case REDUCE:
      reducesLaunched.incr();
      break;
  }
  endWaitingTask(task);
}
 

/**
 * Go through a job and update the member variables with counts for
 * information to output in the page.
 *
 * @param job
 *          the job to get counts for.
 */
private void countTasksAndAttempts(Job job) {
  numReduces = 0;
  numMaps = 0;
  final Map<TaskId, Task> tasks = job.getTasks();
  if (tasks == null) {
    return;
  }
  for (Task task : tasks.values()) {
    // Attempts counts
    Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
    int successful, failed, killed;
    for (TaskAttempt attempt : attempts.values()) {

      successful = 0;
      failed = 0;
      killed = 0;
      if (TaskAttemptStateUI.NEW.correspondsTo(attempt.getState())) {
        // Do Nothing
      } else if (TaskAttemptStateUI.RUNNING.correspondsTo(attempt.getState())) {
        // Do Nothing
      } else if (TaskAttemptStateUI.SUCCESSFUL.correspondsTo(attempt
          .getState())) {
        ++successful;
      } else if (TaskAttemptStateUI.FAILED.correspondsTo(attempt.getState())) {
        ++failed;
      } else if (TaskAttemptStateUI.KILLED.correspondsTo(attempt.getState())) {
        ++killed;
      }

      switch (task.getType()) {
      case MAP:
        successfulMapAttempts += successful;
        failedMapAttempts += failed;
        killedMapAttempts += killed;
        if (attempt.getState() == TaskAttemptState.SUCCEEDED) {
          numMaps++;
          avgMapTime += (attempt.getFinishTime() - attempt.getLaunchTime());
        }
        break;
      case REDUCE:
        successfulReduceAttempts += successful;
        failedReduceAttempts += failed;
        killedReduceAttempts += killed;
        if (attempt.getState() == TaskAttemptState.SUCCEEDED) {
          numReduces++;
          avgShuffleTime += (attempt.getShuffleFinishTime() - attempt
              .getLaunchTime());
          avgMergeTime += attempt.getSortFinishTime()
              - attempt.getShuffleFinishTime();
          avgReduceTime += (attempt.getFinishTime() - attempt
              .getSortFinishTime());
        }
        break;
      }
    }
  }

  if (numMaps > 0) {
    avgMapTime = avgMapTime / numMaps;
  }

  if (numReduces > 0) {
    avgReduceTime = avgReduceTime / numReduces;
    avgShuffleTime = avgShuffleTime / numReduces;
    avgMergeTime = avgMergeTime / numReduces;
  }
}