org.apache.hadoop.mapreduce.v2.api.records.TaskType Java Examples

@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
  // too many fetch failure can only happen for map tasks
  Preconditions
      .checkArgument(taskAttempt.getID().getTaskId().getTaskType() == TaskType.MAP);
  //add to diagnostic
  taskAttempt.addDiagnosticInfo("Too Many fetch failures.Failing the attempt");
  
  if (taskAttempt.getLaunchTime() != 0) {
    taskAttempt.eventHandler
        .handle(createJobCounterUpdateEventTAFailed(taskAttempt, true));
    TaskAttemptUnsuccessfulCompletionEvent tauce =
        createTaskAttemptUnsuccessfulCompletionEvent(taskAttempt,
            TaskAttemptStateInternal.FAILED);
    taskAttempt.eventHandler.handle(new JobHistoryEvent(
        taskAttempt.attemptId.getTaskId().getJobId(), tauce));
  }else {
    LOG.debug("Not generating HistoryFinish event since start event not " +
        "generated for taskAttempt: " + taskAttempt.getID());
  }
  taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
      taskAttempt.attemptId, TaskEventType.T_ATTEMPT_FAILED));
}
 

private void computeProgress() {
  this.readLock.lock();
  try {
    float mapProgress = 0f;
    float reduceProgress = 0f;
    for (Task task : this.tasks.values()) {
      if (task.getType() == TaskType.MAP) {
        mapProgress += (task.isFinished() ? 1f : task.getProgress());
      } else {
        reduceProgress += (task.isFinished() ? 1f : task.getProgress());
      }
    }
    if (this.numMapTasks != 0) {
      mapProgress = mapProgress / this.numMapTasks;
    }
    if (this.numReduceTasks != 0) {
      reduceProgress = reduceProgress / this.numReduceTasks;
    }
    this.mapProgress = mapProgress;
    this.reduceProgress = reduceProgress;
  } finally {
    this.readLock.unlock();
  }
}
 

private void computeProgress() {
  this.readLock.lock();
  try {
    float mapProgress = 0f;
    float reduceProgress = 0f;
    for (Task task : this.tasks.values()) {
      if (task.getType() == TaskType.MAP) {
        mapProgress += (task.isFinished() ? 1f : task.getProgress());
      } else {
        reduceProgress += (task.isFinished() ? 1f : task.getProgress());
      }
    }
    if (this.numMapTasks != 0) {
      mapProgress = mapProgress / this.numMapTasks;
    }
    if (this.numReduceTasks != 0) {
      reduceProgress = reduceProgress / this.numReduceTasks;
    }
    this.mapProgress = mapProgress;
    this.reduceProgress = reduceProgress;
  } finally {
    this.readLock.unlock();
  }
}
 

@Test
public void testKillDuringTaskAttemptCommit() {
  mockTask = createMockTask(TaskType.REDUCE);        
  TaskId taskId = getNewTaskID();
  scheduleTaskAttempt(taskId);
  
  launchTaskAttempt(getLastAttempt().getAttemptId());
  updateLastAttemptState(TaskAttemptState.COMMIT_PENDING);
  commitTaskAttempt(getLastAttempt().getAttemptId());

  TaskAttemptId commitAttempt = getLastAttempt().getAttemptId();
  updateLastAttemptState(TaskAttemptState.KILLED);
  killRunningTaskAttempt(commitAttempt);

  assertFalse(mockTask.canCommit(commitAttempt));
}
 

@Override
public GetTaskReportsResponse getTaskReports(
    GetTaskReportsRequest request) throws IOException {
  JobId jobId = request.getJobId();
  TaskType taskType = request.getTaskType();
  
  GetTaskReportsResponse response = 
    recordFactory.newRecordInstance(GetTaskReportsResponse.class);
  
  Job job = verifyAndGetJob(jobId, JobACL.VIEW_JOB, true);
  Collection<Task> tasks = job.getTasks(taskType).values();
  LOG.info("Getting task report for " + taskType + "   " + jobId
      + ". Report-size will be " + tasks.size());

  // Take lock to allow only one call, otherwise heap will blow up because
  // of counters in the report when there are multiple callers.
  synchronized (getTaskReportsLock) {
    for (Task task : tasks) {
      response.addTaskReport(task.getReport());
    }
  }

  return response;
}
 

boolean remove(TaskAttemptId tId) {
  ContainerId containerId = null;
  if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
    containerId = maps.remove(tId).getId();
  } else {
    containerId = reduces.remove(tId).getId();
    if (containerId != null) {
      boolean preempted = preemptionWaitingReduces.remove(tId);
      if (preempted) {
        LOG.info("Reduce preemption successful " + tId);
      }
    }
  }
  
  if (containerId != null) {
    containerToAttemptMap.remove(containerId);
    return true;
  }
  return false;
}
 

@GET
@Path("/jobs/{jobid}/tasks/{taskid}/attempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptsInfo getJobTaskAttempts(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid) {

  init();
  TaskAttemptsInfo attempts = new TaskAttemptsInfo();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  Task task = getTaskFromTaskIdString(tid, job);

  for (TaskAttempt ta : task.getAttempts().values()) {
    if (ta != null) {
      if (task.getType() == TaskType.REDUCE) {
        attempts.add(new ReduceTaskAttemptInfo(ta, task.getType()));
      } else {
        attempts.add(new TaskAttemptInfo(ta, task.getType(), true));
      }
    }
  }
  return attempts;
}
 

@GET
@Path("/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptInfo getJobTaskAttemptId(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid,
    @PathParam("attemptid") String attId) {

  init();
  Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
  checkAccess(job, hsr);
  Task task = AMWebServices.getTaskFromTaskIdString(tid, job);
  TaskAttempt ta = AMWebServices.getTaskAttemptFromTaskAttemptString(attId,
      task);
  if (task.getType() == TaskType.REDUCE) {
    return new ReduceTaskAttemptInfo(ta, task.getType());
  } else {
    return new TaskAttemptInfo(ta, task.getType(), false);
  }
}
 

@GET
@Path("/jobs/{jobid}/tasks/{taskid}/attempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptsInfo getJobTaskAttempts(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid) {

  init();
  TaskAttemptsInfo attempts = new TaskAttemptsInfo();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  Task task = getTaskFromTaskIdString(tid, job);

  for (TaskAttempt ta : task.getAttempts().values()) {
    if (ta != null) {
      if (task.getType() == TaskType.REDUCE) {
        attempts.add(new ReduceTaskAttemptInfo(ta, task.getType()));
      } else {
        attempts.add(new TaskAttemptInfo(ta, task.getType(), true));
      }
    }
  }
  return attempts;
}
 

@GET
@Path("/jobs/{jobid}/tasks")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TasksInfo getJobTasks(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @QueryParam("type") String type) {

  init();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  TasksInfo allTasks = new TasksInfo();
  for (Task task : job.getTasks().values()) {
    TaskType ttype = null;
    if (type != null && !type.isEmpty()) {
      try {
        ttype = MRApps.taskType(type);
      } catch (YarnRuntimeException e) {
        throw new BadRequestException("tasktype must be either m or r");
      }
    }
    if (ttype != null && task.getType() != ttype) {
      continue;
    }
    allTasks.add(new TaskInfo(task));
  }
  return allTasks;
}
 

private float getReduceProgress() {
  Job job = myAppContext.getJob(myAttemptID.getTaskId().getJobId());
  float runtime = getCodeRuntime();

  Collection<Task> allMapTasks = job.getTasks(TaskType.MAP).values();

  int numberMaps = allMapTasks.size();
  int numberDoneMaps = 0;

  for (Task mapTask : allMapTasks) {
    if (mapTask.isFinished()) {
      ++numberDoneMaps;
    }
  }

  if (numberMaps == numberDoneMaps) {
    shuffleCompletedTime = Math.min(shuffleCompletedTime, clock.getTime());

    return Math.min
        ((float) (clock.getTime() - shuffleCompletedTime)
                    / (runtime * 2000.0F) + 0.5F,
         1.0F);
  } else {
    return ((float) numberDoneMaps) / numberMaps * 0.5F;
  }
}
 

private AtomicInteger containerNeed(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  TaskType taskType = taskID.getTaskType();

  ConcurrentMap<JobId, AtomicInteger> relevantMap
      = taskType == TaskType.MAP ? mapContainerNeeds : reduceContainerNeeds;

  AtomicInteger result = relevantMap.get(jobID);

  if (result == null) {
    relevantMap.putIfAbsent(jobID, new AtomicInteger(0));
    result = relevantMap.get(jobID);
  }

  return result;
}
 

public void verifyAMTaskAttemptXML(Element element, TaskAttempt att,
    TaskType ttype) {
  verifyTaskAttemptGeneric(att, ttype,
      WebServicesTestUtils.getXmlString(element, "id"),
      WebServicesTestUtils.getXmlString(element, "state"),
      WebServicesTestUtils.getXmlString(element, "type"),
      WebServicesTestUtils.getXmlString(element, "rack"),
      WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
      WebServicesTestUtils.getXmlString(element, "diagnostics"),
      WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
      WebServicesTestUtils.getXmlLong(element, "startTime"),
      WebServicesTestUtils.getXmlLong(element, "finishTime"),
      WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
      WebServicesTestUtils.getXmlFloat(element, "progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att,
        WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
  }
}
 

@Test (timeout=100000)
public void testCompletedJob() throws Exception {
  HistoryFileInfo info = mock(HistoryFileInfo.class);
  when(info.getConfFile()).thenReturn(fullConfPath);
  //Re-initialize to verify the delayed load.
  completedJob =
    new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
        info, jobAclsManager);
  //Verify tasks loaded based on loadTask parameter.
  assertEquals(loadTasks, completedJob.tasksLoaded.get());
  assertEquals(1, completedJob.getAMInfos().size());
  assertEquals(10, completedJob.getCompletedMaps());
  assertEquals(1, completedJob.getCompletedReduces());
  assertEquals(12, completedJob.getTasks().size());
  //Verify tasks loaded at this point.
  assertEquals(true, completedJob.tasksLoaded.get());
  assertEquals(10, completedJob.getTasks(TaskType.MAP).size());
  assertEquals(2, completedJob.getTasks(TaskType.REDUCE).size());
  assertEquals("user", completedJob.getUserName());
  assertEquals(JobState.SUCCEEDED, completedJob.getState());
  JobReport jobReport = completedJob.getReport();
  assertEquals("user", jobReport.getUser());
  assertEquals(JobState.SUCCEEDED, jobReport.getJobState());
}
 

public ReduceTaskAttemptInfo(TaskAttempt ta, TaskType type) {
  super(ta, type, false);

  this.shuffleFinishTime = ta.getShuffleFinishTime();
  this.mergeFinishTime = ta.getSortFinishTime();
  this.elapsedShuffleTime = Times.elapsed(this.startTime,
      this.shuffleFinishTime, false);
  if (this.elapsedShuffleTime == -1) {
    this.elapsedShuffleTime = 0;
  }
  this.elapsedMergeTime = Times.elapsed(this.shuffleFinishTime,
      this.mergeFinishTime, false);
  if (this.elapsedMergeTime == -1) {
    this.elapsedMergeTime = 0;
  }
  this.elapsedReduceTime = Times.elapsed(this.mergeFinishTime,
      this.finishTime, false);
  if (this.elapsedReduceTime == -1) {
    this.elapsedReduceTime = 0;
  }
}
 

private static void completeJobTasks(JobImpl job) {
  // complete the map tasks and the reduce tasks so we start committing
  int numMaps = job.getTotalMaps();
  for (int i = 0; i < numMaps; ++i) {
    job.handle(new JobTaskEvent(
        MRBuilderUtils.newTaskId(job.getID(), 1, TaskType.MAP),
        TaskState.SUCCEEDED));
    Assert.assertEquals(JobState.RUNNING, job.getState());
  }
  int numReduces = job.getTotalReduces();
  for (int i = 0; i < numReduces; ++i) {
    job.handle(new JobTaskEvent(
        MRBuilderUtils.newTaskId(job.getID(), 1, TaskType.MAP),
        TaskState.SUCCEEDED));
    Assert.assertEquals(JobState.RUNNING, job.getState());
  }
}
 

@Test 
/**
 * Kill attempt
 * {@link TaskState#SCHEDULED}->{@link TaskState#SCHEDULED}
 */
public void testKillScheduledTaskAttempt() {
  LOG.info("--- START: testKillScheduledTaskAttempt ---");
  mockTask = createMockTask(TaskType.MAP);        
  TaskId taskId = getNewTaskID();
  scheduleTaskAttempt(taskId);
  killScheduledTaskAttempt(getLastAttempt().getAttemptId());
}
 

void incr(Task task) {
  TaskType type = task.getType();
  boolean finished = task.isFinished();
  if (type == TaskType.MAP) {
    if (finished) {
      ++completedMaps;
    }
    ++maps;
  } else if (type == TaskType.REDUCE) {
    if (finished) {
      ++completedReduces;
    }
    ++reduces;
  }
}
 

@Override
public Map<TaskId, Task> getTasks(TaskType taskType) {
  loadAllTasks();
  if (TaskType.MAP.equals(taskType)) {
    return mapTasks;
  } else {//we have only two types of tasks
    return reduceTasks;
  }
}
 

@Override
public long thresholdRuntime(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  TaskType type = taskID.getTaskType();

  DataStatistics statistics
      = dataStatisticsForTask(taskID);

  int completedTasksOfType
      = type == TaskType.MAP
          ? job.getCompletedMaps() : job.getCompletedReduces();

  int totalTasksOfType
      = type == TaskType.MAP
          ? job.getTotalMaps() : job.getTotalReduces();

  if (completedTasksOfType < MINIMUM_COMPLETE_NUMBER_TO_SPECULATE
      || (((float)completedTasksOfType) / totalTasksOfType)
            < MINIMUM_COMPLETE_PROPORTION_TO_SPECULATE ) {
    return Long.MAX_VALUE;
  }

  long result =  statistics == null
      ? Long.MAX_VALUE
      : (long)statistics.outlier(slowTaskRelativeTresholds.get(job));
  return result;
}
 

/**
 * Given the taskId details (JobId, suffix id and task type), it gives the TaskReport
 * @param jobId, the JobId instance
 * @param id, the suffix id as int
 * @param taskType, the task type
 * @return the Task Report
 * @throws IOException
 */
public TaskReport getTaskReport(JobId jobId, int id, TaskType taskType) throws IOException{
	TaskId taskId = YarnCommunicatorUtil.getTaskId(jobId, id, taskType);
	GetTaskReportRequestProto proto = GetTaskReportRequestProto.getDefaultInstance();	
	GetTaskReportRequest getTaskReportRequest = new GetTaskReportRequestPBImpl(proto);
	getTaskReportRequest.setTaskId(taskId);
	GetTaskReportResponse taskReportResponse =  proxy.getTaskReport(getTaskReportRequest);
	return taskReportResponse.getTaskReport();
}
 

boolean remove(TaskAttemptId tId) {
  ContainerRequest req = null;
  if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
    req = maps.remove(tId);
  } else {
    req = reduces.remove(tId);
  }
  
  if (req == null) {
    return false;
  } else {
    decContainerReq(req);
    return true;
  }
}
 

public MockTaskAttemptImpl(TaskId taskId, int id, EventHandler eventHandler,
    TaskAttemptListener taskAttemptListener, Path jobFile, int partition,
    JobConf conf, Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    AppContext appContext, TaskType taskType) {
  super(taskId, id, eventHandler, taskAttemptListener, jobFile, partition, conf,
      dataLocations, jobToken, credentials, clock, appContext);
  this.taskType = taskType;
}
 

private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled(
    TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
  TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
  JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
  
  if (taskType == TaskType.MAP) {
    jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1);
  } else {
    jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1);
  }
  if (!taskAlreadyCompleted) {
    updateMillisCounters(jce, taskAttempt);
  }
  return jce;
}
 

@Override
public Map<TaskId, Task> getTasks(TaskType taskType) {
  loadAllTasks();
  if (TaskType.MAP.equals(taskType)) {
    return mapTasks;
  } else {//we have only two types of tasks
    return reduceTasks;
  }
}
 

public void verifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype,
    String id, String state, String type, String rack,
    String nodeHttpAddress, String diagnostics, String assignedContainerId,
    long startTime, long finishTime, long elapsedTime, float progress) {

  TaskAttemptId attid = ta.getID();
  String attemptId = MRApps.toString(attid);

  WebServicesTestUtils.checkStringMatch("id", attemptId, id);
  WebServicesTestUtils.checkStringMatch("type", ttype.toString(), type);
  WebServicesTestUtils.checkStringMatch("state", ta.getState().toString(),
      state);
  WebServicesTestUtils.checkStringMatch("rack", ta.getNodeRackName(), rack);
  WebServicesTestUtils.checkStringMatch("nodeHttpAddress",
      ta.getNodeHttpAddress(), nodeHttpAddress);

  String expectDiag = "";
  List<String> diagnosticsList = ta.getDiagnostics();
  if (diagnosticsList != null && !diagnostics.isEmpty()) {
    StringBuffer b = new StringBuffer();
    for (String diag : diagnosticsList) {
      b.append(diag);
    }
    expectDiag = b.toString();
  }
  WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
      diagnostics);
  WebServicesTestUtils.checkStringMatch("assignedContainerId",
      ConverterUtils.toString(ta.getAssignedContainerID()),
      assignedContainerId);

  assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
  assertEquals("finishTime wrong", ta.getFinishTime(), finishTime);
  assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
  assertEquals("progress wrong", ta.getProgress() * 100, progress, 1e-3f);
}
 

void add(Container container, TaskAttemptId tId) {
  LOG.info("Assigned container " + container.getId().toString() + " to " + tId);
  containerToAttemptMap.put(container.getId(), tId);
  if (tId.getTaskId().getTaskType().equals(TaskType.MAP)) {
    maps.put(tId, container);
  } else {
    reduces.put(tId, container);
  }
}
 

public void verifyTaskAttemptGeneric(TaskAttempt ta, TaskType ttype,
    String id, String state, String type, String rack,
    String nodeHttpAddress, String diagnostics, String assignedContainerId,
    long startTime, long finishTime, long elapsedTime, float progress) {

  TaskAttemptId attid = ta.getID();
  String attemptId = MRApps.toString(attid);

  WebServicesTestUtils.checkStringMatch("id", attemptId, id);
  WebServicesTestUtils.checkStringMatch("type", ttype.toString(), type);
  WebServicesTestUtils.checkStringMatch("state", ta.getState().toString(),
      state);
  WebServicesTestUtils.checkStringMatch("rack", ta.getNodeRackName(), rack);
  WebServicesTestUtils.checkStringMatch("nodeHttpAddress",
      ta.getNodeHttpAddress(), nodeHttpAddress);

  String expectDiag = "";
  List<String> diagnosticsList = ta.getDiagnostics();
  if (diagnosticsList != null && !diagnostics.isEmpty()) {
    StringBuffer b = new StringBuffer();
    for (String diag : diagnosticsList) {
      b.append(diag);
    }
    expectDiag = b.toString();
  }
  WebServicesTestUtils.checkStringMatch("diagnostics", expectDiag,
      diagnostics);
  WebServicesTestUtils.checkStringMatch("assignedContainerId",
      ConverterUtils.toString(ta.getAssignedContainerID()),
      assignedContainerId);

  assertEquals("startTime wrong", ta.getLaunchTime(), startTime);
  assertEquals("finishTime wrong", ta.getFinishTime(), finishTime);
  assertEquals("elapsedTime wrong", finishTime - startTime, elapsedTime);
  assertEquals("progress wrong", ta.getProgress() * 100, progress, 1e-3f);
}
 

public static TaskId newTaskId(JobId jobId, int id, TaskType taskType) {
  TaskId taskId = Records.newRecord(TaskId.class);
  taskId.setJobId(jobId);
  taskId.setId(id);
  taskId.setTaskType(taskType);
  return taskId;
}
 

@Override
public float getProgress() {
  if (overridingState == TaskAttemptState.NEW) {
    return 0.0F;
  }
  return myAttemptID.getTaskId().getTaskType() == TaskType.MAP ? getMapProgress() : getReduceProgress();
}