Java Code Examples for org.apache.hadoop.mapreduce.v2.app.job.Task#isFinished()

The following examples show how to use org.apache.hadoop.mapreduce.v2.app.job.Task#isFinished() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JobImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private void computeProgress() {
  this.readLock.lock();
  try {
    float mapProgress = 0f;
    float reduceProgress = 0f;
    for (Task task : this.tasks.values()) {
      if (task.getType() == TaskType.MAP) {
        mapProgress += (task.isFinished() ? 1f : task.getProgress());
      } else {
        reduceProgress += (task.isFinished() ? 1f : task.getProgress());
      }
    }
    if (this.numMapTasks != 0) {
      mapProgress = mapProgress / this.numMapTasks;
    }
    if (this.numReduceTasks != 0) {
      reduceProgress = reduceProgress / this.numReduceTasks;
    }
    this.mapProgress = mapProgress;
    this.reduceProgress = reduceProgress;
  } finally {
    this.readLock.unlock();
  }
}
Example 2
Source File: TestRuntimeEstimators.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private float getReduceProgress() {
  Job job = myAppContext.getJob(myAttemptID.getTaskId().getJobId());
  float runtime = getCodeRuntime();

  Collection<Task> allMapTasks = job.getTasks(TaskType.MAP).values();

  int numberMaps = allMapTasks.size();
  int numberDoneMaps = 0;

  for (Task mapTask : allMapTasks) {
    if (mapTask.isFinished()) {
      ++numberDoneMaps;
    }
  }

  if (numberMaps == numberDoneMaps) {
    shuffleCompletedTime = Math.min(shuffleCompletedTime, clock.getTime());

    return Math.min
        ((float) (clock.getTime() - shuffleCompletedTime)
                    / (runtime * 2000.0F) + 0.5F,
         1.0F);
  } else {
    return ((float) numberDoneMaps) / numberMaps * 0.5F;
  }
}
Example 3
Source File: JobImpl.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private void computeProgress() {
  this.readLock.lock();
  try {
    float mapProgress = 0f;
    float reduceProgress = 0f;
    for (Task task : this.tasks.values()) {
      if (task.getType() == TaskType.MAP) {
        mapProgress += (task.isFinished() ? 1f : task.getProgress());
      } else {
        reduceProgress += (task.isFinished() ? 1f : task.getProgress());
      }
    }
    if (this.numMapTasks != 0) {
      mapProgress = mapProgress / this.numMapTasks;
    }
    if (this.numReduceTasks != 0) {
      reduceProgress = reduceProgress / this.numReduceTasks;
    }
    this.mapProgress = mapProgress;
    this.reduceProgress = reduceProgress;
  } finally {
    this.readLock.unlock();
  }
}
Example 4
Source File: TestRuntimeEstimators.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private float getReduceProgress() {
  Job job = myAppContext.getJob(myAttemptID.getTaskId().getJobId());
  float runtime = getCodeRuntime();

  Collection<Task> allMapTasks = job.getTasks(TaskType.MAP).values();

  int numberMaps = allMapTasks.size();
  int numberDoneMaps = 0;

  for (Task mapTask : allMapTasks) {
    if (mapTask.isFinished()) {
      ++numberDoneMaps;
    }
  }

  if (numberMaps == numberDoneMaps) {
    shuffleCompletedTime = Math.min(shuffleCompletedTime, clock.getTime());

    return Math.min
        ((float) (clock.getTime() - shuffleCompletedTime)
                    / (runtime * 2000.0F) + 0.5F,
         1.0F);
  } else {
    return ((float) numberDoneMaps) / numberMaps * 0.5F;
  }
}
Example 5
Source File: JobImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public JobStateInternal transition(JobImpl job, JobEvent event) {
  if(!job.failWaitTriggerScheduledFuture.isCancelled()) {
    for(Task task: job.tasks.values()) {
      if(!task.isFinished()) {
        return JobStateInternal.FAIL_WAIT;
      }
    }
  }
  //Finished waiting. All tasks finished / were killed
  job.failWaitTriggerScheduledFuture.cancel(false);
  job.eventHandler.handle(new CommitterJobAbortEvent(job.jobId,
    job.jobContext, org.apache.hadoop.mapreduce.JobStatus.State.FAILED));
  return JobStateInternal.FAIL_ABORT;
}
Example 6
Source File: MockJobs.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
void incr(Task task) {
  TaskType type = task.getType();
  boolean finished = task.isFinished();
  if (type == TaskType.MAP) {
    if (finished) {
      ++completedMaps;
    }
    ++maps;
  } else if (type == TaskType.REDUCE) {
    if (finished) {
      ++completedReduces;
    }
    ++reduces;
  }
}
Example 7
Source File: JobImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public JobStateInternal transition(JobImpl job, JobEvent event) {
  if(!job.failWaitTriggerScheduledFuture.isCancelled()) {
    for(Task task: job.tasks.values()) {
      if(!task.isFinished()) {
        return JobStateInternal.FAIL_WAIT;
      }
    }
  }
  //Finished waiting. All tasks finished / were killed
  job.failWaitTriggerScheduledFuture.cancel(false);
  job.eventHandler.handle(new CommitterJobAbortEvent(job.jobId,
    job.jobContext, org.apache.hadoop.mapreduce.JobStatus.State.FAILED));
  return JobStateInternal.FAIL_ABORT;
}
Example 8
Source File: MockJobs.java    From big-c with Apache License 2.0 5 votes vote down vote up 
void incr(Task task) {
  TaskType type = task.getType();
  boolean finished = task.isFinished();
  if (type == TaskType.MAP) {
    if (finished) {
      ++completedMaps;
    }
    ++maps;
  } else if (type == TaskType.REDUCE) {
    if (finished) {
      ++completedReduces;
    }
    ++reduces;
  }
}
Example 9
Source File: JobImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
protected JobStateInternal checkJobAfterTaskCompletion(JobImpl job) {
  //check for Job failure
  if (job.failedMapTaskCount*100 > 
    job.allowedMapFailuresPercent*job.numMapTasks ||
    job.failedReduceTaskCount*100 > 
    job.allowedReduceFailuresPercent*job.numReduceTasks) {
    job.setFinishTime();

    String diagnosticMsg = "Job failed as tasks failed. " +
        "failedMaps:" + job.failedMapTaskCount + 
        " failedReduces:" + job.failedReduceTaskCount;
    LOG.info(diagnosticMsg);
    job.addDiagnostic(diagnosticMsg);

    //Send kill signal to all unfinished tasks here.
    boolean allDone = true;
    for (Task task : job.tasks.values()) {
      if(!task.isFinished()) {
        allDone = false;
        job.eventHandler.handle(
          new TaskEvent(task.getID(), TaskEventType.T_KILL));
      }
    }

    //If all tasks are already done, we should go directly to FAIL_ABORT
    if(allDone) {
      job.eventHandler.handle(new CommitterJobAbortEvent(job.jobId,
        job.jobContext, org.apache.hadoop.mapreduce.JobStatus.State.FAILED)
      );
      return JobStateInternal.FAIL_ABORT;
    }

    //Set max timeout to wait for the tasks to get killed
    job.failWaitTriggerScheduledFuture = job.executor.schedule(
      new TriggerScheduledFuture(job, new JobEvent(job.getID(),
        JobEventType.JOB_FAIL_WAIT_TIMEDOUT)), job.conf.getInt(
            MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS,
            MRJobConfig.DEFAULT_MR_AM_COMMITTER_CANCEL_TIMEOUT_MS),
            TimeUnit.MILLISECONDS);
    return JobStateInternal.FAIL_WAIT;
  }
  
  return job.checkReadyForCommit();
}
Example 10
Source File: JobImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up 
protected JobStateInternal checkJobAfterTaskCompletion(JobImpl job) {
  //check for Job failure
  if (job.failedMapTaskCount*100 > 
    job.allowedMapFailuresPercent*job.numMapTasks ||
    job.failedReduceTaskCount*100 > 
    job.allowedReduceFailuresPercent*job.numReduceTasks) {
    job.setFinishTime();

    String diagnosticMsg = "Job failed as tasks failed. " +
        "failedMaps:" + job.failedMapTaskCount + 
        " failedReduces:" + job.failedReduceTaskCount;
    LOG.info(diagnosticMsg);
    job.addDiagnostic(diagnosticMsg);

    //Send kill signal to all unfinished tasks here.
    boolean allDone = true;
    for (Task task : job.tasks.values()) {
      if(!task.isFinished()) {
        allDone = false;
        job.eventHandler.handle(
          new TaskEvent(task.getID(), TaskEventType.T_KILL));
      }
    }

    //If all tasks are already done, we should go directly to FAIL_ABORT
    if(allDone) {
      job.eventHandler.handle(new CommitterJobAbortEvent(job.jobId,
        job.jobContext, org.apache.hadoop.mapreduce.JobStatus.State.FAILED)
      );
      return JobStateInternal.FAIL_ABORT;
    }

    //Set max timeout to wait for the tasks to get killed
    job.failWaitTriggerScheduledFuture = job.executor.schedule(
      new TriggerScheduledFuture(job, new JobEvent(job.getID(),
        JobEventType.JOB_FAIL_WAIT_TIMEDOUT)), job.conf.getInt(
            MRJobConfig.MR_AM_COMMITTER_CANCEL_TIMEOUT_MS,
            MRJobConfig.DEFAULT_MR_AM_COMMITTER_CANCEL_TIMEOUT_MS),
            TimeUnit.MILLISECONDS);
    return JobStateInternal.FAIL_WAIT;
  }
  
  return job.checkReadyForCommit();
}