Java Code Examples for org.apache.hadoop.mapreduce.v2.api.records.TaskType#MAP

The following examples show how to use org.apache.hadoop.mapreduce.v2.api.records.TaskType#MAP . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JobImpl.java From big-c with Apache License 2.0

6 votes

private void actOnUnusableNode(NodeId nodeId, NodeState nodeState) {
  // rerun previously successful map tasks
  List<TaskAttemptId> taskAttemptIdList = nodesToSucceededTaskAttempts.get(nodeId);
  if(taskAttemptIdList != null) {
    String mesg = "TaskAttempt killed because it ran on unusable node "
        + nodeId;
    for(TaskAttemptId id : taskAttemptIdList) {
      if(TaskType.MAP == id.getTaskId().getTaskType()) {
        // reschedule only map tasks because their outputs maybe unusable
        LOG.info(mesg + ". AttemptId:" + id);
        eventHandler.handle(new TaskAttemptKillEvent(id, mesg));
      }
    }
  }
  // currently running task attempts on unusable nodes are handled in
  // RMContainerAllocator
}

Example 2

Source File: JobImpl.java From hadoop with Apache License 2.0

6 votes

protected void addTask(Task task) {
  synchronized (tasksSyncHandle) {
    if (lazyTasksCopyNeeded) {
      Map<TaskId, Task> newTasks = new LinkedHashMap<TaskId, Task>();
      newTasks.putAll(tasks);
      tasks = newTasks;
      lazyTasksCopyNeeded = false;
    }
  }
  tasks.put(task.getID(), task);
  if (task.getType() == TaskType.MAP) {
    mapTasks.add(task.getID());
  } else if (task.getType() == TaskType.REDUCE) {
    reduceTasks.add(task.getID());
  }
  metrics.waitingTask(task);
}

Example 3

Source File: DefaultSpeculator.java From big-c with Apache License 2.0

6 votes

private AtomicInteger containerNeed(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  TaskType taskType = taskID.getTaskType();

  ConcurrentMap<JobId, AtomicInteger> relevantMap
      = taskType == TaskType.MAP ? mapContainerNeeds : reduceContainerNeeds;

  AtomicInteger result = relevantMap.get(jobID);

  if (result == null) {
    relevantMap.putIfAbsent(jobID, new AtomicInteger(0));
    result = relevantMap.get(jobID);
  }

  return result;
}

Example 4

Source File: StartEndTimesBase.java From big-c with Apache License 2.0

6 votes

protected DataStatistics dataStatisticsForTask(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  if (job == null) {
    return null;
  }

  Task task = job.getTask(taskID);

  if (task == null) {
    return null;
  }

  return task.getType() == TaskType.MAP
          ? mapperStatistics.get(job)
          : task.getType() == TaskType.REDUCE
              ? reducerStatistics.get(job)
              : null;
}

Example 5

Source File: MapTaskImpl.java From big-c with Apache License 2.0

5 votes

public MapTaskImpl(JobId jobId, int partition, EventHandler eventHandler,
    Path remoteJobConfFile, JobConf conf,
    TaskSplitMetaInfo taskSplitMetaInfo,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    int appAttemptId, MRAppMetrics metrics, AppContext appContext) {
  super(jobId, TaskType.MAP, partition, eventHandler, remoteJobConfFile,
      conf, taskAttemptListener, jobToken, credentials, clock,
      appAttemptId, metrics, appContext);
  this.taskSplitMetaInfo = taskSplitMetaInfo;
}

Example 6

Source File: JobImpl.java From hadoop with Apache License 2.0

5 votes

private void taskSucceeded(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.succeededMapTaskCount++;
  } else {
    job.succeededReduceTaskCount++;
  }
  job.metrics.completedTask(task);
}

Example 7

Source File: TaskAttemptImpl.java From big-c with Apache License 2.0

5 votes

private int getMemoryRequired(Configuration conf, TaskType taskType) {
  int memory = 1024;
  if (taskType == TaskType.MAP)  {
    memory =
        conf.getInt(MRJobConfig.MAP_MEMORY_MB,
            MRJobConfig.DEFAULT_MAP_MEMORY_MB);
  } else if (taskType == TaskType.REDUCE) {
    memory =
        conf.getInt(MRJobConfig.REDUCE_MEMORY_MB,
            MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
  }
  
  return memory;
}

Example 8

Source File: MapTaskImpl.java From hadoop with Apache License 2.0

5 votes

public MapTaskImpl(JobId jobId, int partition, EventHandler eventHandler,
    Path remoteJobConfFile, JobConf conf,
    TaskSplitMetaInfo taskSplitMetaInfo,
    TaskAttemptListener taskAttemptListener,
    Token<JobTokenIdentifier> jobToken,
    Credentials credentials, Clock clock,
    int appAttemptId, MRAppMetrics metrics, AppContext appContext) {
  super(jobId, TaskType.MAP, partition, eventHandler, remoteJobConfFile,
      conf, taskAttemptListener, jobToken, credentials, clock,
      appAttemptId, metrics, appContext);
  this.taskSplitMetaInfo = taskSplitMetaInfo;
}

Example 9

Source File: JobImpl.java From big-c with Apache License 2.0

5 votes

private void taskFailed(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.failedMapTaskCount++;
  } else if (task.getType() == TaskType.REDUCE) {
    job.failedReduceTaskCount++;
  }
  job.addDiagnostic("Task failed " + task.getID());
  job.metrics.failedTask(task);
}

Example 10

Source File: TaskAttemptImpl.java From big-c with Apache License 2.0

5 votes

private static void updateMillisCounters(JobCounterUpdateEvent jce,
    TaskAttemptImpl taskAttempt) {
  TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
  long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
  int mbRequired =
      taskAttempt.getMemoryRequired(taskAttempt.conf, taskType);
  int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType);

  int minSlotMemSize = taskAttempt.conf.getInt(
    YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
    YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);

  int simSlotsRequired =
      minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) mbRequired
          / minSlotMemSize);

  if (taskType == TaskType.MAP) {
    jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, simSlotsRequired * duration);
    jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbRequired);
    jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, duration * vcoresRequired);
    jce.addCounterUpdate(JobCounter.MILLIS_MAPS, duration);
  } else {
    jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, simSlotsRequired * duration);
    jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, duration * mbRequired);
    jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, duration * vcoresRequired);
    jce.addCounterUpdate(JobCounter.MILLIS_REDUCES, duration);
  }
}

Example 11

Source File: TaskAttemptImpl.java From hadoop with Apache License 2.0

5 votes

private int getMemoryRequired(Configuration conf, TaskType taskType) {
  int memory = 1024;
  if (taskType == TaskType.MAP)  {
    memory =
        conf.getInt(MRJobConfig.MAP_MEMORY_MB,
            MRJobConfig.DEFAULT_MAP_MEMORY_MB);
  } else if (taskType == TaskType.REDUCE) {
    memory =
        conf.getInt(MRJobConfig.REDUCE_MEMORY_MB,
            MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
  }
  
  return memory;
}

Example 12

Source File: TestRuntimeEstimators.java From big-c with Apache License 2.0

5 votes

@Override
public float getProgress() {
  if (overridingState == TaskAttemptState.NEW) {
    return 0.0F;
  }
  return myAttemptID.getTaskId().getTaskType() == TaskType.MAP ? getMapProgress() : getReduceProgress();
}

Example 13

Source File: JobImpl.java From big-c with Apache License 2.0

5 votes

private void taskSucceeded(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.succeededMapTaskCount++;
  } else {
    job.succeededReduceTaskCount++;
  }
  job.metrics.completedTask(task);
}

Example 14

Source File: TaskAttemptImpl.java From big-c with Apache License 2.0

4 votes

@SuppressWarnings({ "unchecked" })
private void logAttemptFinishedEvent(TaskAttemptStateInternal state) {
  //Log finished events only if an attempt started.
  if (getLaunchTime() == 0) return; 
  String containerHostName = this.container == null ? "UNKNOWN"
       : this.container.getNodeId().getHost();
  int containerNodePort =
      this.container == null ? -1 : this.container.getNodeId().getPort();
  if (attemptId.getTaskId().getTaskType() == TaskType.MAP) {
    MapAttemptFinishedEvent mfe =
       new MapAttemptFinishedEvent(TypeConverter.fromYarn(attemptId),
       TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()),
       state.toString(),
       this.reportedStatus.mapFinishTime,
       finishTime,
       containerHostName,
       containerNodePort,
       this.nodeRackName == null ? "UNKNOWN" : this.nodeRackName,
       this.reportedStatus.stateString,
       getCounters(),
       getProgressSplitBlock().burst());
       eventHandler.handle(
         new JobHistoryEvent(attemptId.getTaskId().getJobId(), mfe));
  } else {
     ReduceAttemptFinishedEvent rfe =
       new ReduceAttemptFinishedEvent(TypeConverter.fromYarn(attemptId),
       TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()),
       state.toString(),
       this.reportedStatus.shuffleFinishTime,
       this.reportedStatus.sortFinishTime,
       finishTime,
       containerHostName,
       containerNodePort,
       this.nodeRackName == null ? "UNKNOWN" : this.nodeRackName,
       this.reportedStatus.stateString,
       getCounters(),
       getProgressSplitBlock().burst());
       eventHandler.handle(
         new JobHistoryEvent(attemptId.getTaskId().getJobId(), rfe));
  }
}

Example 15

Source File: MRApps.java From hadoop with Apache License 2.0

4 votes

public static TaskType taskType(String symbol) {
  // JDK 7 supports switch on strings
  if (symbol.equals("m")) return TaskType.MAP;
  if (symbol.equals("r")) return TaskType.REDUCE;
  throw new YarnRuntimeException("Unknown task symbol: "+ symbol);
}

Example 16

Source File: TestTaskImpl.java From hadoop with Apache License 2.0

4 votes

@Test
public void testCountersWithSpeculation() {
  mockTask = new MockTaskImpl(jobId, partition, dispatcher.getEventHandler(),
      remoteJobConfFile, conf, taskAttemptListener, jobToken,
      credentials, clock, startCount, metrics, appContext, TaskType.MAP) {
        @Override
        protected int getMaxAttempts() {
          return 1;
        }
  };
  TaskId taskId = getNewTaskID();
  scheduleTaskAttempt(taskId);
  launchTaskAttempt(getLastAttempt().getAttemptId());
  updateLastAttemptState(TaskAttemptState.RUNNING);
  MockTaskAttemptImpl baseAttempt = getLastAttempt();

  // add a speculative attempt
  mockTask.handle(new TaskTAttemptEvent(getLastAttempt().getAttemptId(),
      TaskEventType.T_ADD_SPEC_ATTEMPT));
  launchTaskAttempt(getLastAttempt().getAttemptId());
  updateLastAttemptState(TaskAttemptState.RUNNING);
  MockTaskAttemptImpl specAttempt = getLastAttempt();
  assertEquals(2, taskAttempts.size());

  Counters specAttemptCounters = new Counters();
  Counter cpuCounter = specAttemptCounters.findCounter(
      TaskCounter.CPU_MILLISECONDS);
  cpuCounter.setValue(1000);
  specAttempt.setCounters(specAttemptCounters);

  // have the spec attempt succeed but second attempt at 1.0 progress as well
  commitTaskAttempt(specAttempt.getAttemptId());
  specAttempt.setProgress(1.0f);
  specAttempt.setState(TaskAttemptState.SUCCEEDED);
  mockTask.handle(new TaskTAttemptEvent(specAttempt.getAttemptId(),
      TaskEventType.T_ATTEMPT_SUCCEEDED));
  assertEquals(TaskState.SUCCEEDED, mockTask.getState());
  baseAttempt.setProgress(1.0f);

  Counters taskCounters = mockTask.getCounters();
  assertEquals("wrong counters for task", specAttemptCounters, taskCounters);
}

Example 17

Source File: TestTaskImpl.java From big-c with Apache License 2.0

4 votes

@Test
public void testCountersWithSpeculation() {
  mockTask = new MockTaskImpl(jobId, partition, dispatcher.getEventHandler(),
      remoteJobConfFile, conf, taskAttemptListener, jobToken,
      credentials, clock, startCount, metrics, appContext, TaskType.MAP) {
        @Override
        protected int getMaxAttempts() {
          return 1;
        }
  };
  TaskId taskId = getNewTaskID();
  scheduleTaskAttempt(taskId);
  launchTaskAttempt(getLastAttempt().getAttemptId());
  updateLastAttemptState(TaskAttemptState.RUNNING);
  MockTaskAttemptImpl baseAttempt = getLastAttempt();

  // add a speculative attempt
  mockTask.handle(new TaskTAttemptEvent(getLastAttempt().getAttemptId(),
      TaskEventType.T_ADD_SPEC_ATTEMPT));
  launchTaskAttempt(getLastAttempt().getAttemptId());
  updateLastAttemptState(TaskAttemptState.RUNNING);
  MockTaskAttemptImpl specAttempt = getLastAttempt();
  assertEquals(2, taskAttempts.size());

  Counters specAttemptCounters = new Counters();
  Counter cpuCounter = specAttemptCounters.findCounter(
      TaskCounter.CPU_MILLISECONDS);
  cpuCounter.setValue(1000);
  specAttempt.setCounters(specAttemptCounters);

  // have the spec attempt succeed but second attempt at 1.0 progress as well
  commitTaskAttempt(specAttempt.getAttemptId());
  specAttempt.setProgress(1.0f);
  specAttempt.setState(TaskAttemptState.SUCCEEDED);
  mockTask.handle(new TaskTAttemptEvent(specAttempt.getAttemptId(),
      TaskEventType.T_ATTEMPT_SUCCEEDED));
  assertEquals(TaskState.SUCCEEDED, mockTask.getState());
  baseAttempt.setProgress(1.0f);

  Counters taskCounters = mockTask.getCounters();
  assertEquals("wrong counters for task", specAttemptCounters, taskCounters);
}

Example 18

Source File: TestRuntimeEstimators.java From hadoop with Apache License 2.0

4 votes

@Override
public Map<TaskId, Task> getTasks(TaskType taskType) {
  return taskType == TaskType.MAP ? mapTasks : reduceTasks;
}

Example 19

Source File: MapTaskImpl.java From big-c with Apache License 2.0

4 votes

@Override
public TaskType getType() {
  return TaskType.MAP;
}

Example 20

Source File: TestRuntimeEstimators.java From big-c with Apache License 2.0

4 votes

int taskTypeSlots(TaskType type) {
  return type == TaskType.MAP ? MAP_SLOT_REQUIREMENT : REDUCE_SLOT_REQUIREMENT;
}