* Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *     http://www.apache.org/licenses/LICENSE-2.0
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * See the License for the specific language governing permissions and
 * limitations under the License.

package org.apache.hadoop.mapred;

import static org.apache.hadoop.mapred.Task.Counter.COMBINE_INPUT_RECORDS;
import static org.apache.hadoop.mapred.Task.Counter.COMBINE_OUTPUT_RECORDS;
import static org.apache.hadoop.mapred.Task.Counter.MAP_INPUT_BYTES;
import static org.apache.hadoop.mapred.Task.Counter.MAP_INPUT_RECORDS;
import static org.apache.hadoop.mapred.Task.Counter.MAP_OUTPUT_BYTES;
import static org.apache.hadoop.mapred.Task.Counter.MAP_OUTPUT_RECORDS;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.io.serializer.Deserializer;
import org.apache.hadoop.io.serializer.SerializationFactory;
import org.apache.hadoop.io.serializer.Serializer;
import org.apache.hadoop.mapred.IFile.Writer;
import org.apache.hadoop.mapred.Merger.Segment;
import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.IndexedSortable;
import org.apache.hadoop.util.IndexedSorter;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.util.QuickSort;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ResourceCalculatorPlugin.ProcResourceValues;

/** A Map task. */
class MapTask extends Task {
   * The size of each record in the index file for the map-outputs.
  public static final int MAP_OUTPUT_INDEX_RECORD_LENGTH = 24;

  private BytesWritable split = new BytesWritable();
  private String splitClass;
  public final static int APPROX_HEADER_LENGTH = 150;
  private static final Log LOG = LogFactory.getLog(MapTask.class.getName());

  {   // set phase for this task

  public MapTask() {

  public MapTask(String jobFile, TaskAttemptID taskId, 
                 int partition, String splitClass, BytesWritable split,
                 int numSlotsRequired, String username) {
    super(jobFile, taskId, partition, numSlotsRequired, username);
    this.splitClass = splitClass;
    this.split = split;
  public boolean isMapTask() {
    return true;

  public void localizeConfiguration(JobConf conf) throws IOException {
    if (isMapOrReduce()) {
      Path localSplit = new Path(new Path(getJobFile()).getParent(), 
      LOG.debug("Writing local split to " + localSplit);
      DataOutputStream out = FileSystem.getLocal(conf).create(localSplit);
      Text.writeString(out, splitClass);
  public TaskRunner createRunner(TaskTracker tracker, 
      TaskTracker.TaskInProgress tip) {
    return new MapTaskRunner(tip, tracker, this.conf);

  public void write(DataOutput out) throws IOException {
    if (isMapOrReduce()) {
      Text.writeString(out, splitClass);
      split = null;
  public void readFields(DataInput in) throws IOException {
    if (isMapOrReduce()) {
      splitClass = Text.readString(in);

   * This class wraps the user's record reader to update the counters and progress
   * as records are read.
   * @param <K>
   * @param <V>
  class TrackedRecordReader<K, V> 
      implements RecordReader<K,V> {
    private RecordReader<K,V> rawIn;
    private Counters.Counter inputByteCounter;
    private Counters.Counter inputRecordCounter;
    private TaskReporter reporter;
    private long beforePos = -1;
    private long afterPos = -1;
    TrackedRecordReader(RecordReader<K,V> raw, TaskReporter reporter) 
      throws IOException{
      rawIn = raw;
      inputRecordCounter = reporter.getCounter(MAP_INPUT_RECORDS);
      inputByteCounter = reporter.getCounter(MAP_INPUT_BYTES);
      this.reporter = reporter;

    public K createKey() {
      return rawIn.createKey();
    public V createValue() {
      return rawIn.createValue();
    public synchronized boolean next(K key, V value)
    throws IOException {
      boolean ret = moveToNext(key, value);
      if (ret) {
      return ret;
    protected void incrCounters() {