org.apache.tez.common.counters.TezCounter Java Examples

public ValuesIterator (TezRawKeyValueIterator in, 
                       RawComparator<KEY> comparator, 
                       Class<KEY> keyClass,
                       Class<VALUE> valClass, Configuration conf,
                       TezCounter inputKeyCounter,
                       TezCounter inputValueCounter)
  throws IOException {
  this.in = in;
  this.comparator = comparator;
  this.inputKeyCounter = inputKeyCounter;
  this.inputValueCounter = inputValueCounter;
  SerializationFactory serializationFactory = new SerializationFactory(conf);
  this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
  this.keyDeserializer.open(keyIn);
  this.valDeserializer = serializationFactory.getDeserializer(valClass);
  this.valDeserializer.open(this.valueIn);
}
 

public static <K extends Object, V extends Object>
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass,
                          List<Segment> segments,
                          int mergeFactor, Path tmpDir,
                          RawComparator comparator, Progressable reporter,
                          boolean sortSegments,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
    throws IOException {
  return new MergeQueue(conf, fs, segments, comparator, reporter,
                         sortSegments, false).merge(keyClass, valueClass,
                                             mergeFactor, tmpDir,
                                             readsCounter, writesCounter,
                                             bytesReadCounter, mergePhase);
}
 

public static 
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass, 
                          CompressionCodec codec, boolean ifileReadAhead,
                          int ifileReadAheadLength, int ifileBufferSize,
                          Path[] inputs, boolean deleteInputs, 
                          int mergeFactor, Path tmpDir,
                          RawComparator comparator,
                          Progressable reporter,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter mergedMapOutputsCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
throws IOException {
  return 
    new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead,
                         ifileReadAheadLength, ifileBufferSize, false, comparator, 
                         reporter, mergedMapOutputsCounter).merge(
                                         keyClass, valueClass,
                                         mergeFactor, tmpDir,
                                         readsCounter, writesCounter,
                                         bytesReadCounter,
                                         mergePhase);
}
 

public static
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass, 
                          CompressionCodec codec, boolean ifileReadAhead,
                          int ifileReadAheadLength, int ifileBufferSize,
                          Path[] inputs, boolean deleteInputs, 
                          int mergeFactor, Path tmpDir,
                          RawComparator comparator, Progressable reporter,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
throws IOException {
  return 
    new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead,
                         ifileReadAheadLength, ifileBufferSize, false, comparator, 
                         reporter, null).merge(keyClass, valueClass,
                                         mergeFactor, tmpDir,
                                         readsCounter, writesCounter,
                                         bytesReadCounter,
                                         mergePhase);
}
 

public MRReaderMapReduce(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters,
    TezCounter inputRecordCounter, long clusterId, int vertexIndex, int appId, int taskIndex,
    int taskAttemptNumber, InputContext context) throws IOException {
  super(context);
  this.inputRecordCounter = inputRecordCounter;
  this.taskAttemptContext = new TaskAttemptContextImpl(jobConf, tezCounters, clusterId,
      vertexIndex, appId, taskIndex, taskAttemptNumber, true, null);

  Class<? extends org.apache.hadoop.mapreduce.InputFormat<?, ?>> inputFormatClazz;

  try {
    inputFormatClazz = taskAttemptContext.getInputFormatClass();
  } catch (ClassNotFoundException e) {
    throw new IOException("Unable to instantiate InputFormat class", e);
  }

  inputFormat = ReflectionUtils.newInstance(inputFormatClazz, jobConf);

  if (inputSplit != null) {
    this.inputSplit = inputSplit;
    setupNewRecordReader();
  }
}
 

public Segment(Configuration conf, FileSystem fs, Path file,
    long segmentOffset, long segmentLength, CompressionCodec codec,
    boolean ifileReadAhead, int ifileReadAheadLength, int bufferSize,
    boolean preserve, TezCounter mergedMapOutputsCounter)
throws IOException {
  this.conf = conf;
  this.fs = fs;
  this.file = file;
  this.codec = codec;
  this.preserve = preserve;
  this.ifileReadAhead = ifileReadAhead;
  this.ifileReadAheadLength =ifileReadAheadLength;
  this.bufferSize = bufferSize;

  this.segmentOffset = segmentOffset;
  this.segmentLength = segmentLength;
  
  this.mapOutputsCounter = mergedMapOutputsCounter;
}
 

@Test(timeout = 5000)
public void testInterruptOnNext() throws IOException, InterruptedException {
  ShuffleManager shuffleManager = mock(ShuffleManager.class);

  // Simulate an interrupt while waiting for the next fetched input.
  doThrow(new InterruptedException()).when(shuffleManager).getNextInput();
  TezCounters counters = new TezCounters();
  TezCounter inputRecords = counters.findCounter(TaskCounter.INPUT_RECORDS_PROCESSED);
  UnorderedKVReader<Text, Text> reader =
      new UnorderedKVReader<Text, Text>(shuffleManager, defaultConf, null, false, -1, -1,
          inputRecords, mock(InputContext.class));

  try {
    reader.next();
    fail("No data available to reader. Should not be able to access any record");
  } catch (IOInterruptedException e) {
    // Expected exception. Any other should fail the test.
  }
}
 

public static Counters fromTez(TezCounters tezCounters) {
  if (tezCounters == null) {
    return null;
  }
  Counters counters = new Counters();
  for (CounterGroup xGrp : tezCounters) {
    counters.addGroup(xGrp.getName(), xGrp.getDisplayName());
    for (TezCounter xCounter : xGrp) {
      Counter counter =
          counters.findCounter(xGrp.getName(), xCounter.getName());
      counter.setValue(xCounter.getValue());

    }
  }
  return counters;
}
 

@SuppressWarnings("unchecked")
private ValuesIterator createEmptyIterator(boolean inMemory)
    throws IOException, InterruptedException {
  if (!inMemory) {
    streamPaths = new Path[0];
    //This will return EmptyIterator
    rawKeyValueIterator =
        TezMerger.merge(conf, fs, keyClass, valClass, null,
            false, -1, 1024, streamPaths, false, mergeFactor, tmpDir, comparator,
            new ProgressReporter(), null, null, null, null);
  } else {
    List<TezMerger.Segment> segments = Lists.newLinkedList();
    //This will return EmptyIterator
    rawKeyValueIterator =
        TezMerger.merge(conf, fs, keyClass, valClass, segments, mergeFactor, tmpDir,
            comparator, new ProgressReporter(), new GenericCounter("readsCounter", "y"),
            new GenericCounter("writesCounter", "y1"),
            new GenericCounter("bytesReadCounter", "y2"), new Progress());
  }
  return new ValuesIterator(rawKeyValueIterator, comparator,
      keyClass, valClass, conf, (TezCounter) new GenericCounter("inputKeyCounter", "y3"),
      (TezCounter) new GenericCounter("inputValueCounter", "y4"));
}
 

public ValuesIterator (TezRawKeyValueIterator in, 
                       RawComparator<KEY> comparator, 
                       Class<KEY> keyClass,
                       Class<VALUE> valClass, Configuration conf,
                       TezCounter inputKeyCounter,
                       TezCounter inputValueCounter)
  throws IOException {
  this.in = in;
  this.comparator = comparator;
  this.inputKeyCounter = inputKeyCounter;
  this.inputValueCounter = inputValueCounter;
  SerializationFactory serializationFactory = new SerializationFactory(conf);
  this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
  this.keyDeserializer.open(keyIn);
  this.valDeserializer = serializationFactory.getDeserializer(valClass);
  this.valDeserializer.open(this.valueIn);
}
 

public static Counters fromTez(TezCounters tezCounters) {
  if (tezCounters == null) {
    return null;
  }
  Counters counters = new Counters();
  for (CounterGroup xGrp : tezCounters) {
    counters.addGroup(xGrp.getName(), xGrp.getDisplayName());
    for (TezCounter xCounter : xGrp) {
      Counter counter =
          counters.findCounter(xGrp.getName(), xCounter.getName());
      counter.setValue(xCounter.getValue());

    }
  }
  return counters;
}
 

@Test
public void testExceedsKVWithPipelinedShuffle() throws IOException {
  this.numOutputs = 1;
  this.initialAvailableMem = 1 *1024 * 1024;
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
  conf.setInt(TezRuntimeConfiguration
      .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 1);
  PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs,
      initialAvailableMem);

  writeData(sorter, 5, 1<<20);

  // final merge is disabled. Final output file would not be populated in this case.
  assertTrue(sorter.finalOutputFile == null);
  TezCounter numShuffleChunks = outputContext.getCounters().findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT);
  assertTrue(sorter.getNumSpills() == numShuffleChunks.getValue());
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true);
}
 

public static
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass, 
                          CompressionCodec codec, boolean ifileReadAhead,
                          int ifileReadAheadLength, int ifileBufferSize,
                          Path[] inputs, boolean deleteInputs, 
                          int mergeFactor, Path tmpDir,
                          RawComparator comparator, Progressable reporter,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
    throws IOException, InterruptedException {
  return 
    new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead,
                         ifileReadAheadLength, ifileBufferSize, false, comparator, 
                         reporter, null).merge(keyClass, valueClass,
                                         mergeFactor, tmpDir,
                                         readsCounter, writesCounter,
                                         bytesReadCounter,
                                         mergePhase);
}
 

public static <K extends Object, V extends Object>
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass,
                          List<Segment> segments,
                          int mergeFactor, Path tmpDir,
                          RawComparator comparator, Progressable reporter,
                          boolean sortSegments,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
    throws IOException, InterruptedException {
  return new MergeQueue(conf, fs, segments, comparator, reporter,
                         sortSegments, false).merge(keyClass, valueClass,
                                             mergeFactor, tmpDir,
                                             readsCounter, writesCounter,
                                             bytesReadCounter, mergePhase);
}
 

@Test
  public void testEmptyDataWithPipelinedShuffle() throws IOException {
    this.numOutputs = 1;
    this.initialAvailableMem = 1 *1024 * 1024;
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
    conf.setInt(TezRuntimeConfiguration
        .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 1);
    PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs,
        initialAvailableMem);

    writeData(sorter, 0, 1<<20);

    // final merge is disabled. Final output file would not be populated in this case.
    assertTrue(sorter.finalOutputFile == null);
    TezCounter numShuffleChunks = outputContext.getCounters().findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT);
//    assertTrue(sorter.getNumSpills() == numShuffleChunks.getValue());
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true);

  }
 

/**
 * Create sample data (in memory), with an attached counter  and return ValuesIterator
 *
 * @param inMemory
 * @param keyCounter
 * @param tupleCounter
 * @return ValuesIterator
 * @throws IOException
 */
@SuppressWarnings("unchecked")
private ValuesIterator createCountedIterator(boolean inMemory, TezCounter keyCounter, TezCounter tupleCounter)
    throws IOException, InterruptedException {
  if (!inMemory) {
    streamPaths = createFiles();
    //Merge all files to get KeyValueIterator
    rawKeyValueIterator =
        TezMerger.merge(conf, fs, keyClass, valClass, null,
            false, -1, 1024, streamPaths, false, mergeFactor, tmpDir, comparator,
            new ProgressReporter(), null, null, null, null);
  } else {
    List<TezMerger.Segment> segments = createInMemStreams();
    rawKeyValueIterator =
        TezMerger.merge(conf, fs, keyClass, valClass, segments, mergeFactor, tmpDir,
            comparator, new ProgressReporter(), new GenericCounter("readsCounter", "y"),
            new GenericCounter("writesCounter", "y1"),
            new GenericCounter("bytesReadCounter", "y2"), new Progress());
  }
  return new ValuesIterator(rawKeyValueIterator, comparator,
      keyClass, valClass, conf, keyCounter, tupleCounter);
}
 

public MRReaderMapReduce(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters,
    TezCounter inputRecordCounter, long clusterId, int vertexIndex, int appId, int taskIndex,
    int taskAttemptNumber) throws IOException {
  this.inputRecordCounter = inputRecordCounter;
  this.taskAttemptContext = new TaskAttemptContextImpl(jobConf, tezCounters, clusterId,
      vertexIndex, appId, taskIndex, taskAttemptNumber, true, null);

  Class<? extends org.apache.hadoop.mapreduce.InputFormat<?, ?>> inputFormatClazz;
  try {
    inputFormatClazz = taskAttemptContext.getInputFormatClass();
  } catch (ClassNotFoundException e) {
    throw new IOException("Unable to instantiate InputFormat class", e);
  }
  inputFormat = ReflectionUtils.newInstance(inputFormatClazz, jobConf);

  if (inputSplit != null) {
    this.inputSplit = inputSplit;
    setupNewRecordReader();
  }
}
 

public Writer(Configuration conf, FSDataOutputStream outputStream,
    Class keyClass, Class valueClass,
    CompressionCodec codec, TezCounter writesCounter, TezCounter serializedBytesCounter,
    boolean rle) throws IOException {
  this.rawOut = outputStream;
  this.writtenRecordsCounter = writesCounter;
  this.serializedUncompressedBytes = serializedBytesCounter;
  this.start = this.rawOut.getPos();
  this.rle = rle;

  setupOutputStream(codec);

  writeHeader(outputStream);

  if (keyClass != null) {
    this.closeSerializers = true;
    SerializationFactory serializationFactory =
      new SerializationFactory(conf);
    this.keySerializer = serializationFactory.getSerializer(keyClass);
    this.keySerializer.open(buffer);
    this.valueSerializer = serializationFactory.getSerializer(valueClass);
    this.valueSerializer.open(buffer);
  } else {
    this.closeSerializers = false;
  }
}
 

public static <K extends Object, V extends Object>
  TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass,
                          CompressionCodec codec,
                          List<Segment> segments,
                          int mergeFactor, int inMemSegments, Path tmpDir,
                          RawComparator comparator, Progressable reporter,
                          boolean sortSegments,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
      throws IOException, InterruptedException {
  return new MergeQueue(conf, fs, segments, comparator, reporter,
                         sortSegments, codec, false).merge(keyClass, valueClass,
                                             mergeFactor, inMemSegments,
                                             tmpDir,
                                             readsCounter, writesCounter,
                                             bytesReadCounter,
                                             mergePhase);
}
 

private float getAvgCounter(Collection<TaskAttemptInfo> taskAttemptInfos, String
    counterGroupName, String counterName) {
  long total = 0;
  int taskCount = 0;
  for (TaskAttemptInfo attemptInfo : taskAttemptInfos) {
    TezCounters tezCounters = attemptInfo.getTezCounters();
    TezCounter counter = tezCounters.findCounter(counterGroupName, counterName);
    if (counter != null) {
      total += counter.getValue();
      taskCount++;
    } else {
      LOG.info("Could not find counterGroupName=" + counterGroupName + ", counter=" +
          counterName + " in " + attemptInfo);
    }
  }
  return (taskCount > 0) ? (total * 1.0f / taskCount) : 0;
}
 

private void verifyCountedIteratorReader(boolean inMemory) throws IOException, InterruptedException {
  TezCounter keyCounter = new GenericCounter("inputKeyCounter", "y3");
  TezCounter tupleCounter = new GenericCounter("inputValuesCounter", "y4");
  ValuesIterator iterator = createCountedIterator(inMemory, keyCounter,
      tupleCounter);
  List<Integer> sequence = verifyIteratorData(iterator);
  if (expectedTestResult) {
    assertEquals((long) sequence.size(), keyCounter.getValue());
    long rows = 0;
    for (Integer i : sequence) {
      rows += i.longValue();
    }
    assertEquals(rows, tupleCounter.getValue());
  }
}
 

public DiskSegment(FileSystem fs, Path file,
                 CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLenth,
                 int bufferSize, boolean preserve, TezCounter mergedMapOutputsCounter)
throws IOException {
    this(fs, file, 0, fs.getFileStatus(file).getLen(), codec,
        ifileReadAhead, ifileReadAheadLenth, bufferSize, preserve,
        mergedMapOutputsCounter);
  }
 

@Override
public TezCounter findCounter(String groupName, String counterName) {
  if (groupName.equals(TaskCounter.class.getName())) {
    groupName = TaskCounter.class.getSimpleName();
  }
  String modifiedGroupName = groupName + "_" + this.groupModifier;
  return original.findCounter(modifiedGroupName, counterName);
}
 

@Override
public TezCounter findCounter(String groupName, String counterName) {
  String simpleGroupName;
  if (groupName.equals(TaskCounter.class.getName())) {
    simpleGroupName = TaskCounter.class.getSimpleName();
  } else  {
    simpleGroupName = groupName;
  }
  String modifiedGroupName = simpleGroupName + "_" + this.groupModifier;
  final TezCounter modifiedGroupCounter = original.findCounter(modifiedGroupName, counterName);
  final TezCounter originalGroupCounter = original.findCounter(groupName, counterName);
  return new CompositeCounter(modifiedGroupCounter, originalGroupCounter);
}
 

public Writer(Configuration conf, FileSystem fs, Path file,
              Class keyClass, Class valueClass,
              CompressionCodec codec,
              TezCounter writesCounter,
              TezCounter serializedBytesCounter) throws IOException {
  this(conf, fs.create(file), keyClass, valueClass, codec,
       writesCounter, serializedBytesCounter);
  ownOutputStream = true;
}
 

@Override
public boolean incrCounter(String group, String name, long delta) {
    if (context == null) {
        return false;
    }
    TezCounter counter = context.getCounters().getGroup(group).findCounter(name);
    counter.increment(delta);
    return true;
}
 

private void verifyCounters(PipelinedSorter sorter, OutputContext context) {
  TezCounter numShuffleChunks = context.getCounters().findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT);
  TezCounter additionalSpills =
      context.getCounters().findCounter(TaskCounter.ADDITIONAL_SPILL_COUNT);
  TezCounter additionalSpillBytesWritten =
      context.getCounters().findCounter(TaskCounter.ADDITIONAL_SPILLS_BYTES_WRITTEN);
  TezCounter additionalSpillBytesRead =
      context.getCounters().findCounter(TaskCounter.ADDITIONAL_SPILLS_BYTES_READ);

  if (sorter.isFinalMergeEnabled()) {
    assertTrue(additionalSpills.getValue() == (sorter.getNumSpills() - 1));
    //Number of files served by shuffle-handler
    assertTrue(1 == numShuffleChunks.getValue());
    if (sorter.getNumSpills() > 1) {
      assertTrue(additionalSpillBytesRead.getValue() > 0);
      assertTrue(additionalSpillBytesWritten.getValue() > 0);
    }
  } else {
    assertTrue(0 == additionalSpills.getValue());
    //Number of files served by shuffle-handler
    assertTrue(sorter.getNumSpills() == numShuffleChunks.getValue());
    assertTrue(additionalSpillBytesRead.getValue() == 0);
    assertTrue(additionalSpillBytesWritten.getValue() == 0);
  }

  TezCounter finalOutputBytes =
      context.getCounters().findCounter(TaskCounter.OUTPUT_BYTES_PHYSICAL);
  assertTrue(finalOutputBytes.getValue() >= 0);

  TezCounter outputBytesWithOverheadCounter = context.getCounters().findCounter
      (TaskCounter.OUTPUT_BYTES_WITH_OVERHEAD);
  assertTrue(outputBytesWithOverheadCounter.getValue() >= 0);
}
 

public MRReaderMapred(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters,
    TezCounter inputRecordCounter, InputContext context) throws IOException {
  super(context);
  this.jobConf = jobConf;
  this.tezCounters = tezCounters;
  this.inputRecordCounter = inputRecordCounter;
  inputFormat = this.jobConf.getInputFormat();

  if (inputSplit != null) {
    this.inputSplit = inputSplit;
    setupOldRecordReader();
  }
}
 

private String getCounterValue(TaskCounter counter, String counterGroupName,
    TaskAttemptInfo attemptInfo) {
  Map<String, TezCounter> tezCounterMap = attemptInfo.getCounter(counter.toString());
  if (tezCounterMap != null) {
    for (Map.Entry<String, TezCounter> entry : tezCounterMap.entrySet()) {
      String groupName = entry.getKey();
      long val = entry.getValue().getValue();
      if (groupName.equals(counterGroupName)) {
        return Long.toString(val);
      }
    }
  }
  return "";
}
 

@SuppressWarnings("unchecked")
public static InputSplit getOldSplitDetailsFromDisk(TaskSplitIndex splitMetaInfo,
    JobConf jobConf, TezCounter splitBytesCounter) throws IOException {
  Path file = new Path(splitMetaInfo.getSplitLocation());
  FileSystem fs = FileSystem.getLocal(jobConf);
  file = fs.makeQualified(file);
  LOG.info("Reading input split file from : " + file);
  long offset = splitMetaInfo.getStartOffset();

  FSDataInputStream inFile = fs.open(file);
  inFile.seek(offset);
  String className = Text.readString(inFile);
  Class<org.apache.hadoop.mapred.InputSplit> cls;
  try {
    cls = (Class<org.apache.hadoop.mapred.InputSplit>) jobConf.getClassByName(className);
  } catch (ClassNotFoundException ce) {
    IOException wrap = new IOException("Split class " + className + " not found");
    wrap.initCause(ce);
    throw wrap;
  }
  SerializationFactory factory = new SerializationFactory(jobConf);
  Deserializer<org.apache.hadoop.mapred.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapred.InputSplit>) factory
      .getDeserializer(cls);
  deserializer.open(inFile);
  org.apache.hadoop.mapred.InputSplit split = deserializer.deserialize(null);
  long pos = inFile.getPos();
  if (splitBytesCounter != null) {
    splitBytesCounter.increment(pos - offset);
  }
  inFile.close();
  return split;
}