org.apache.tez.common.counters.TezCounter Java Examples

The following examples show how to use org.apache.tez.common.counters.TezCounter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ValuesIterator.java    From tez with Apache License 2.0 6 votes vote down vote up
public ValuesIterator (TezRawKeyValueIterator in, 
                       RawComparator<KEY> comparator, 
                       Class<KEY> keyClass,
                       Class<VALUE> valClass, Configuration conf,
                       TezCounter inputKeyCounter,
                       TezCounter inputValueCounter)
  throws IOException {
  this.in = in;
  this.comparator = comparator;
  this.inputKeyCounter = inputKeyCounter;
  this.inputValueCounter = inputValueCounter;
  SerializationFactory serializationFactory = new SerializationFactory(conf);
  this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
  this.keyDeserializer.open(keyIn);
  this.valDeserializer = serializationFactory.getDeserializer(valClass);
  this.valDeserializer.open(this.valueIn);
}
 
Example #2
Source File: TezMerger.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public static <K extends Object, V extends Object>
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass,
                          List<Segment> segments,
                          int mergeFactor, Path tmpDir,
                          RawComparator comparator, Progressable reporter,
                          boolean sortSegments,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
    throws IOException {
  return new MergeQueue(conf, fs, segments, comparator, reporter,
                         sortSegments, false).merge(keyClass, valueClass,
                                             mergeFactor, tmpDir,
                                             readsCounter, writesCounter,
                                             bytesReadCounter, mergePhase);
}
 
Example #3
Source File: TezMerger.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public static 
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass, 
                          CompressionCodec codec, boolean ifileReadAhead,
                          int ifileReadAheadLength, int ifileBufferSize,
                          Path[] inputs, boolean deleteInputs, 
                          int mergeFactor, Path tmpDir,
                          RawComparator comparator,
                          Progressable reporter,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter mergedMapOutputsCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
throws IOException {
  return 
    new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead,
                         ifileReadAheadLength, ifileBufferSize, false, comparator, 
                         reporter, mergedMapOutputsCounter).merge(
                                         keyClass, valueClass,
                                         mergeFactor, tmpDir,
                                         readsCounter, writesCounter,
                                         bytesReadCounter,
                                         mergePhase);
}
 
Example #4
Source File: TezMerger.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public static
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass, 
                          CompressionCodec codec, boolean ifileReadAhead,
                          int ifileReadAheadLength, int ifileBufferSize,
                          Path[] inputs, boolean deleteInputs, 
                          int mergeFactor, Path tmpDir,
                          RawComparator comparator, Progressable reporter,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
throws IOException {
  return 
    new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead,
                         ifileReadAheadLength, ifileBufferSize, false, comparator, 
                         reporter, null).merge(keyClass, valueClass,
                                         mergeFactor, tmpDir,
                                         readsCounter, writesCounter,
                                         bytesReadCounter,
                                         mergePhase);
}
 
Example #5
Source File: MRReaderMapReduce.java    From tez with Apache License 2.0 6 votes vote down vote up
public MRReaderMapReduce(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters,
    TezCounter inputRecordCounter, long clusterId, int vertexIndex, int appId, int taskIndex,
    int taskAttemptNumber, InputContext context) throws IOException {
  super(context);
  this.inputRecordCounter = inputRecordCounter;
  this.taskAttemptContext = new TaskAttemptContextImpl(jobConf, tezCounters, clusterId,
      vertexIndex, appId, taskIndex, taskAttemptNumber, true, null);

  Class<? extends org.apache.hadoop.mapreduce.InputFormat<?, ?>> inputFormatClazz;

  try {
    inputFormatClazz = taskAttemptContext.getInputFormatClass();
  } catch (ClassNotFoundException e) {
    throw new IOException("Unable to instantiate InputFormat class", e);
  }

  inputFormat = ReflectionUtils.newInstance(inputFormatClazz, jobConf);

  if (inputSplit != null) {
    this.inputSplit = inputSplit;
    setupNewRecordReader();
  }
}
 
Example #6
Source File: TezMerger.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public Segment(Configuration conf, FileSystem fs, Path file,
    long segmentOffset, long segmentLength, CompressionCodec codec,
    boolean ifileReadAhead, int ifileReadAheadLength, int bufferSize,
    boolean preserve, TezCounter mergedMapOutputsCounter)
throws IOException {
  this.conf = conf;
  this.fs = fs;
  this.file = file;
  this.codec = codec;
  this.preserve = preserve;
  this.ifileReadAhead = ifileReadAhead;
  this.ifileReadAheadLength =ifileReadAheadLength;
  this.bufferSize = bufferSize;

  this.segmentOffset = segmentOffset;
  this.segmentLength = segmentLength;
  
  this.mapOutputsCounter = mergedMapOutputsCounter;
}
 
Example #7
Source File: TestUnorderedKVReader.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 5000)
public void testInterruptOnNext() throws IOException, InterruptedException {
  ShuffleManager shuffleManager = mock(ShuffleManager.class);

  // Simulate an interrupt while waiting for the next fetched input.
  doThrow(new InterruptedException()).when(shuffleManager).getNextInput();
  TezCounters counters = new TezCounters();
  TezCounter inputRecords = counters.findCounter(TaskCounter.INPUT_RECORDS_PROCESSED);
  UnorderedKVReader<Text, Text> reader =
      new UnorderedKVReader<Text, Text>(shuffleManager, defaultConf, null, false, -1, -1,
          inputRecords, mock(InputContext.class));

  try {
    reader.next();
    fail("No data available to reader. Should not be able to access any record");
  } catch (IOInterruptedException e) {
    // Expected exception. Any other should fail the test.
  }
}
 
Example #8
Source File: TezTypeConverters.java    From tez with Apache License 2.0 6 votes vote down vote up
public static Counters fromTez(TezCounters tezCounters) {
  if (tezCounters == null) {
    return null;
  }
  Counters counters = new Counters();
  for (CounterGroup xGrp : tezCounters) {
    counters.addGroup(xGrp.getName(), xGrp.getDisplayName());
    for (TezCounter xCounter : xGrp) {
      Counter counter =
          counters.findCounter(xGrp.getName(), xCounter.getName());
      counter.setValue(xCounter.getValue());

    }
  }
  return counters;
}
 
Example #9
Source File: TestValuesIterator.java    From tez with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private ValuesIterator createEmptyIterator(boolean inMemory)
    throws IOException, InterruptedException {
  if (!inMemory) {
    streamPaths = new Path[0];
    //This will return EmptyIterator
    rawKeyValueIterator =
        TezMerger.merge(conf, fs, keyClass, valClass, null,
            false, -1, 1024, streamPaths, false, mergeFactor, tmpDir, comparator,
            new ProgressReporter(), null, null, null, null);
  } else {
    List<TezMerger.Segment> segments = Lists.newLinkedList();
    //This will return EmptyIterator
    rawKeyValueIterator =
        TezMerger.merge(conf, fs, keyClass, valClass, segments, mergeFactor, tmpDir,
            comparator, new ProgressReporter(), new GenericCounter("readsCounter", "y"),
            new GenericCounter("writesCounter", "y1"),
            new GenericCounter("bytesReadCounter", "y2"), new Progress());
  }
  return new ValuesIterator(rawKeyValueIterator, comparator,
      keyClass, valClass, conf, (TezCounter) new GenericCounter("inputKeyCounter", "y3"),
      (TezCounter) new GenericCounter("inputValueCounter", "y4"));
}
 
Example #10
Source File: ValuesIterator.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public ValuesIterator (TezRawKeyValueIterator in, 
                       RawComparator<KEY> comparator, 
                       Class<KEY> keyClass,
                       Class<VALUE> valClass, Configuration conf,
                       TezCounter inputKeyCounter,
                       TezCounter inputValueCounter)
  throws IOException {
  this.in = in;
  this.comparator = comparator;
  this.inputKeyCounter = inputKeyCounter;
  this.inputValueCounter = inputValueCounter;
  SerializationFactory serializationFactory = new SerializationFactory(conf);
  this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
  this.keyDeserializer.open(keyIn);
  this.valDeserializer = serializationFactory.getDeserializer(valClass);
  this.valDeserializer.open(this.valueIn);
}
 
Example #11
Source File: TezTypeConverters.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public static Counters fromTez(TezCounters tezCounters) {
  if (tezCounters == null) {
    return null;
  }
  Counters counters = new Counters();
  for (CounterGroup xGrp : tezCounters) {
    counters.addGroup(xGrp.getName(), xGrp.getDisplayName());
    for (TezCounter xCounter : xGrp) {
      Counter counter =
          counters.findCounter(xGrp.getName(), xCounter.getName());
      counter.setValue(xCounter.getValue());

    }
  }
  return counters;
}
 
Example #12
Source File: TestPipelinedSorter.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test
public void testExceedsKVWithPipelinedShuffle() throws IOException {
  this.numOutputs = 1;
  this.initialAvailableMem = 1 *1024 * 1024;
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
  conf.setInt(TezRuntimeConfiguration
      .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 1);
  PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs,
      initialAvailableMem);

  writeData(sorter, 5, 1<<20);

  // final merge is disabled. Final output file would not be populated in this case.
  assertTrue(sorter.finalOutputFile == null);
  TezCounter numShuffleChunks = outputContext.getCounters().findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT);
  assertTrue(sorter.getNumSpills() == numShuffleChunks.getValue());
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true);
}
 
Example #13
Source File: TezMerger.java    From tez with Apache License 2.0 6 votes vote down vote up
public static
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass, 
                          CompressionCodec codec, boolean ifileReadAhead,
                          int ifileReadAheadLength, int ifileBufferSize,
                          Path[] inputs, boolean deleteInputs, 
                          int mergeFactor, Path tmpDir,
                          RawComparator comparator, Progressable reporter,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
    throws IOException, InterruptedException {
  return 
    new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead,
                         ifileReadAheadLength, ifileBufferSize, false, comparator, 
                         reporter, null).merge(keyClass, valueClass,
                                         mergeFactor, tmpDir,
                                         readsCounter, writesCounter,
                                         bytesReadCounter,
                                         mergePhase);
}
 
Example #14
Source File: TezMerger.java    From tez with Apache License 2.0 6 votes vote down vote up
public static <K extends Object, V extends Object>
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass,
                          List<Segment> segments,
                          int mergeFactor, Path tmpDir,
                          RawComparator comparator, Progressable reporter,
                          boolean sortSegments,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
    throws IOException, InterruptedException {
  return new MergeQueue(conf, fs, segments, comparator, reporter,
                         sortSegments, false).merge(keyClass, valueClass,
                                             mergeFactor, tmpDir,
                                             readsCounter, writesCounter,
                                             bytesReadCounter, mergePhase);
}
 
Example #15
Source File: TestPipelinedSorter.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test
  public void testEmptyDataWithPipelinedShuffle() throws IOException {
    this.numOutputs = 1;
    this.initialAvailableMem = 1 *1024 * 1024;
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
    conf.setInt(TezRuntimeConfiguration
        .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 1);
    PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs,
        initialAvailableMem);

    writeData(sorter, 0, 1<<20);

    // final merge is disabled. Final output file would not be populated in this case.
    assertTrue(sorter.finalOutputFile == null);
    TezCounter numShuffleChunks = outputContext.getCounters().findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT);
//    assertTrue(sorter.getNumSpills() == numShuffleChunks.getValue());
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true);

  }
 
Example #16
Source File: TestValuesIterator.java    From tez with Apache License 2.0 6 votes vote down vote up
/**
 * Create sample data (in memory), with an attached counter  and return ValuesIterator
 *
 * @param inMemory
 * @param keyCounter
 * @param tupleCounter
 * @return ValuesIterator
 * @throws IOException
 */
@SuppressWarnings("unchecked")
private ValuesIterator createCountedIterator(boolean inMemory, TezCounter keyCounter, TezCounter tupleCounter)
    throws IOException, InterruptedException {
  if (!inMemory) {
    streamPaths = createFiles();
    //Merge all files to get KeyValueIterator
    rawKeyValueIterator =
        TezMerger.merge(conf, fs, keyClass, valClass, null,
            false, -1, 1024, streamPaths, false, mergeFactor, tmpDir, comparator,
            new ProgressReporter(), null, null, null, null);
  } else {
    List<TezMerger.Segment> segments = createInMemStreams();
    rawKeyValueIterator =
        TezMerger.merge(conf, fs, keyClass, valClass, segments, mergeFactor, tmpDir,
            comparator, new ProgressReporter(), new GenericCounter("readsCounter", "y"),
            new GenericCounter("writesCounter", "y1"),
            new GenericCounter("bytesReadCounter", "y2"), new Progress());
  }
  return new ValuesIterator(rawKeyValueIterator, comparator,
      keyClass, valClass, conf, keyCounter, tupleCounter);
}
 
Example #17
Source File: MRReaderMapReduce.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
public MRReaderMapReduce(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters,
    TezCounter inputRecordCounter, long clusterId, int vertexIndex, int appId, int taskIndex,
    int taskAttemptNumber) throws IOException {
  this.inputRecordCounter = inputRecordCounter;
  this.taskAttemptContext = new TaskAttemptContextImpl(jobConf, tezCounters, clusterId,
      vertexIndex, appId, taskIndex, taskAttemptNumber, true, null);

  Class<? extends org.apache.hadoop.mapreduce.InputFormat<?, ?>> inputFormatClazz;
  try {
    inputFormatClazz = taskAttemptContext.getInputFormatClass();
  } catch (ClassNotFoundException e) {
    throw new IOException("Unable to instantiate InputFormat class", e);
  }
  inputFormat = ReflectionUtils.newInstance(inputFormatClazz, jobConf);

  if (inputSplit != null) {
    this.inputSplit = inputSplit;
    setupNewRecordReader();
  }
}
 
Example #18
Source File: IFile.java    From tez with Apache License 2.0 6 votes vote down vote up
public Writer(Configuration conf, FSDataOutputStream outputStream,
    Class keyClass, Class valueClass,
    CompressionCodec codec, TezCounter writesCounter, TezCounter serializedBytesCounter,
    boolean rle) throws IOException {
  this.rawOut = outputStream;
  this.writtenRecordsCounter = writesCounter;
  this.serializedUncompressedBytes = serializedBytesCounter;
  this.start = this.rawOut.getPos();
  this.rle = rle;

  setupOutputStream(codec);

  writeHeader(outputStream);

  if (keyClass != null) {
    this.closeSerializers = true;
    SerializationFactory serializationFactory =
      new SerializationFactory(conf);
    this.keySerializer = serializationFactory.getSerializer(keyClass);
    this.keySerializer.open(buffer);
    this.valueSerializer = serializationFactory.getSerializer(valueClass);
    this.valueSerializer.open(buffer);
  } else {
    this.closeSerializers = false;
  }
}
 
Example #19
Source File: TezMerger.java    From tez with Apache License 2.0 6 votes vote down vote up
public static <K extends Object, V extends Object>
  TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass,
                          CompressionCodec codec,
                          List<Segment> segments,
                          int mergeFactor, int inMemSegments, Path tmpDir,
                          RawComparator comparator, Progressable reporter,
                          boolean sortSegments,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
      throws IOException, InterruptedException {
  return new MergeQueue(conf, fs, segments, comparator, reporter,
                         sortSegments, codec, false).merge(keyClass, valueClass,
                                             mergeFactor, inMemSegments,
                                             tmpDir,
                                             readsCounter, writesCounter,
                                             bytesReadCounter,
                                             mergePhase);
}
 
Example #20
Source File: SlowNodeAnalyzer.java    From tez with Apache License 2.0 6 votes vote down vote up
private float getAvgCounter(Collection<TaskAttemptInfo> taskAttemptInfos, String
    counterGroupName, String counterName) {
  long total = 0;
  int taskCount = 0;
  for (TaskAttemptInfo attemptInfo : taskAttemptInfos) {
    TezCounters tezCounters = attemptInfo.getTezCounters();
    TezCounter counter = tezCounters.findCounter(counterGroupName, counterName);
    if (counter != null) {
      total += counter.getValue();
      taskCount++;
    } else {
      LOG.info("Could not find counterGroupName=" + counterGroupName + ", counter=" +
          counterName + " in " + attemptInfo);
    }
  }
  return (taskCount > 0) ? (total * 1.0f / taskCount) : 0;
}
 
Example #21
Source File: TestValuesIterator.java    From tez with Apache License 2.0 5 votes vote down vote up
private void verifyCountedIteratorReader(boolean inMemory) throws IOException, InterruptedException {
  TezCounter keyCounter = new GenericCounter("inputKeyCounter", "y3");
  TezCounter tupleCounter = new GenericCounter("inputValuesCounter", "y4");
  ValuesIterator iterator = createCountedIterator(inMemory, keyCounter,
      tupleCounter);
  List<Integer> sequence = verifyIteratorData(iterator);
  if (expectedTestResult) {
    assertEquals((long) sequence.size(), keyCounter.getValue());
    long rows = 0;
    for (Integer i : sequence) {
      rows += i.longValue();
    }
    assertEquals(rows, tupleCounter.getValue());
  }
}
 
Example #22
Source File: TezMerger.java    From tez with Apache License 2.0 5 votes vote down vote up
public DiskSegment(FileSystem fs, Path file,
                 CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLenth,
                 int bufferSize, boolean preserve, TezCounter mergedMapOutputsCounter)
throws IOException {
    this(fs, file, 0, fs.getFileStatus(file).getLen(), codec,
        ifileReadAhead, ifileReadAheadLenth, bufferSize, preserve,
        mergedMapOutputsCounter);
  }
 
Example #23
Source File: TezCountersDelegate.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@Override
public TezCounter findCounter(String groupName, String counterName) {
  if (groupName.equals(TaskCounter.class.getName())) {
    groupName = TaskCounter.class.getSimpleName();
  }
  String modifiedGroupName = groupName + "_" + this.groupModifier;
  return original.findCounter(modifiedGroupName, counterName);
}
 
Example #24
Source File: TezCountersDelegate.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public TezCounter findCounter(String groupName, String counterName) {
  String simpleGroupName;
  if (groupName.equals(TaskCounter.class.getName())) {
    simpleGroupName = TaskCounter.class.getSimpleName();
  } else  {
    simpleGroupName = groupName;
  }
  String modifiedGroupName = simpleGroupName + "_" + this.groupModifier;
  final TezCounter modifiedGroupCounter = original.findCounter(modifiedGroupName, counterName);
  final TezCounter originalGroupCounter = original.findCounter(groupName, counterName);
  return new CompositeCounter(modifiedGroupCounter, originalGroupCounter);
}
 
Example #25
Source File: IFile.java    From tez with Apache License 2.0 5 votes vote down vote up
public Writer(Configuration conf, FileSystem fs, Path file,
              Class keyClass, Class valueClass,
              CompressionCodec codec,
              TezCounter writesCounter,
              TezCounter serializedBytesCounter) throws IOException {
  this(conf, fs.create(file), keyClass, valueClass, codec,
       writesCounter, serializedBytesCounter);
  ownOutputStream = true;
}
 
Example #26
Source File: TezTaskContext.java    From spork with Apache License 2.0 5 votes vote down vote up
@Override
public boolean incrCounter(String group, String name, long delta) {
    if (context == null) {
        return false;
    }
    TezCounter counter = context.getCounters().getGroup(group).findCounter(name);
    counter.increment(delta);
    return true;
}
 
Example #27
Source File: TestPipelinedSorter.java    From tez with Apache License 2.0 5 votes vote down vote up
private void verifyCounters(PipelinedSorter sorter, OutputContext context) {
  TezCounter numShuffleChunks = context.getCounters().findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT);
  TezCounter additionalSpills =
      context.getCounters().findCounter(TaskCounter.ADDITIONAL_SPILL_COUNT);
  TezCounter additionalSpillBytesWritten =
      context.getCounters().findCounter(TaskCounter.ADDITIONAL_SPILLS_BYTES_WRITTEN);
  TezCounter additionalSpillBytesRead =
      context.getCounters().findCounter(TaskCounter.ADDITIONAL_SPILLS_BYTES_READ);

  if (sorter.isFinalMergeEnabled()) {
    assertTrue(additionalSpills.getValue() == (sorter.getNumSpills() - 1));
    //Number of files served by shuffle-handler
    assertTrue(1 == numShuffleChunks.getValue());
    if (sorter.getNumSpills() > 1) {
      assertTrue(additionalSpillBytesRead.getValue() > 0);
      assertTrue(additionalSpillBytesWritten.getValue() > 0);
    }
  } else {
    assertTrue(0 == additionalSpills.getValue());
    //Number of files served by shuffle-handler
    assertTrue(sorter.getNumSpills() == numShuffleChunks.getValue());
    assertTrue(additionalSpillBytesRead.getValue() == 0);
    assertTrue(additionalSpillBytesWritten.getValue() == 0);
  }

  TezCounter finalOutputBytes =
      context.getCounters().findCounter(TaskCounter.OUTPUT_BYTES_PHYSICAL);
  assertTrue(finalOutputBytes.getValue() >= 0);

  TezCounter outputBytesWithOverheadCounter = context.getCounters().findCounter
      (TaskCounter.OUTPUT_BYTES_WITH_OVERHEAD);
  assertTrue(outputBytesWithOverheadCounter.getValue() >= 0);
}
 
Example #28
Source File: MRReaderMapred.java    From tez with Apache License 2.0 5 votes vote down vote up
public MRReaderMapred(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters,
    TezCounter inputRecordCounter, InputContext context) throws IOException {
  super(context);
  this.jobConf = jobConf;
  this.tezCounters = tezCounters;
  this.inputRecordCounter = inputRecordCounter;
  inputFormat = this.jobConf.getInputFormat();

  if (inputSplit != null) {
    this.inputSplit = inputSplit;
    setupOldRecordReader();
  }
}
 
Example #29
Source File: ShuffleTimeAnalyzer.java    From tez with Apache License 2.0 5 votes vote down vote up
private String getCounterValue(TaskCounter counter, String counterGroupName,
    TaskAttemptInfo attemptInfo) {
  Map<String, TezCounter> tezCounterMap = attemptInfo.getCounter(counter.toString());
  if (tezCounterMap != null) {
    for (Map.Entry<String, TezCounter> entry : tezCounterMap.entrySet()) {
      String groupName = entry.getKey();
      long val = entry.getValue().getValue();
      if (groupName.equals(counterGroupName)) {
        return Long.toString(val);
      }
    }
  }
  return "";
}
 
Example #30
Source File: MRInputUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public static InputSplit getOldSplitDetailsFromDisk(TaskSplitIndex splitMetaInfo,
    JobConf jobConf, TezCounter splitBytesCounter) throws IOException {
  Path file = new Path(splitMetaInfo.getSplitLocation());
  FileSystem fs = FileSystem.getLocal(jobConf);
  file = fs.makeQualified(file);
  LOG.info("Reading input split file from : " + file);
  long offset = splitMetaInfo.getStartOffset();

  FSDataInputStream inFile = fs.open(file);
  inFile.seek(offset);
  String className = Text.readString(inFile);
  Class<org.apache.hadoop.mapred.InputSplit> cls;
  try {
    cls = (Class<org.apache.hadoop.mapred.InputSplit>) jobConf.getClassByName(className);
  } catch (ClassNotFoundException ce) {
    IOException wrap = new IOException("Split class " + className + " not found");
    wrap.initCause(ce);
    throw wrap;
  }
  SerializationFactory factory = new SerializationFactory(jobConf);
  Deserializer<org.apache.hadoop.mapred.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapred.InputSplit>) factory
      .getDeserializer(cls);
  deserializer.open(inFile);
  org.apache.hadoop.mapred.InputSplit split = deserializer.deserialize(null);
  long pos = inFile.getPos();
  if (splitBytesCounter != null) {
    splitBytesCounter.increment(pos - offset);
  }
  inFile.close();
  return split;
}