org.apache.tez.common.counters.TezCounter Java Examples
The following examples show how to use
org.apache.tez.common.counters.TezCounter.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ValuesIterator.java From tez with Apache License 2.0 | 6 votes |
public ValuesIterator (TezRawKeyValueIterator in, RawComparator<KEY> comparator, Class<KEY> keyClass, Class<VALUE> valClass, Configuration conf, TezCounter inputKeyCounter, TezCounter inputValueCounter) throws IOException { this.in = in; this.comparator = comparator; this.inputKeyCounter = inputKeyCounter; this.inputValueCounter = inputValueCounter; SerializationFactory serializationFactory = new SerializationFactory(conf); this.keyDeserializer = serializationFactory.getDeserializer(keyClass); this.keyDeserializer.open(keyIn); this.valDeserializer = serializationFactory.getDeserializer(valClass); this.valDeserializer.open(this.valueIn); }
Example #2
Source File: TezMerger.java From incubator-tez with Apache License 2.0 | 6 votes |
public static <K extends Object, V extends Object> TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, Class keyClass, Class valueClass, List<Segment> segments, int mergeFactor, Path tmpDir, RawComparator comparator, Progressable reporter, boolean sortSegments, TezCounter readsCounter, TezCounter writesCounter, TezCounter bytesReadCounter, Progress mergePhase) throws IOException { return new MergeQueue(conf, fs, segments, comparator, reporter, sortSegments, false).merge(keyClass, valueClass, mergeFactor, tmpDir, readsCounter, writesCounter, bytesReadCounter, mergePhase); }
Example #3
Source File: TezMerger.java From incubator-tez with Apache License 2.0 | 6 votes |
public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, Class keyClass, Class valueClass, CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, int ifileBufferSize, Path[] inputs, boolean deleteInputs, int mergeFactor, Path tmpDir, RawComparator comparator, Progressable reporter, TezCounter readsCounter, TezCounter writesCounter, TezCounter mergedMapOutputsCounter, TezCounter bytesReadCounter, Progress mergePhase) throws IOException { return new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead, ifileReadAheadLength, ifileBufferSize, false, comparator, reporter, mergedMapOutputsCounter).merge( keyClass, valueClass, mergeFactor, tmpDir, readsCounter, writesCounter, bytesReadCounter, mergePhase); }
Example #4
Source File: TezMerger.java From incubator-tez with Apache License 2.0 | 6 votes |
public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, Class keyClass, Class valueClass, CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, int ifileBufferSize, Path[] inputs, boolean deleteInputs, int mergeFactor, Path tmpDir, RawComparator comparator, Progressable reporter, TezCounter readsCounter, TezCounter writesCounter, TezCounter bytesReadCounter, Progress mergePhase) throws IOException { return new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead, ifileReadAheadLength, ifileBufferSize, false, comparator, reporter, null).merge(keyClass, valueClass, mergeFactor, tmpDir, readsCounter, writesCounter, bytesReadCounter, mergePhase); }
Example #5
Source File: MRReaderMapReduce.java From tez with Apache License 2.0 | 6 votes |
public MRReaderMapReduce(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters, TezCounter inputRecordCounter, long clusterId, int vertexIndex, int appId, int taskIndex, int taskAttemptNumber, InputContext context) throws IOException { super(context); this.inputRecordCounter = inputRecordCounter; this.taskAttemptContext = new TaskAttemptContextImpl(jobConf, tezCounters, clusterId, vertexIndex, appId, taskIndex, taskAttemptNumber, true, null); Class<? extends org.apache.hadoop.mapreduce.InputFormat<?, ?>> inputFormatClazz; try { inputFormatClazz = taskAttemptContext.getInputFormatClass(); } catch (ClassNotFoundException e) { throw new IOException("Unable to instantiate InputFormat class", e); } inputFormat = ReflectionUtils.newInstance(inputFormatClazz, jobConf); if (inputSplit != null) { this.inputSplit = inputSplit; setupNewRecordReader(); } }
Example #6
Source File: TezMerger.java From incubator-tez with Apache License 2.0 | 6 votes |
public Segment(Configuration conf, FileSystem fs, Path file, long segmentOffset, long segmentLength, CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, int bufferSize, boolean preserve, TezCounter mergedMapOutputsCounter) throws IOException { this.conf = conf; this.fs = fs; this.file = file; this.codec = codec; this.preserve = preserve; this.ifileReadAhead = ifileReadAhead; this.ifileReadAheadLength =ifileReadAheadLength; this.bufferSize = bufferSize; this.segmentOffset = segmentOffset; this.segmentLength = segmentLength; this.mapOutputsCounter = mergedMapOutputsCounter; }
Example #7
Source File: TestUnorderedKVReader.java From tez with Apache License 2.0 | 6 votes |
@Test(timeout = 5000) public void testInterruptOnNext() throws IOException, InterruptedException { ShuffleManager shuffleManager = mock(ShuffleManager.class); // Simulate an interrupt while waiting for the next fetched input. doThrow(new InterruptedException()).when(shuffleManager).getNextInput(); TezCounters counters = new TezCounters(); TezCounter inputRecords = counters.findCounter(TaskCounter.INPUT_RECORDS_PROCESSED); UnorderedKVReader<Text, Text> reader = new UnorderedKVReader<Text, Text>(shuffleManager, defaultConf, null, false, -1, -1, inputRecords, mock(InputContext.class)); try { reader.next(); fail("No data available to reader. Should not be able to access any record"); } catch (IOInterruptedException e) { // Expected exception. Any other should fail the test. } }
Example #8
Source File: TezTypeConverters.java From tez with Apache License 2.0 | 6 votes |
public static Counters fromTez(TezCounters tezCounters) { if (tezCounters == null) { return null; } Counters counters = new Counters(); for (CounterGroup xGrp : tezCounters) { counters.addGroup(xGrp.getName(), xGrp.getDisplayName()); for (TezCounter xCounter : xGrp) { Counter counter = counters.findCounter(xGrp.getName(), xCounter.getName()); counter.setValue(xCounter.getValue()); } } return counters; }
Example #9
Source File: TestValuesIterator.java From tez with Apache License 2.0 | 6 votes |
@SuppressWarnings("unchecked") private ValuesIterator createEmptyIterator(boolean inMemory) throws IOException, InterruptedException { if (!inMemory) { streamPaths = new Path[0]; //This will return EmptyIterator rawKeyValueIterator = TezMerger.merge(conf, fs, keyClass, valClass, null, false, -1, 1024, streamPaths, false, mergeFactor, tmpDir, comparator, new ProgressReporter(), null, null, null, null); } else { List<TezMerger.Segment> segments = Lists.newLinkedList(); //This will return EmptyIterator rawKeyValueIterator = TezMerger.merge(conf, fs, keyClass, valClass, segments, mergeFactor, tmpDir, comparator, new ProgressReporter(), new GenericCounter("readsCounter", "y"), new GenericCounter("writesCounter", "y1"), new GenericCounter("bytesReadCounter", "y2"), new Progress()); } return new ValuesIterator(rawKeyValueIterator, comparator, keyClass, valClass, conf, (TezCounter) new GenericCounter("inputKeyCounter", "y3"), (TezCounter) new GenericCounter("inputValueCounter", "y4")); }
Example #10
Source File: ValuesIterator.java From incubator-tez with Apache License 2.0 | 6 votes |
public ValuesIterator (TezRawKeyValueIterator in, RawComparator<KEY> comparator, Class<KEY> keyClass, Class<VALUE> valClass, Configuration conf, TezCounter inputKeyCounter, TezCounter inputValueCounter) throws IOException { this.in = in; this.comparator = comparator; this.inputKeyCounter = inputKeyCounter; this.inputValueCounter = inputValueCounter; SerializationFactory serializationFactory = new SerializationFactory(conf); this.keyDeserializer = serializationFactory.getDeserializer(keyClass); this.keyDeserializer.open(keyIn); this.valDeserializer = serializationFactory.getDeserializer(valClass); this.valDeserializer.open(this.valueIn); }
Example #11
Source File: TezTypeConverters.java From incubator-tez with Apache License 2.0 | 6 votes |
public static Counters fromTez(TezCounters tezCounters) { if (tezCounters == null) { return null; } Counters counters = new Counters(); for (CounterGroup xGrp : tezCounters) { counters.addGroup(xGrp.getName(), xGrp.getDisplayName()); for (TezCounter xCounter : xGrp) { Counter counter = counters.findCounter(xGrp.getName(), xCounter.getName()); counter.setValue(xCounter.getValue()); } } return counters; }
Example #12
Source File: TestPipelinedSorter.java From tez with Apache License 2.0 | 6 votes |
@Test public void testExceedsKVWithPipelinedShuffle() throws IOException { this.numOutputs = 1; this.initialAvailableMem = 1 *1024 * 1024; conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false); conf.setInt(TezRuntimeConfiguration .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 1); PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs, initialAvailableMem); writeData(sorter, 5, 1<<20); // final merge is disabled. Final output file would not be populated in this case. assertTrue(sorter.finalOutputFile == null); TezCounter numShuffleChunks = outputContext.getCounters().findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT); assertTrue(sorter.getNumSpills() == numShuffleChunks.getValue()); conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true); }
Example #13
Source File: TezMerger.java From tez with Apache License 2.0 | 6 votes |
public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, Class keyClass, Class valueClass, CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, int ifileBufferSize, Path[] inputs, boolean deleteInputs, int mergeFactor, Path tmpDir, RawComparator comparator, Progressable reporter, TezCounter readsCounter, TezCounter writesCounter, TezCounter bytesReadCounter, Progress mergePhase) throws IOException, InterruptedException { return new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead, ifileReadAheadLength, ifileBufferSize, false, comparator, reporter, null).merge(keyClass, valueClass, mergeFactor, tmpDir, readsCounter, writesCounter, bytesReadCounter, mergePhase); }
Example #14
Source File: TezMerger.java From tez with Apache License 2.0 | 6 votes |
public static <K extends Object, V extends Object> TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, Class keyClass, Class valueClass, List<Segment> segments, int mergeFactor, Path tmpDir, RawComparator comparator, Progressable reporter, boolean sortSegments, TezCounter readsCounter, TezCounter writesCounter, TezCounter bytesReadCounter, Progress mergePhase) throws IOException, InterruptedException { return new MergeQueue(conf, fs, segments, comparator, reporter, sortSegments, false).merge(keyClass, valueClass, mergeFactor, tmpDir, readsCounter, writesCounter, bytesReadCounter, mergePhase); }
Example #15
Source File: TestPipelinedSorter.java From tez with Apache License 2.0 | 6 votes |
@Test public void testEmptyDataWithPipelinedShuffle() throws IOException { this.numOutputs = 1; this.initialAvailableMem = 1 *1024 * 1024; conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false); conf.setInt(TezRuntimeConfiguration .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 1); PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs, initialAvailableMem); writeData(sorter, 0, 1<<20); // final merge is disabled. Final output file would not be populated in this case. assertTrue(sorter.finalOutputFile == null); TezCounter numShuffleChunks = outputContext.getCounters().findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT); // assertTrue(sorter.getNumSpills() == numShuffleChunks.getValue()); conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true); }
Example #16
Source File: TestValuesIterator.java From tez with Apache License 2.0 | 6 votes |
/** * Create sample data (in memory), with an attached counter and return ValuesIterator * * @param inMemory * @param keyCounter * @param tupleCounter * @return ValuesIterator * @throws IOException */ @SuppressWarnings("unchecked") private ValuesIterator createCountedIterator(boolean inMemory, TezCounter keyCounter, TezCounter tupleCounter) throws IOException, InterruptedException { if (!inMemory) { streamPaths = createFiles(); //Merge all files to get KeyValueIterator rawKeyValueIterator = TezMerger.merge(conf, fs, keyClass, valClass, null, false, -1, 1024, streamPaths, false, mergeFactor, tmpDir, comparator, new ProgressReporter(), null, null, null, null); } else { List<TezMerger.Segment> segments = createInMemStreams(); rawKeyValueIterator = TezMerger.merge(conf, fs, keyClass, valClass, segments, mergeFactor, tmpDir, comparator, new ProgressReporter(), new GenericCounter("readsCounter", "y"), new GenericCounter("writesCounter", "y1"), new GenericCounter("bytesReadCounter", "y2"), new Progress()); } return new ValuesIterator(rawKeyValueIterator, comparator, keyClass, valClass, conf, keyCounter, tupleCounter); }
Example #17
Source File: MRReaderMapReduce.java From incubator-tez with Apache License 2.0 | 6 votes |
public MRReaderMapReduce(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters, TezCounter inputRecordCounter, long clusterId, int vertexIndex, int appId, int taskIndex, int taskAttemptNumber) throws IOException { this.inputRecordCounter = inputRecordCounter; this.taskAttemptContext = new TaskAttemptContextImpl(jobConf, tezCounters, clusterId, vertexIndex, appId, taskIndex, taskAttemptNumber, true, null); Class<? extends org.apache.hadoop.mapreduce.InputFormat<?, ?>> inputFormatClazz; try { inputFormatClazz = taskAttemptContext.getInputFormatClass(); } catch (ClassNotFoundException e) { throw new IOException("Unable to instantiate InputFormat class", e); } inputFormat = ReflectionUtils.newInstance(inputFormatClazz, jobConf); if (inputSplit != null) { this.inputSplit = inputSplit; setupNewRecordReader(); } }
Example #18
Source File: IFile.java From tez with Apache License 2.0 | 6 votes |
public Writer(Configuration conf, FSDataOutputStream outputStream, Class keyClass, Class valueClass, CompressionCodec codec, TezCounter writesCounter, TezCounter serializedBytesCounter, boolean rle) throws IOException { this.rawOut = outputStream; this.writtenRecordsCounter = writesCounter; this.serializedUncompressedBytes = serializedBytesCounter; this.start = this.rawOut.getPos(); this.rle = rle; setupOutputStream(codec); writeHeader(outputStream); if (keyClass != null) { this.closeSerializers = true; SerializationFactory serializationFactory = new SerializationFactory(conf); this.keySerializer = serializationFactory.getSerializer(keyClass); this.keySerializer.open(buffer); this.valueSerializer = serializationFactory.getSerializer(valueClass); this.valueSerializer.open(buffer); } else { this.closeSerializers = false; } }
Example #19
Source File: TezMerger.java From tez with Apache License 2.0 | 6 votes |
public static <K extends Object, V extends Object> TezRawKeyValueIterator merge(Configuration conf, FileSystem fs, Class keyClass, Class valueClass, CompressionCodec codec, List<Segment> segments, int mergeFactor, int inMemSegments, Path tmpDir, RawComparator comparator, Progressable reporter, boolean sortSegments, TezCounter readsCounter, TezCounter writesCounter, TezCounter bytesReadCounter, Progress mergePhase) throws IOException, InterruptedException { return new MergeQueue(conf, fs, segments, comparator, reporter, sortSegments, codec, false).merge(keyClass, valueClass, mergeFactor, inMemSegments, tmpDir, readsCounter, writesCounter, bytesReadCounter, mergePhase); }
Example #20
Source File: SlowNodeAnalyzer.java From tez with Apache License 2.0 | 6 votes |
private float getAvgCounter(Collection<TaskAttemptInfo> taskAttemptInfos, String counterGroupName, String counterName) { long total = 0; int taskCount = 0; for (TaskAttemptInfo attemptInfo : taskAttemptInfos) { TezCounters tezCounters = attemptInfo.getTezCounters(); TezCounter counter = tezCounters.findCounter(counterGroupName, counterName); if (counter != null) { total += counter.getValue(); taskCount++; } else { LOG.info("Could not find counterGroupName=" + counterGroupName + ", counter=" + counterName + " in " + attemptInfo); } } return (taskCount > 0) ? (total * 1.0f / taskCount) : 0; }
Example #21
Source File: TestValuesIterator.java From tez with Apache License 2.0 | 5 votes |
private void verifyCountedIteratorReader(boolean inMemory) throws IOException, InterruptedException { TezCounter keyCounter = new GenericCounter("inputKeyCounter", "y3"); TezCounter tupleCounter = new GenericCounter("inputValuesCounter", "y4"); ValuesIterator iterator = createCountedIterator(inMemory, keyCounter, tupleCounter); List<Integer> sequence = verifyIteratorData(iterator); if (expectedTestResult) { assertEquals((long) sequence.size(), keyCounter.getValue()); long rows = 0; for (Integer i : sequence) { rows += i.longValue(); } assertEquals(rows, tupleCounter.getValue()); } }
Example #22
Source File: TezMerger.java From tez with Apache License 2.0 | 5 votes |
public DiskSegment(FileSystem fs, Path file, CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLenth, int bufferSize, boolean preserve, TezCounter mergedMapOutputsCounter) throws IOException { this(fs, file, 0, fs.getFileStatus(file).getLen(), codec, ifileReadAhead, ifileReadAheadLenth, bufferSize, preserve, mergedMapOutputsCounter); }
Example #23
Source File: TezCountersDelegate.java From incubator-tez with Apache License 2.0 | 5 votes |
@Override public TezCounter findCounter(String groupName, String counterName) { if (groupName.equals(TaskCounter.class.getName())) { groupName = TaskCounter.class.getSimpleName(); } String modifiedGroupName = groupName + "_" + this.groupModifier; return original.findCounter(modifiedGroupName, counterName); }
Example #24
Source File: TezCountersDelegate.java From tez with Apache License 2.0 | 5 votes |
@Override public TezCounter findCounter(String groupName, String counterName) { String simpleGroupName; if (groupName.equals(TaskCounter.class.getName())) { simpleGroupName = TaskCounter.class.getSimpleName(); } else { simpleGroupName = groupName; } String modifiedGroupName = simpleGroupName + "_" + this.groupModifier; final TezCounter modifiedGroupCounter = original.findCounter(modifiedGroupName, counterName); final TezCounter originalGroupCounter = original.findCounter(groupName, counterName); return new CompositeCounter(modifiedGroupCounter, originalGroupCounter); }
Example #25
Source File: IFile.java From tez with Apache License 2.0 | 5 votes |
public Writer(Configuration conf, FileSystem fs, Path file, Class keyClass, Class valueClass, CompressionCodec codec, TezCounter writesCounter, TezCounter serializedBytesCounter) throws IOException { this(conf, fs.create(file), keyClass, valueClass, codec, writesCounter, serializedBytesCounter); ownOutputStream = true; }
Example #26
Source File: TezTaskContext.java From spork with Apache License 2.0 | 5 votes |
@Override public boolean incrCounter(String group, String name, long delta) { if (context == null) { return false; } TezCounter counter = context.getCounters().getGroup(group).findCounter(name); counter.increment(delta); return true; }
Example #27
Source File: TestPipelinedSorter.java From tez with Apache License 2.0 | 5 votes |
private void verifyCounters(PipelinedSorter sorter, OutputContext context) { TezCounter numShuffleChunks = context.getCounters().findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT); TezCounter additionalSpills = context.getCounters().findCounter(TaskCounter.ADDITIONAL_SPILL_COUNT); TezCounter additionalSpillBytesWritten = context.getCounters().findCounter(TaskCounter.ADDITIONAL_SPILLS_BYTES_WRITTEN); TezCounter additionalSpillBytesRead = context.getCounters().findCounter(TaskCounter.ADDITIONAL_SPILLS_BYTES_READ); if (sorter.isFinalMergeEnabled()) { assertTrue(additionalSpills.getValue() == (sorter.getNumSpills() - 1)); //Number of files served by shuffle-handler assertTrue(1 == numShuffleChunks.getValue()); if (sorter.getNumSpills() > 1) { assertTrue(additionalSpillBytesRead.getValue() > 0); assertTrue(additionalSpillBytesWritten.getValue() > 0); } } else { assertTrue(0 == additionalSpills.getValue()); //Number of files served by shuffle-handler assertTrue(sorter.getNumSpills() == numShuffleChunks.getValue()); assertTrue(additionalSpillBytesRead.getValue() == 0); assertTrue(additionalSpillBytesWritten.getValue() == 0); } TezCounter finalOutputBytes = context.getCounters().findCounter(TaskCounter.OUTPUT_BYTES_PHYSICAL); assertTrue(finalOutputBytes.getValue() >= 0); TezCounter outputBytesWithOverheadCounter = context.getCounters().findCounter (TaskCounter.OUTPUT_BYTES_WITH_OVERHEAD); assertTrue(outputBytesWithOverheadCounter.getValue() >= 0); }
Example #28
Source File: MRReaderMapred.java From tez with Apache License 2.0 | 5 votes |
public MRReaderMapred(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters, TezCounter inputRecordCounter, InputContext context) throws IOException { super(context); this.jobConf = jobConf; this.tezCounters = tezCounters; this.inputRecordCounter = inputRecordCounter; inputFormat = this.jobConf.getInputFormat(); if (inputSplit != null) { this.inputSplit = inputSplit; setupOldRecordReader(); } }
Example #29
Source File: ShuffleTimeAnalyzer.java From tez with Apache License 2.0 | 5 votes |
private String getCounterValue(TaskCounter counter, String counterGroupName, TaskAttemptInfo attemptInfo) { Map<String, TezCounter> tezCounterMap = attemptInfo.getCounter(counter.toString()); if (tezCounterMap != null) { for (Map.Entry<String, TezCounter> entry : tezCounterMap.entrySet()) { String groupName = entry.getKey(); long val = entry.getValue().getValue(); if (groupName.equals(counterGroupName)) { return Long.toString(val); } } } return ""; }
Example #30
Source File: MRInputUtils.java From incubator-tez with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") public static InputSplit getOldSplitDetailsFromDisk(TaskSplitIndex splitMetaInfo, JobConf jobConf, TezCounter splitBytesCounter) throws IOException { Path file = new Path(splitMetaInfo.getSplitLocation()); FileSystem fs = FileSystem.getLocal(jobConf); file = fs.makeQualified(file); LOG.info("Reading input split file from : " + file); long offset = splitMetaInfo.getStartOffset(); FSDataInputStream inFile = fs.open(file); inFile.seek(offset); String className = Text.readString(inFile); Class<org.apache.hadoop.mapred.InputSplit> cls; try { cls = (Class<org.apache.hadoop.mapred.InputSplit>) jobConf.getClassByName(className); } catch (ClassNotFoundException ce) { IOException wrap = new IOException("Split class " + className + " not found"); wrap.initCause(ce); throw wrap; } SerializationFactory factory = new SerializationFactory(jobConf); Deserializer<org.apache.hadoop.mapred.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapred.InputSplit>) factory .getDeserializer(cls); deserializer.open(inFile); org.apache.hadoop.mapred.InputSplit split = deserializer.deserialize(null); long pos = inFile.getPos(); if (splitBytesCounter != null) { splitBytesCounter.increment(pos - offset); } inFile.close(); return split; }