Java Code Examples for org.apache.hadoop.io.DataInputBuffer#getPosition()
The following examples show how to use
org.apache.hadoop.io.DataInputBuffer#getPosition() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: IFile.java From hadoop with Apache License 2.0 | 6 votes |
public void append(DataInputBuffer key, DataInputBuffer value) throws IOException { int keyLength = key.getLength() - key.getPosition(); if (keyLength < 0) { throw new IOException("Negative key-length not allowed: " + keyLength + " for " + key); } int valueLength = value.getLength() - value.getPosition(); if (valueLength < 0) { throw new IOException("Negative value-length not allowed: " + valueLength + " for " + value); } WritableUtils.writeVInt(out, keyLength); WritableUtils.writeVInt(out, valueLength); out.write(key.getData(), key.getPosition(), keyLength); out.write(value.getData(), value.getPosition(), valueLength); // Update bytes written decompressedBytesWritten += keyLength + valueLength + WritableUtils.getVIntSize(keyLength) + WritableUtils.getVIntSize(valueLength); ++numRecordsWritten; }
Example 2
Source File: ReduceTask.java From hadoop-gpu with Apache License 2.0 | 6 votes |
public boolean next(DataInputBuffer key, DataInputBuffer value) throws IOException { if (kvIter.next()) { final DataInputBuffer kb = kvIter.getKey(); final DataInputBuffer vb = kvIter.getValue(); final int kp = kb.getPosition(); final int klen = kb.getLength() - kp; key.reset(kb.getData(), kp, klen); final int vp = vb.getPosition(); final int vlen = vb.getLength() - vp; value.reset(vb.getData(), vp, vlen); bytesRead += klen + vlen; return true; } return false; }
Example 3
Source File: InMemoryWriter.java From hadoop with Apache License 2.0 | 6 votes |
public void append(DataInputBuffer key, DataInputBuffer value) throws IOException { int keyLength = key.getLength() - key.getPosition(); if (keyLength < 0) { throw new IOException("Negative key-length not allowed: " + keyLength + " for " + key); } int valueLength = value.getLength() - value.getPosition(); if (valueLength < 0) { throw new IOException("Negative value-length not allowed: " + valueLength + " for " + value); } WritableUtils.writeVInt(out, keyLength); WritableUtils.writeVInt(out, valueLength); out.write(key.getData(), key.getPosition(), keyLength); out.write(value.getData(), value.getPosition(), valueLength); }
Example 4
Source File: IFile.java From tez with Apache License 2.0 | 6 votes |
/** * Send key/value to be appended to IFile. To represent same key as previous * one, send IFile.REPEAT_KEY as key parameter. Should not call this method with * IFile.REPEAT_KEY as the first key. It is caller's responsibility to pass non-negative * key/value lengths. Otherwise,IndexOutOfBoundsException could be thrown at runtime. * * * @param key * @param value * @throws IOException */ public void append(DataInputBuffer key, DataInputBuffer value) throws IOException { int keyLength = key.getLength() - key.getPosition(); assert(key == REPEAT_KEY || keyLength >=0); int valueLength = value.getLength() - value.getPosition(); assert(valueLength >= 0); sameKey = (key == REPEAT_KEY); if (!sameKey && rle) { sameKey = (keyLength != 0) && (BufferUtils.compare(previous, key) == 0); } if (!sameKey) { writeKVPair(key.getData(), key.getPosition(), keyLength, value.getData(), value.getPosition(), valueLength); if (rle) { BufferUtils.copy(key, previous); } } else { writeValue(value.getData(), value.getPosition(), valueLength); } prevKey = (sameKey) ? REPEAT_KEY : key; ++numRecordsWritten; }
Example 5
Source File: MergeManager.java From incubator-tez with Apache License 2.0 | 5 votes |
@Override public KeyState readRawKey(DataInputBuffer key) throws IOException { if (kvIter.next()) { final DataInputBuffer kb = kvIter.getKey(); final int kp = kb.getPosition(); final int klen = kb.getLength() - kp; key.reset(kb.getData(), kp, klen); bytesRead += klen; return KeyState.NEW_KEY; } return KeyState.NO_KEY; }
Example 6
Source File: Merger.java From hadoop-gpu with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") protected boolean lessThan(Object a, Object b) { DataInputBuffer key1 = ((Segment<K, V>)a).getKey(); DataInputBuffer key2 = ((Segment<K, V>)b).getKey(); int s1 = key1.getPosition(); int l1 = key1.getLength() - s1; int s2 = key2.getPosition(); int l2 = key2.getLength() - s2; return comparator.compare(key1.getData(), s1, l1, key2.getData(), s2, l2) < 0; }
Example 7
Source File: PipelinedSorter.java From incubator-tez with Apache License 2.0 | 5 votes |
public void reset(DataInputBuffer clone) { byte[] data = clone.getData(); int start = clone.getPosition(); int length = clone.getLength(); resize(length); System.arraycopy(data, start, buffer, 0, length); super.reset(buffer, 0, length); }
Example 8
Source File: BufferUtils.java From tez with Apache License 2.0 | 5 votes |
public static int compare(DataInputBuffer buf1, DataInputBuffer buf2) { byte[] b1 = buf1.getData(); byte[] b2 = buf2.getData(); int s1 = buf1.getPosition(); int s2 = buf2.getPosition(); int l1 = buf1.getLength(); int l2 = buf2.getLength(); return FastByteComparisons.compareTo(b1, s1, l1, b2, s2, l2); }
Example 9
Source File: MergeManagerImpl.java From big-c with Apache License 2.0 | 5 votes |
public boolean nextRawKey(DataInputBuffer key) throws IOException { if (kvIter.next()) { final DataInputBuffer kb = kvIter.getKey(); final int kp = kb.getPosition(); final int klen = kb.getLength() - kp; key.reset(kb.getData(), kp, klen); bytesRead += klen; return true; } return false; }
Example 10
Source File: IFile.java From incubator-tez with Apache License 2.0 | 5 votes |
/** * Send key/value to be appended to IFile. To represent same key as previous * one, send IFile.REPEAT_KEY as key parameter. Should not call this method with * IFile.REPEAT_KEY as the first key. * * @param key * @param value * @throws IOException */ public void append(DataInputBuffer key, DataInputBuffer value) throws IOException { int keyLength = key.getLength() - key.getPosition(); checkState((key == REPEAT_KEY || keyLength >= 0), "Negative key-length not allowed: %d for %s", keyLength, key); int valueLength = value.getLength() - value.getPosition(); checkState(valueLength >= 0, "Negative value-length not allowed: %d for %s", valueLength, value); boolean sameKey = (key == REPEAT_KEY); if (!sameKey && rle) { sameKey = (keyLength != 0) && (BufferUtils.compare(previous, key) == 0); } if (!sameKey) { writeKVPair(key.getData(), key.getPosition(), keyLength, value.getData(), value.getPosition(), valueLength); if (rle) { BufferUtils.copy(key, previous); } } else { writeValue(value.getData(), value.getPosition(), valueLength); } prevKey = (sameKey) ? REPEAT_KEY : key; ++numRecordsWritten; }
Example 11
Source File: MergeManager.java From tez with Apache License 2.0 | 5 votes |
public void nextRawValue(DataInputBuffer value) throws IOException { final DataInputBuffer vb = kvIter.getValue(); final int vp = vb.getPosition(); final int vlen = vb.getLength() - vp; value.reset(vb.getData(), vp, vlen); bytesRead += vlen; }
Example 12
Source File: BackupStore.java From big-c with Apache License 2.0 | 5 votes |
/** * Write the key and value to the cache in the IFile format * @param key * @param value * @throws IOException */ public void write(DataInputBuffer key, DataInputBuffer value) throws IOException { int keyLength = key.getLength() - key.getPosition(); int valueLength = value.getLength() - value.getPosition(); WritableUtils.writeVInt(dataOut, keyLength); WritableUtils.writeVInt(dataOut, valueLength); dataOut.write(key.getData(), key.getPosition(), keyLength); dataOut.write(value.getData(), value.getPosition(), valueLength); usedSize += keyLength + valueLength + WritableUtils.getVIntSize(keyLength) + WritableUtils.getVIntSize(valueLength); LOG.debug("ID: " + segmentList.size() + " WRITE TO MEM"); }
Example 13
Source File: BackupStore.java From big-c with Apache License 2.0 | 5 votes |
boolean reserveSpace(DataInputBuffer key, DataInputBuffer value) throws IOException { int keyLength = key.getLength() - key.getPosition(); int valueLength = value.getLength() - value.getPosition(); int requestedSize = keyLength + valueLength + WritableUtils.getVIntSize(keyLength) + WritableUtils.getVIntSize(valueLength); return reserveSpace(requestedSize); }
Example 14
Source File: BufferUtils.java From tez with Apache License 2.0 | 5 votes |
public static void copy(DataInputBuffer src, DataOutputBuffer dst) throws IOException { byte[] b1 = src.getData(); int s1 = src.getPosition(); int l1 = src.getLength(); dst.reset(); dst.write(b1, s1, l1 - s1); }
Example 15
Source File: MergeManagerImpl.java From hadoop with Apache License 2.0 | 5 votes |
public void nextRawValue(DataInputBuffer value) throws IOException { final DataInputBuffer vb = kvIter.getValue(); final int vp = vb.getPosition(); final int vlen = vb.getLength() - vp; value.reset(vb.getData(), vp, vlen); bytesRead += vlen; }
Example 16
Source File: MergeManagerImpl.java From hadoop with Apache License 2.0 | 5 votes |
public boolean nextRawKey(DataInputBuffer key) throws IOException { if (kvIter.next()) { final DataInputBuffer kb = kvIter.getKey(); final int kp = kb.getPosition(); final int klen = kb.getLength() - kp; key.reset(kb.getData(), kp, klen); bytesRead += klen; return true; } return false; }
Example 17
Source File: MergeManager.java From tez with Apache License 2.0 | 5 votes |
@Override public KeyState readRawKey(DataInputBuffer key) throws IOException { if (kvIter.next()) { final DataInputBuffer kb = kvIter.getKey(); final int kp = kb.getPosition(); final int klen = kb.getLength() - kp; key.reset(kb.getData(), kp, klen); bytesRead += klen; return KeyState.NEW_KEY; } return KeyState.NO_KEY; }
Example 18
Source File: TezMerger.java From incubator-tez with Apache License 2.0 | 5 votes |
protected boolean lessThan(Object a, Object b) { DataInputBuffer key1 = ((Segment)a).getKey(); DataInputBuffer key2 = ((Segment)b).getKey(); int s1 = key1.getPosition(); int l1 = key1.getLength() - s1; int s2 = key2.getPosition(); int l2 = key2.getLength() - s2; return comparator.compare(key1.getData(), s1, l1, key2.getData(), s2, l2) < 0; }
Example 19
Source File: ReduceContextImpl.java From hadoop with Apache License 2.0 | 4 votes |
/** * Advance to the next key/value pair. */ @Override public boolean nextKeyValue() throws IOException, InterruptedException { if (!hasMore) { key = null; value = null; return false; } firstValue = !nextKeyIsSame; DataInputBuffer nextKey = input.getKey(); currentRawKey.set(nextKey.getData(), nextKey.getPosition(), nextKey.getLength() - nextKey.getPosition()); buffer.reset(currentRawKey.getBytes(), 0, currentRawKey.getLength()); key = keyDeserializer.deserialize(key); DataInputBuffer nextVal = input.getValue(); buffer.reset(nextVal.getData(), nextVal.getPosition(), nextVal.getLength() - nextVal.getPosition()); value = valueDeserializer.deserialize(value); currentKeyLength = nextKey.getLength() - nextKey.getPosition(); currentValueLength = nextVal.getLength() - nextVal.getPosition(); if (isMarked) { backupStore.write(nextKey, nextVal); } hasMore = input.next(); if (hasMore) { nextKey = input.getKey(); nextKeyIsSame = comparator.compare(currentRawKey.getBytes(), 0, currentRawKey.getLength(), nextKey.getData(), nextKey.getPosition(), nextKey.getLength() - nextKey.getPosition() ) == 0; } else { nextKeyIsSame = false; } inputValueCounter.increment(1); return true; }
Example 20
Source File: PipelinedSorter.java From tez with Apache License 2.0 | 4 votes |
public void reset(DataInputBuffer clone) { byte[] data = clone.getData(); int start = clone.getPosition(); int length = clone.getLength() - start; super.reset(data, start, length); }