Java Code Examples for org.apache.hadoop.io.DataInputBuffer#getPosition()

The following examples show how to use org.apache.hadoop.io.DataInputBuffer#getPosition() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: IFile.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public void append(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  if (keyLength < 0) {
    throw new IOException("Negative key-length not allowed: " + keyLength + 
                          " for " + key);
  }
  
  int valueLength = value.getLength() - value.getPosition();
  if (valueLength < 0) {
    throw new IOException("Negative value-length not allowed: " + 
                          valueLength + " for " + value);
  }

  WritableUtils.writeVInt(out, keyLength);
  WritableUtils.writeVInt(out, valueLength);
  out.write(key.getData(), key.getPosition(), keyLength); 
  out.write(value.getData(), value.getPosition(), valueLength); 

  // Update bytes written
  decompressedBytesWritten += keyLength + valueLength + 
                  WritableUtils.getVIntSize(keyLength) + 
                  WritableUtils.getVIntSize(valueLength);
  ++numRecordsWritten;
}
Example 2
Source File: ReduceTask.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up 
public boolean next(DataInputBuffer key, DataInputBuffer value)
    throws IOException {
  if (kvIter.next()) {
    final DataInputBuffer kb = kvIter.getKey();
    final DataInputBuffer vb = kvIter.getValue();
    final int kp = kb.getPosition();
    final int klen = kb.getLength() - kp;
    key.reset(kb.getData(), kp, klen);
    final int vp = vb.getPosition();
    final int vlen = vb.getLength() - vp;
    value.reset(vb.getData(), vp, vlen);
    bytesRead += klen + vlen;
    return true;
  }
  return false;
}
Example 3
Source File: InMemoryWriter.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public void append(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  if (keyLength < 0) {
    throw new IOException("Negative key-length not allowed: " + keyLength + 
                          " for " + key);
  }
  
  int valueLength = value.getLength() - value.getPosition();
  if (valueLength < 0) {
    throw new IOException("Negative value-length not allowed: " + 
                          valueLength + " for " + value);
  }

  WritableUtils.writeVInt(out, keyLength);
  WritableUtils.writeVInt(out, valueLength);
  out.write(key.getData(), key.getPosition(), keyLength); 
  out.write(value.getData(), value.getPosition(), valueLength); 
}
Example 4
Source File: IFile.java    From tez with Apache License 2.0 6 votes vote down vote up 
/**
 * Send key/value to be appended to IFile. To represent same key as previous
 * one, send IFile.REPEAT_KEY as key parameter.  Should not call this method with
 * IFile.REPEAT_KEY as the first key. It is caller's responsibility to pass non-negative
 * key/value lengths. Otherwise,IndexOutOfBoundsException could be thrown at runtime.
 *
 *
 * @param key
 * @param value
 * @throws IOException
 */
public void append(DataInputBuffer key, DataInputBuffer value) throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  assert(key == REPEAT_KEY || keyLength >=0);

  int valueLength = value.getLength() - value.getPosition();
  assert(valueLength >= 0);

  sameKey = (key == REPEAT_KEY);
  if (!sameKey && rle) {
    sameKey = (keyLength != 0) && (BufferUtils.compare(previous, key) == 0);
  }

  if (!sameKey) {
    writeKVPair(key.getData(), key.getPosition(), keyLength,
        value.getData(), value.getPosition(), valueLength);
    if (rle) {
      BufferUtils.copy(key, previous);
    }
  } else {
    writeValue(value.getData(), value.getPosition(), valueLength);
  }
  prevKey = (sameKey) ? REPEAT_KEY : key;
  ++numRecordsWritten;
}
Example 5
Source File: MergeManager.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
@Override
public KeyState readRawKey(DataInputBuffer key) throws IOException {
  if (kvIter.next()) {
    final DataInputBuffer kb = kvIter.getKey();
    final int kp = kb.getPosition();
    final int klen = kb.getLength() - kp;
    key.reset(kb.getData(), kp, klen);
    bytesRead += klen;
    return KeyState.NEW_KEY;
  }
  return KeyState.NO_KEY;
}
Example 6
Source File: Merger.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
protected boolean lessThan(Object a, Object b) {
  DataInputBuffer key1 = ((Segment<K, V>)a).getKey();
  DataInputBuffer key2 = ((Segment<K, V>)b).getKey();
  int s1 = key1.getPosition();
  int l1 = key1.getLength() - s1;
  int s2 = key2.getPosition();
  int l2 = key2.getLength() - s2;

  return comparator.compare(key1.getData(), s1, l1, key2.getData(), s2, l2) < 0;
}
Example 7
Source File: PipelinedSorter.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
public void reset(DataInputBuffer clone) {
  byte[] data = clone.getData();
  int start = clone.getPosition();
  int length = clone.getLength();
  resize(length);
  System.arraycopy(data, start, buffer, 0, length);
  super.reset(buffer, 0, length);
}
Example 8
Source File: BufferUtils.java    From tez with Apache License 2.0 5 votes vote down vote up 
public static int compare(DataInputBuffer buf1, DataInputBuffer buf2) {
  byte[] b1 = buf1.getData();
  byte[] b2 = buf2.getData();
  int s1 = buf1.getPosition();
  int s2 = buf2.getPosition();
  int l1 = buf1.getLength();
  int l2 = buf2.getLength();
  return FastByteComparisons.compareTo(b1, s1, l1, b2, s2, l2);
}
Example 9
Source File: MergeManagerImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public boolean nextRawKey(DataInputBuffer key) throws IOException {
  if (kvIter.next()) {
    final DataInputBuffer kb = kvIter.getKey();
    final int kp = kb.getPosition();
    final int klen = kb.getLength() - kp;
    key.reset(kb.getData(), kp, klen);
    bytesRead += klen;
    return true;
  }
  return false;
}
Example 10
Source File: IFile.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
/**
 * Send key/value to be appended to IFile. To represent same key as previous
 * one, send IFile.REPEAT_KEY as key parameter.  Should not call this method with
 * IFile.REPEAT_KEY as the first key.
 *
 * @param key
 * @param value
 * @throws IOException
 */
public void append(DataInputBuffer key, DataInputBuffer value) throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  checkState((key == REPEAT_KEY || keyLength >= 0),
    "Negative key-length not allowed: %d for %s", keyLength, key);

  int valueLength = value.getLength() - value.getPosition();
  checkState(valueLength >= 0,
    "Negative value-length not allowed: %d for %s", valueLength, value);

  boolean sameKey = (key == REPEAT_KEY);
  if (!sameKey && rle) {
    sameKey = (keyLength != 0) && (BufferUtils.compare(previous, key) == 0);
  }

  if (!sameKey) {
    writeKVPair(key.getData(), key.getPosition(), keyLength,
        value.getData(), value.getPosition(), valueLength);
    if (rle) {
      BufferUtils.copy(key, previous);
    }
  } else {
    writeValue(value.getData(), value.getPosition(), valueLength);
  }
  prevKey = (sameKey) ? REPEAT_KEY : key;
  ++numRecordsWritten;
}
Example 11
Source File: MergeManager.java    From tez with Apache License 2.0 5 votes vote down vote up 
public void nextRawValue(DataInputBuffer value) throws IOException {
  final DataInputBuffer vb = kvIter.getValue();
  final int vp = vb.getPosition();
  final int vlen = vb.getLength() - vp;
  value.reset(vb.getData(), vp, vlen);
  bytesRead += vlen;
}
Example 12
Source File: BackupStore.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Write the key and value to the cache in the IFile format
 * @param key
 * @param value
 * @throws IOException
 */
public void write(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  int valueLength = value.getLength() - value.getPosition();
  WritableUtils.writeVInt(dataOut, keyLength);
  WritableUtils.writeVInt(dataOut, valueLength);
  dataOut.write(key.getData(), key.getPosition(), keyLength);
  dataOut.write(value.getData(), value.getPosition(), valueLength);
  usedSize += keyLength + valueLength + 
    WritableUtils.getVIntSize(keyLength) +
    WritableUtils.getVIntSize(valueLength);
  LOG.debug("ID: " + segmentList.size() + " WRITE TO MEM");
}
Example 13
Source File: BackupStore.java    From big-c with Apache License 2.0 5 votes vote down vote up 
boolean reserveSpace(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  int valueLength = value.getLength() - value.getPosition();

  int requestedSize = keyLength + valueLength + 
    WritableUtils.getVIntSize(keyLength) +
    WritableUtils.getVIntSize(valueLength);
  return reserveSpace(requestedSize);
}
Example 14
Source File: BufferUtils.java    From tez with Apache License 2.0 5 votes vote down vote up 
public static void copy(DataInputBuffer src, DataOutputBuffer dst) throws IOException {
  byte[] b1 = src.getData();
  int s1 = src.getPosition();
  int l1 = src.getLength();
  dst.reset();
  dst.write(b1, s1, l1 - s1);
}
Example 15
Source File: MergeManagerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public void nextRawValue(DataInputBuffer value) throws IOException {
  final DataInputBuffer vb = kvIter.getValue();
  final int vp = vb.getPosition();
  final int vlen = vb.getLength() - vp;
  value.reset(vb.getData(), vp, vlen);
  bytesRead += vlen;
}
Example 16
Source File: MergeManagerImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public boolean nextRawKey(DataInputBuffer key) throws IOException {
  if (kvIter.next()) {
    final DataInputBuffer kb = kvIter.getKey();
    final int kp = kb.getPosition();
    final int klen = kb.getLength() - kp;
    key.reset(kb.getData(), kp, klen);
    bytesRead += klen;
    return true;
  }
  return false;
}
Example 17
Source File: MergeManager.java    From tez with Apache License 2.0 5 votes vote down vote up 
@Override
public KeyState readRawKey(DataInputBuffer key) throws IOException {
  if (kvIter.next()) {
    final DataInputBuffer kb = kvIter.getKey();
    final int kp = kb.getPosition();
    final int klen = kb.getLength() - kp;
    key.reset(kb.getData(), kp, klen);
    bytesRead += klen;
    return KeyState.NEW_KEY;
  }
  return KeyState.NO_KEY;
}
Example 18
Source File: TezMerger.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up 
protected boolean lessThan(Object a, Object b) {
  DataInputBuffer key1 = ((Segment)a).getKey();
  DataInputBuffer key2 = ((Segment)b).getKey();
  int s1 = key1.getPosition();
  int l1 = key1.getLength() - s1;
  int s2 = key2.getPosition();
  int l2 = key2.getLength() - s2;

  return comparator.compare(key1.getData(), s1, l1, key2.getData(), s2, l2) < 0;
}
Example 19
Source File: ReduceContextImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Advance to the next key/value pair.
 */
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
  if (!hasMore) {
    key = null;
    value = null;
    return false;
  }
  firstValue = !nextKeyIsSame;
  DataInputBuffer nextKey = input.getKey();
  currentRawKey.set(nextKey.getData(), nextKey.getPosition(), 
                    nextKey.getLength() - nextKey.getPosition());
  buffer.reset(currentRawKey.getBytes(), 0, currentRawKey.getLength());
  key = keyDeserializer.deserialize(key);
  DataInputBuffer nextVal = input.getValue();
  buffer.reset(nextVal.getData(), nextVal.getPosition(), nextVal.getLength()
      - nextVal.getPosition());
  value = valueDeserializer.deserialize(value);

  currentKeyLength = nextKey.getLength() - nextKey.getPosition();
  currentValueLength = nextVal.getLength() - nextVal.getPosition();

  if (isMarked) {
    backupStore.write(nextKey, nextVal);
  }

  hasMore = input.next();
  if (hasMore) {
    nextKey = input.getKey();
    nextKeyIsSame = comparator.compare(currentRawKey.getBytes(), 0, 
                                   currentRawKey.getLength(),
                                   nextKey.getData(),
                                   nextKey.getPosition(),
                                   nextKey.getLength() - nextKey.getPosition()
                                       ) == 0;
  } else {
    nextKeyIsSame = false;
  }
  inputValueCounter.increment(1);
  return true;
}
Example 20
Source File: PipelinedSorter.java    From tez with Apache License 2.0 4 votes vote down vote up 
public void reset(DataInputBuffer clone) {
  byte[] data = clone.getData();
  int start = clone.getPosition();
  int length = clone.getLength() - start;
  super.reset(data, start, length);
}