Java Code Examples for org.apache.hadoop.hbase.KeyValueUtil#keyLength()

The following examples show how to use org.apache.hadoop.hbase.KeyValueUtil#keyLength() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: HFilePrettyPrinter.java From hbase with Apache License 2.0

5 votes

public void collect(Cell cell) {
  valLen.update(cell.getValueLength());
  if (prevCell != null &&
      CellComparator.getInstance().compareRows(prevCell, cell) != 0) {
    // new row
    collectRow();
  }
  curRowBytes += cell.getSerializedSize();
  curRowKeyLength = KeyValueUtil.keyLength(cell);
  curRowCols++;
  prevCell = cell;
}

Example 2

Source File: PrefixKeyDeltaEncoder.java From hbase with Apache License 2.0

5 votes

@Override
public int internalEncode(Cell cell, HFileBlockDefaultEncodingContext encodingContext,
    DataOutputStream out) throws IOException {
  int klength = KeyValueUtil.keyLength(cell);
  int vlength = cell.getValueLength();
  EncodingState state = encodingContext.getEncodingState();
  if (state.prevCell == null) {
    // copy the key, there is no common prefix with none
    ByteBufferUtils.putCompressedInt(out, klength);
    ByteBufferUtils.putCompressedInt(out, vlength);
    ByteBufferUtils.putCompressedInt(out, 0);
    PrivateCellUtil.writeFlatKey(cell, (DataOutput)out);
  } else {
    // find a common prefix and skip it
    int common = PrivateCellUtil.findCommonPrefixInFlatKey(cell, state.prevCell, true, true);
    ByteBufferUtils.putCompressedInt(out, klength - common);
    ByteBufferUtils.putCompressedInt(out, vlength);
    ByteBufferUtils.putCompressedInt(out, common);
    writeKeyExcludingCommon(cell, common, out);
  }
  // Write the value part
  PrivateCellUtil.writeValue(out, cell, vlength);
  int size = klength + vlength + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE;
  size += afterEncodingKeyValue(cell, out, encodingContext);
  state.prevCell = cell;
  return size;
}

Example 3

Source File: KeyOnlyFilter.java From hbase with Apache License 2.0

4 votes

public KeyOnlyCell(Cell c, boolean lenAsVal) {
  this.cell = c;
  this.lenAsVal = lenAsVal;
  this.keyLen = KeyValueUtil.keyLength(c);
}

Example 4

Source File: TestKeyOnlyFilter.java From hbase with Apache License 2.0

4 votes

@Test
public void testKeyOnly() throws Exception {
  byte[] r = Bytes.toBytes("row1");
  byte[] f = Bytes.toBytes("cf1");
  byte[] q = Bytes.toBytes("qual1");
  byte[] v = Bytes.toBytes("val1");
  byte[] tags = Bytes.toBytes("tag1");
  KeyValue kv = new KeyValue(r, f, q, 0, q.length, 1234L, Type.Put, v, 0,
      v.length, tags);

  ByteBuffer buffer = ByteBuffer.wrap(kv.getBuffer());
  ByteBufferKeyValue bbCell = new ByteBufferKeyValue(buffer, 0,
      buffer.remaining());

  // KV format: <keylen:4><valuelen:4><key:keylen><value:valuelen>
  // Rebuild as: <keylen:4><0:4><key:keylen>
  int dataLen = lenAsVal ? Bytes.SIZEOF_INT : 0;
  int keyOffset = (2 * Bytes.SIZEOF_INT);
  int keyLen = KeyValueUtil.keyLength(kv);
  byte[] newBuffer = new byte[keyLen + keyOffset + dataLen];
  Bytes.putInt(newBuffer, 0, keyLen);
  Bytes.putInt(newBuffer, Bytes.SIZEOF_INT, dataLen);
  KeyValueUtil.appendKeyTo(kv, newBuffer, keyOffset);
  if (lenAsVal) {
    Bytes.putInt(newBuffer, newBuffer.length - dataLen, kv.getValueLength());
  }
  KeyValue KeyOnlyKeyValue = new KeyValue(newBuffer);

  KeyOnlyCell keyOnlyCell = new KeyOnlyCell(kv, lenAsVal);
  KeyOnlyByteBufferExtendedCell keyOnlyByteBufferedCell = new KeyOnlyByteBufferExtendedCell(
      bbCell, lenAsVal);

  assertTrue(CellUtil.matchingRows(KeyOnlyKeyValue, keyOnlyCell));
  assertTrue(CellUtil.matchingRows(KeyOnlyKeyValue, keyOnlyByteBufferedCell));

  assertTrue(CellUtil.matchingFamily(KeyOnlyKeyValue, keyOnlyCell));
  assertTrue(CellUtil
      .matchingFamily(KeyOnlyKeyValue, keyOnlyByteBufferedCell));

  assertTrue(CellUtil.matchingQualifier(KeyOnlyKeyValue, keyOnlyCell));
  assertTrue(CellUtil.matchingQualifier(KeyOnlyKeyValue,
      keyOnlyByteBufferedCell));

  assertTrue(CellUtil.matchingValue(KeyOnlyKeyValue, keyOnlyCell));
  assertTrue(KeyOnlyKeyValue.getValueLength() == keyOnlyByteBufferedCell
      .getValueLength());
  assertEquals(8 + keyLen + (lenAsVal ? 4 : 0), KeyOnlyKeyValue.getSerializedSize());
  assertEquals(8 + keyLen + (lenAsVal ? 4 : 0), keyOnlyCell.getSerializedSize());
  if (keyOnlyByteBufferedCell.getValueLength() > 0) {
    assertTrue(CellUtil.matchingValue(KeyOnlyKeyValue,
        keyOnlyByteBufferedCell));
  }

  assertTrue(KeyOnlyKeyValue.getTimestamp() == keyOnlyCell.getTimestamp());
  assertTrue(KeyOnlyKeyValue.getTimestamp() == keyOnlyByteBufferedCell
      .getTimestamp());

  assertTrue(KeyOnlyKeyValue.getTypeByte() == keyOnlyCell.getTypeByte());
  assertTrue(KeyOnlyKeyValue.getTypeByte() == keyOnlyByteBufferedCell
      .getTypeByte());

  assertTrue(KeyOnlyKeyValue.getTagsLength() == keyOnlyCell.getTagsLength());
  assertTrue(KeyOnlyKeyValue.getTagsLength() == keyOnlyByteBufferedCell
      .getTagsLength());
}

Example 5

Source File: DiffKeyDeltaEncoder.java From hbase with Apache License 2.0

4 votes

private int compressSingleKeyValue(DataOutputStream out, Cell cell, Cell prevCell)
    throws IOException {
  int flag = 0; // Do not use more bits that can fit into a byte
  int kLength = KeyValueUtil.keyLength(cell);
  int vLength = cell.getValueLength();

  long timestamp;
  long diffTimestamp = 0;
  int diffTimestampFitsInBytes = 0;
  int timestampFitsInBytes;
  int commonPrefix = 0;

  if (prevCell == null) {
    timestamp = cell.getTimestamp();
    if (timestamp < 0) {
      flag |= FLAG_TIMESTAMP_SIGN;
      timestamp = -timestamp;
    }
    timestampFitsInBytes = ByteBufferUtils.longFitsIn(timestamp);
    flag |= (timestampFitsInBytes - 1) << SHIFT_TIMESTAMP_LENGTH;
    // put column family
    byte familyLength = cell.getFamilyLength();
    out.write(familyLength);
    PrivateCellUtil.writeFamily(out, cell, familyLength);
  } else {
    // Finding common prefix
    int preKeyLength = KeyValueUtil.keyLength(prevCell);
    commonPrefix = PrivateCellUtil.findCommonPrefixInFlatKey(cell, prevCell, true, false);
    if (kLength == preKeyLength) {
      flag |= FLAG_SAME_KEY_LENGTH;
    }
    if (vLength == prevCell.getValueLength()) {
      flag |= FLAG_SAME_VALUE_LENGTH;
    }
    if (cell.getTypeByte() == prevCell.getTypeByte()) {
      flag |= FLAG_SAME_TYPE;
    }
    // don't compress timestamp and type using prefix encode timestamp
    timestamp = cell.getTimestamp();
    diffTimestamp = prevCell.getTimestamp() - timestamp;
    boolean negativeTimestamp = timestamp < 0;
    if (negativeTimestamp) {
      timestamp = -timestamp;
    }
    timestampFitsInBytes = ByteBufferUtils.longFitsIn(timestamp);
    boolean minusDiffTimestamp = diffTimestamp < 0;
    if (minusDiffTimestamp) {
      diffTimestamp = -diffTimestamp;
    }
    diffTimestampFitsInBytes = ByteBufferUtils.longFitsIn(diffTimestamp);
    if (diffTimestampFitsInBytes < timestampFitsInBytes) {
      flag |= (diffTimestampFitsInBytes - 1) << SHIFT_TIMESTAMP_LENGTH;
      flag |= FLAG_TIMESTAMP_IS_DIFF;
      if (minusDiffTimestamp) {
        flag |= FLAG_TIMESTAMP_SIGN;
      }
    } else {
      flag |= (timestampFitsInBytes - 1) << SHIFT_TIMESTAMP_LENGTH;
      if (negativeTimestamp) {
        flag |= FLAG_TIMESTAMP_SIGN;
      }
    }
  }
  out.write(flag);
  if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
    ByteBufferUtils.putCompressedInt(out, kLength);
  }
  if ((flag & FLAG_SAME_VALUE_LENGTH) == 0) {
    ByteBufferUtils.putCompressedInt(out, vLength);
  }
  ByteBufferUtils.putCompressedInt(out, commonPrefix);
  short rLen = cell.getRowLength();
  if (commonPrefix < rLen + KeyValue.ROW_LENGTH_SIZE) {
    // Previous and current rows are different. Copy the differing part of
    // the row, skip the column family, and copy the qualifier.
    PrivateCellUtil.writeRowKeyExcludingCommon(cell, rLen, commonPrefix, out);
    PrivateCellUtil.writeQualifier(out, cell, cell.getQualifierLength());
  } else {
    // The common part includes the whole row. As the column family is the
    // same across the whole file, it will automatically be included in the
    // common prefix, so we need not special-case it here.
    // What we write here is the non common part of the qualifier
    int commonQualPrefix = commonPrefix - (rLen + KeyValue.ROW_LENGTH_SIZE)
        - (cell.getFamilyLength() + KeyValue.FAMILY_LENGTH_SIZE);
    PrivateCellUtil.writeQualifierSkippingBytes(out, cell, cell.getQualifierLength(),
      commonQualPrefix);
  }
  if ((flag & FLAG_TIMESTAMP_IS_DIFF) == 0) {
    ByteBufferUtils.putLong(out, timestamp, timestampFitsInBytes);
  } else {
    ByteBufferUtils.putLong(out, diffTimestamp, diffTimestampFitsInBytes);
  }

  if ((flag & FLAG_SAME_TYPE) == 0) {
    out.write(cell.getTypeByte());
  }
  PrivateCellUtil.writeValue(out, cell, vLength);
  return kLength + vLength + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE;
}

Example 6

Source File: FastDiffDeltaEncoder.java From hbase with Apache License 2.0

4 votes

private int compressSingleKeyValue(DataOutputStream out, Cell cell, Cell prevCell)
    throws IOException {
  int flag = 0; // Do not use more bits than will fit into a byte
  int kLength = KeyValueUtil.keyLength(cell);
  int vLength = cell.getValueLength();

  if (prevCell == null) {
    // copy the key, there is no common prefix with none
    out.write(flag);
    ByteBufferUtils.putCompressedInt(out, kLength);
    ByteBufferUtils.putCompressedInt(out, vLength);
    ByteBufferUtils.putCompressedInt(out, 0);
    PrivateCellUtil.writeFlatKey(cell, (DataOutput)out);
    // Write the value part
    PrivateCellUtil.writeValue(out, cell, cell.getValueLength());
  } else {
    int preKeyLength = KeyValueUtil.keyLength(prevCell);
    int preValLength = prevCell.getValueLength();
    // find a common prefix and skip it
    int commonPrefix = PrivateCellUtil.findCommonPrefixInFlatKey(cell, prevCell, true, false);

    if (kLength == preKeyLength) {
      flag |= FLAG_SAME_KEY_LENGTH;
    }
    if (vLength == prevCell.getValueLength()) {
      flag |= FLAG_SAME_VALUE_LENGTH;
    }
    if (cell.getTypeByte() == prevCell.getTypeByte()) {
      flag |= FLAG_SAME_TYPE;
    }

    byte[] curTsBuf = Bytes.toBytes(cell.getTimestamp());
    int commonTimestampPrefix = findCommonTimestampPrefix(curTsBuf,
        Bytes.toBytes(prevCell.getTimestamp()));

    flag |= commonTimestampPrefix << SHIFT_TIMESTAMP_LENGTH;

    // Check if current and previous values are the same. Compare value
    // length first as an optimization.
    if (vLength == preValLength
        && PrivateCellUtil.matchingValue(cell, prevCell, vLength, preValLength)) {
      flag |= FLAG_SAME_VALUE;
    }

    out.write(flag);
    if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
      ByteBufferUtils.putCompressedInt(out, kLength);
    }
    if ((flag & FLAG_SAME_VALUE_LENGTH) == 0) {
      ByteBufferUtils.putCompressedInt(out, vLength);
    }
    ByteBufferUtils.putCompressedInt(out, commonPrefix);
    short rLen = cell.getRowLength();
    if (commonPrefix < rLen + KeyValue.ROW_LENGTH_SIZE) {
      // Previous and current rows are different. Copy the differing part of
      // the row, skip the column family, and copy the qualifier.
      PrivateCellUtil.writeRowKeyExcludingCommon(cell, rLen, commonPrefix, out);
      PrivateCellUtil.writeQualifier(out, cell, cell.getQualifierLength());
    } else {
      // The common part includes the whole row. As the column family is the
      // same across the whole file, it will automatically be included in the
      // common prefix, so we need not special-case it here.
      // What we write here is the non common part of the qualifier
      int commonQualPrefix = commonPrefix - (rLen + KeyValue.ROW_LENGTH_SIZE)
          - (cell.getFamilyLength() + KeyValue.FAMILY_LENGTH_SIZE);
      PrivateCellUtil.writeQualifierSkippingBytes(out, cell, cell.getQualifierLength(),
        commonQualPrefix);
    }
    // Write non common ts part
    out.write(curTsBuf, commonTimestampPrefix, KeyValue.TIMESTAMP_SIZE - commonTimestampPrefix);

    // Write the type if it is not the same as before.
    if ((flag & FLAG_SAME_TYPE) == 0) {
      out.write(cell.getTypeByte());
    }

    // Write the value if it is not the same as before.
    if ((flag & FLAG_SAME_VALUE) == 0) {
      PrivateCellUtil.writeValue(out, cell, vLength);
    }
  }
  return kLength + vLength + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE;
}