Java Code Examples for org.apache.hadoop.hbase.KeyValue#getTagsLength()

The following examples show how to use org.apache.hadoop.hbase.KeyValue#getTagsLength() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestTagCompressionContext.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testCompressUncompressTags1() throws Exception {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE);
  KeyValue kv1 = createKVWithTags(2);
  int tagsLength1 = kv1.getTagsLength();
  ByteBuffer ib = ByteBuffer.wrap(kv1.getTagsArray());
  context.compressTags(baos, ib, kv1.getTagsOffset(), tagsLength1);
  KeyValue kv2 = createKVWithTags(3);
  int tagsLength2 = kv2.getTagsLength();
  ib = ByteBuffer.wrap(kv2.getTagsArray());
  context.compressTags(baos, ib, kv2.getTagsOffset(), tagsLength2);

  context.clear();

  byte[] dest = new byte[tagsLength1];
  ByteBuffer ob = ByteBuffer.wrap(baos.toByteArray());
  context.uncompressTags(new SingleByteBuff(ob), dest, 0, tagsLength1);
  assertTrue(Bytes.equals(kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1, dest, 0,
      tagsLength1));
  dest = new byte[tagsLength2];
  context.uncompressTags(new SingleByteBuff(ob), dest, 0, tagsLength2);
  assertTrue(Bytes.equals(kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2, dest, 0,
      tagsLength2));
}
 
Example 2
Source File: TestTagCompressionContext.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testCompressUncompressTags2() throws Exception {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  TagCompressionContext context = new TagCompressionContext(LRUDictionary.class, Byte.MAX_VALUE);
  KeyValue kv1 = createKVWithTags(1);
  int tagsLength1 = kv1.getTagsLength();
  context.compressTags(baos, kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1);
  KeyValue kv2 = createKVWithTags(3);
  int tagsLength2 = kv2.getTagsLength();
  context.compressTags(baos, kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2);

  context.clear();

  ByteArrayInputStream bais = new ByteArrayInputStream(baos.getBuffer());
  byte[] dest = new byte[tagsLength1];
  context.uncompressTags(bais, dest, 0, tagsLength1);
  assertTrue(Bytes.equals(kv1.getTagsArray(), kv1.getTagsOffset(), tagsLength1, dest, 0,
      tagsLength1));
  dest = new byte[tagsLength2];
  context.uncompressTags(bais, dest, 0, tagsLength2);
  assertTrue(Bytes.equals(kv2.getTagsArray(), kv2.getTagsOffset(), tagsLength2, dest, 0,
      tagsLength2));
}
 
Example 3
Source File: TestByteRangeWithKVSerialization.java    From hbase with Apache License 2.0 5 votes vote down vote up
static void writeCell(PositionedByteRange pbr, KeyValue kv) throws Exception {
  pbr.putInt(kv.getKeyLength());
  pbr.putInt(kv.getValueLength());
  pbr.put(kv.getBuffer(), kv.getKeyOffset(), kv.getKeyLength());
  pbr.put(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
  int tagsLen = kv.getTagsLength();
  pbr.put((byte) (tagsLen >> 8 & 0xff));
  pbr.put((byte) (tagsLen & 0xff));
  pbr.put(kv.getTagsArray(), kv.getTagsOffset(), tagsLen);
  pbr.putVLong(kv.getSequenceId());
}
 
Example 4
Source File: DataBlockEncodingTool.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Check statistics for given HFile for different data block encoders.
 * @param scanner Of file which will be compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @throws IOException thrown if scanner is invalid
 */
public void checkStatistics(final KeyValueScanner scanner, final int kvLimit)
    throws IOException {
  scanner.seek(KeyValue.LOWESTKEY);

  KeyValue currentKV;

  byte[] previousKey = null;
  byte[] currentKey;

  DataBlockEncoding[] encodings = DataBlockEncoding.values();

  ByteArrayOutputStream uncompressedOutputStream =
      new ByteArrayOutputStream();

  int j = 0;
  while ((currentKV = KeyValueUtil.ensureKeyValue(scanner.next())) != null && j < kvLimit) {
    // Iterates through key/value pairs
    j++;
    currentKey = currentKV.getKey();
    if (previousKey != null) {
      for (int i = 0; i < previousKey.length && i < currentKey.length &&
          previousKey[i] == currentKey[i]; ++i) {
        totalKeyRedundancyLength++;
      }
    }

    // Add tagsLen zero to cells don't include tags. Since the process of
    // scanner converts byte array to KV would abandon tagsLen part if tagsLen
    // is zero. But we still needs the tagsLen part to check if current cell
    // include tags. If USE_TAG is true, HFile contains cells with tags,
    // if the cell tagsLen equals 0, it means other cells may have tags.
    if (USE_TAG && currentKV.getTagsLength() == 0) {
      uncompressedOutputStream.write(currentKV.getBuffer(),
          currentKV.getOffset(), currentKV.getLength());
      // write tagsLen = 0.
      uncompressedOutputStream.write(Bytes.toBytes((short) 0));
    } else {
      uncompressedOutputStream.write(currentKV.getBuffer(),
          currentKV.getOffset(), currentKV.getLength());
    }

    if(includesMemstoreTS) {
      WritableUtils.writeVLong(
          new DataOutputStream(uncompressedOutputStream), currentKV.getSequenceId());
    }

    previousKey = currentKey;

    int kLen = currentKV.getKeyLength();
    int vLen = currentKV.getValueLength();
    int cfLen = currentKV.getFamilyLength(currentKV.getFamilyOffset());
    int restLen = currentKV.getLength() - kLen - vLen;

    totalKeyLength += kLen;
    totalValueLength += vLen;
    totalPrefixLength += restLen;
    totalCFLength += cfLen;
  }

  rawKVs = uncompressedOutputStream.toByteArray();
  for (DataBlockEncoding encoding : encodings) {
    if (encoding == DataBlockEncoding.NONE) {
      continue;
    }
    DataBlockEncoder d = encoding.getEncoder();
    HFileContext meta = new HFileContextBuilder()
        .withDataBlockEncoding(encoding)
        .withCompression(Compression.Algorithm.NONE)
        .withIncludesMvcc(includesMemstoreTS)
        .withIncludesTags(USE_TAG).build();
    codecs.add(new EncodedDataBlock(d, encoding, rawKVs, meta ));
  }
}
 
Example 5
Source File: TestBufferedDataBlockEncoder.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testKVCodecWithTagsForDecodedCellsWithNoTags() throws Exception {
  KeyValue kv1 = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("1"),
      HConstants.LATEST_TIMESTAMP, Bytes.toBytes("1"));
  // kv1.getKey() return a copy of the Key bytes which starts from RK_length. Means from offsets,
  // we need to reduce the KL and VL parts.
  OnheapDecodedCell c1 = new OnheapDecodedCell(kv1.getKey(), kv1.getRowLength(),
      kv1.getFamilyOffset() - KeyValue.ROW_OFFSET, kv1.getFamilyLength(),
      kv1.getQualifierOffset() - KeyValue.ROW_OFFSET, kv1.getQualifierLength(),
      kv1.getTimestamp(), kv1.getTypeByte(), kv1.getValueArray(), kv1.getValueOffset(),
      kv1.getValueLength(), kv1.getSequenceId(), kv1.getTagsArray(), kv1.getTagsOffset(),
      kv1.getTagsLength());
  KeyValue kv2 = new KeyValue(Bytes.toBytes("r2"), Bytes.toBytes("f"), Bytes.toBytes("2"),
      HConstants.LATEST_TIMESTAMP, Bytes.toBytes("2"));
  OnheapDecodedCell c2 = new OnheapDecodedCell(kv2.getKey(), kv2.getRowLength(),
      kv2.getFamilyOffset() - KeyValue.ROW_OFFSET, kv2.getFamilyLength(),
      kv2.getQualifierOffset() - KeyValue.ROW_OFFSET, kv2.getQualifierLength(),
      kv2.getTimestamp(), kv2.getTypeByte(), kv2.getValueArray(), kv2.getValueOffset(),
      kv2.getValueLength(), kv2.getSequenceId(), kv2.getTagsArray(), kv2.getTagsOffset(),
      kv2.getTagsLength());
  KeyValue kv3 = new KeyValue(Bytes.toBytes("r3"), Bytes.toBytes("cf"), Bytes.toBytes("qual"),
      HConstants.LATEST_TIMESTAMP, Bytes.toBytes("3"));
  BufferedDataBlockEncoder.OffheapDecodedExtendedCell
      c3 = new BufferedDataBlockEncoder.OffheapDecodedExtendedCell(ByteBuffer.wrap(kv2.getKey()),
      kv2.getRowLength(), kv2.getFamilyOffset() - KeyValue.ROW_OFFSET, kv2.getFamilyLength(),
      kv2.getQualifierOffset() - KeyValue.ROW_OFFSET, kv2.getQualifierLength(),
      kv2.getTimestamp(), kv2.getTypeByte(), ByteBuffer.wrap(kv2.getValueArray()),
      kv2.getValueOffset(), kv2.getValueLength(), kv2.getSequenceId(),
      ByteBuffer.wrap(kv2.getTagsArray()), kv2.getTagsOffset(), kv2.getTagsLength());
  ByteArrayOutputStream os = new ByteArrayOutputStream();
  KeyValueCodecWithTags codec = new KeyValueCodecWithTags();
  Encoder encoder = codec.getEncoder(os);
  encoder.write(c1);
  encoder.write(c2);
  encoder.write(c3);
  ByteArrayInputStream is = new ByteArrayInputStream(os.toByteArray());
  Decoder decoder = codec.getDecoder(is);
  assertTrue(decoder.advance());
  assertTrue(CellUtil.equals(c1, decoder.current()));
  assertTrue(decoder.advance());
  assertTrue(CellUtil.equals(c2, decoder.current()));
  assertTrue(decoder.advance());
  assertTrue(CellUtil.equals(c3, decoder.current()));
  assertFalse(decoder.advance());
}