Java Code Examples for org.apache.hadoop.hbase.KeyValue#KeyOnlyKeyValue

The following examples show how to use org.apache.hadoop.hbase.KeyValue#KeyOnlyKeyValue . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hbase   File: StoreFileReader.java    License: Apache License 2.0 6 votes vote down vote up
public Map<byte[], byte[]> loadFileInfo() throws IOException {
  Map<byte [], byte []> fi = reader.getHFileInfo();

  byte[] b = fi.get(BLOOM_FILTER_TYPE_KEY);
  if (b != null) {
    bloomFilterType = BloomType.valueOf(Bytes.toString(b));
  }

  byte[] p = fi.get(BLOOM_FILTER_PARAM_KEY);
  if (bloomFilterType ==  BloomType.ROWPREFIX_FIXED_LENGTH) {
    prefixLength = Bytes.toInt(p);
  }

  lastBloomKey = fi.get(LAST_BLOOM_KEY);
  if(bloomFilterType == BloomType.ROWCOL) {
    lastBloomKeyOnlyKV = new KeyValue.KeyOnlyKeyValue(lastBloomKey, 0, lastBloomKey.length);
  }
  byte[] cnt = fi.get(DELETE_FAMILY_COUNT);
  if (cnt != null) {
    deleteFamilyCnt = Bytes.toLong(cnt);
  }

  return fi;
}
 
Example 2
Source Project: hbase   File: HFileReaderImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Cell getKey() {
  assertSeeked();
  // Create a new object so that this getKey is cached as firstKey, lastKey
  ObjectIntPair<ByteBuffer> keyPair = new ObjectIntPair<>();
  blockBuffer.asSubByteBuffer(blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen, keyPair);
  ByteBuffer keyBuf = keyPair.getFirst();
  if (keyBuf.hasArray()) {
    return new KeyValue.KeyOnlyKeyValue(keyBuf.array(), keyBuf.arrayOffset()
        + keyPair.getSecond(), currKeyLen);
  } else {
    // Better to do a copy here instead of holding on to this BB so that
    // we could release the blocks referring to this key. This key is specifically used
    // in HalfStoreFileReader to get the firstkey and lastkey by creating a new scanner
    // every time. So holding onto the BB (incase of DBB) is not advised here.
    byte[] key = new byte[currKeyLen];
    ByteBufferUtils.copyFromBufferToArray(key, keyBuf, keyPair.getSecond(), 0, currKeyLen);
    return new KeyValue.KeyOnlyKeyValue(key, 0, currKeyLen);
  }
}
 
Example 3
/**
 * In case of top half store, the passed key will be with the start key of the daughter region.
 * But in the actual HFiles, the key will be with the start key of the old parent region. In
 * order to make the real seek in the HFiles, we need to build the old key.
 *
 * The logic here is just replace daughter region start key with parent region start key
 * in the key part.
 *
 * @param key
 *
 */
private KeyValue getKeyPresentInHFiles(Cell keyValue) {
    int rowLength = keyValue.getRowLength();
    int rowOffset = keyValue.getRowOffset();

    short length = (short) (rowLength - reader.getSplitRow().length + reader.getOffset());
    byte[] replacedKey =
            new byte[length + keyValue.getRowArray().length - (rowOffset + rowLength) + ROW_LENGTH_SIZE];
    System.arraycopy(Bytes.toBytes(length), 0, replacedKey, 0, ROW_LENGTH_SIZE);
    System.arraycopy(reader.getRegionStartKeyInHFile(), 0, replacedKey, ROW_LENGTH_SIZE, reader.getOffset());
    System.arraycopy(keyValue.getRowArray(), keyValue.getRowOffset() + reader.getSplitRow().length,
        replacedKey, reader.getOffset() + ROW_LENGTH_SIZE, rowLength
                - reader.getSplitRow().length);
    System.arraycopy(keyValue.getRowArray(), rowOffset + rowLength, replacedKey,
        reader.getOffset() + keyValue.getRowLength() - reader.getSplitRow().length
                + ROW_LENGTH_SIZE, keyValue.getRowArray().length - (rowOffset + rowLength));
    return new KeyValue.KeyOnlyKeyValue(replacedKey);
}
 
Example 4
/**
 * 
 * @param kv
 * @param isSeek pass true for seek, false for reseek.
 * @return 
 * @throws IOException
 */
public boolean seekOrReseek(Cell cell, boolean isSeek) throws IOException{
    Cell keyToSeek = cell;
    KeyValue splitKeyValue = new KeyValue.KeyOnlyKeyValue(reader.getSplitkey());
    if (reader.isTop()) {
        if(this.comparator.compare(cell, splitKeyValue, true) < 0){
            if(!isSeek && realSeekDone()) {
                return true;
            }
            return seekOrReseekToProperKey(isSeek, keyToSeek);
        }
        keyToSeek = getKeyPresentInHFiles(cell);
        return seekOrReseekToProperKey(isSeek, keyToSeek);
    } else {
        if (this.comparator.compare(cell, splitKeyValue, true) >= 0) {
            close();
            return false;
        }
        if(!isSeek && reader.getRegionInfo().getStartKey().length == 0 && reader.getSplitRow().length > reader.getRegionStartKeyInHFile().length) {
            keyToSeek = getKeyPresentInHFiles(cell);
        }
    }
    return seekOrReseekToProperKey(isSeek, keyToSeek);
}
 
Example 5
Source Project: hbase   File: HFileInfo.java    License: Apache License 2.0 5 votes vote down vote up
private void loadMetaInfo(HFileBlock.BlockIterator blockIter, HFileContext hfileContext)
    throws IOException {
  read(blockIter.nextBlockWithBlockType(BlockType.FILE_INFO).getByteStream());
  byte[] creationTimeBytes = get(HFileInfo.CREATE_TIME_TS);
  hfileContext.setFileCreateTime(creationTimeBytes == null ?
      0 : Bytes.toLong(creationTimeBytes));
  byte[] tmp = get(HFileInfo.MAX_TAGS_LEN);
  // max tag length is not present in the HFile means tags were not at all written to file.
  if (tmp != null) {
    hfileContext.setIncludesTags(true);
    tmp = get(HFileInfo.TAGS_COMPRESSED);
    if (tmp != null && Bytes.toBoolean(tmp)) {
      hfileContext.setCompressTags(true);
    }
  }
  // parse meta info
  if (get(HFileInfo.LASTKEY) != null) {
    lastKeyCell = new KeyValue.KeyOnlyKeyValue(get(HFileInfo.LASTKEY));
  }
  avgKeyLen = Bytes.toInt(get(HFileInfo.AVG_KEY_LEN));
  avgValueLen = Bytes.toInt(get(HFileInfo.AVG_VALUE_LEN));
  byte [] keyValueFormatVersion = get(HFileWriterImpl.KEY_VALUE_VERSION);
  includesMemstoreTS = keyValueFormatVersion != null &&
      Bytes.toInt(keyValueFormatVersion) == HFileWriterImpl.KEY_VALUE_VER_WITH_MEMSTORE;
  hfileContext.setIncludesMvcc(includesMemstoreTS);
  if (includesMemstoreTS) {
    decodeMemstoreTS = Bytes.toLong(get(HFileWriterImpl.MAX_MEMSTORE_TS_KEY)) > 0;
  }
}
 
Example 6
Source Project: hbase   File: HFileBlockIndex.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Adds a new entry in the root block index. Only used when reading.
 *
 * @param key Last key in the block
 * @param offset file offset where the block is stored
 * @param dataSize the uncompressed data size
 */
@Override
protected void add(final byte[] key, final long offset, final int dataSize) {
  blockOffsets[rootCount] = offset;
  // Create the blockKeys as Cells once when the reader is opened
  blockKeys[rootCount] = new KeyValue.KeyOnlyKeyValue(key, 0, key.length);
  blockDataSizes[rootCount] = dataSize;
  rootCount++;
}
 
Example 7
Source Project: hbase   File: HFileReaderImpl.java    License: Apache License 2.0 5 votes vote down vote up
protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) {
  ByteBuff buffer = curBlock.getBufferWithoutHeader();
  // It is safe to manipulate this buffer because we own the buffer object.
  buffer.rewind();
  int klen = buffer.getInt();
  buffer.skip(Bytes.SIZEOF_INT);// Skip value len part
  ByteBuffer keyBuff = buffer.asSubByteBuffer(klen);
  if (keyBuff.hasArray()) {
    return new KeyValue.KeyOnlyKeyValue(keyBuff.array(), keyBuff.arrayOffset()
        + keyBuff.position(), klen);
  } else {
    return new ByteBufferKeyOnlyKeyValue(keyBuff, keyBuff.position(), klen);
  }
}
 
Example 8
Source Project: hbase   File: HalfStoreFileReader.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a half file reader for a hfile referred to by an hfilelink.
 * @param context Reader context info
 * @param fileInfo HFile info
 * @param cacheConf CacheConfig
 * @param r original reference file (contains top or bottom)
 * @param refCount reference count
 * @param conf Configuration
 */
public HalfStoreFileReader(final ReaderContext context, final HFileInfo fileInfo,
    final CacheConfig cacheConf, final Reference r,
    AtomicInteger refCount, final Configuration conf) throws IOException {
  super(context, fileInfo, cacheConf, refCount, conf);
  // This is not actual midkey for this half-file; its just border
  // around which we split top and bottom.  Have to look in files to find
  // actual last and first keys for bottom and top halves.  Half-files don't
  // have an actual midkey themselves. No midkey is how we indicate file is
  // not splittable.
  this.splitkey = r.getSplitKey();
  this.splitCell = new KeyValue.KeyOnlyKeyValue(this.splitkey, 0, this.splitkey.length);
  // Is it top or bottom half?
  this.top = Reference.isTopFileRegion(r.getFileRegion());
}
 
Example 9
Source Project: hbase   File: AbstractDataBlockEncoder.java    License: Apache License 2.0 5 votes vote down vote up
protected Cell createFirstKeyCell(ByteBuffer key, int keyLength) {
  if (key.hasArray()) {
    return new KeyValue.KeyOnlyKeyValue(key.array(), key.arrayOffset()
        + key.position(), keyLength);
  } else {
    return new ByteBufferKeyOnlyKeyValue(key, key.position(), keyLength);
  }
}
 
Example 10
Source Project: hbase   File: RowIndexSeekerV1.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Cell getKey() {
  if (current.keyBuffer.hasArray()) {
    return new KeyValue.KeyOnlyKeyValue(current.keyBuffer.array(),
        current.keyBuffer.arrayOffset() + current.keyBuffer.position(),
        current.keyLength);
  } else {
    byte[] key = new byte[current.keyLength];
    ByteBufferUtils.copyFromBufferToArray(key, current.keyBuffer,
        current.keyBuffer.position(), 0, current.keyLength);
    return new KeyValue.KeyOnlyKeyValue(key, 0, current.keyLength);
  }
}
 
Example 11
Source Project: hbase   File: HFileBlockIndex.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Cell midkey(CachingBlockReader cachingBlockReader) throws IOException {
  if (rootCount == 0)
    throw new IOException("HFile empty");

  Cell targetMidKey = this.midKey.get();
  if (targetMidKey != null) {
    return targetMidKey;
  }

  if (midLeafBlockOffset >= 0) {
    if (cachingBlockReader == null) {
      throw new IOException("Have to read the middle leaf block but " +
          "no block reader available");
    }

    // Caching, using pread, assuming this is not a compaction.
    HFileBlock midLeafBlock = cachingBlockReader.readBlock(
        midLeafBlockOffset, midLeafBlockOnDiskSize, true, true, false, true,
        BlockType.LEAF_INDEX, null);
    try {
      ByteBuff b = midLeafBlock.getBufferWithoutHeader();
      int numDataBlocks = b.getIntAfterPosition(0);
      int keyRelOffset = b.getIntAfterPosition(Bytes.SIZEOF_INT * (midKeyEntry + 1));
      int keyLen = b.getIntAfterPosition(Bytes.SIZEOF_INT * (midKeyEntry + 2)) - keyRelOffset
          - SECONDARY_INDEX_ENTRY_OVERHEAD;
      int keyOffset =
          Bytes.SIZEOF_INT * (numDataBlocks + 2) + keyRelOffset
              + SECONDARY_INDEX_ENTRY_OVERHEAD;
      byte[] bytes = b.toBytes(keyOffset, keyLen);
      targetMidKey = new KeyValue.KeyOnlyKeyValue(bytes, 0, bytes.length);
    } finally {
      midLeafBlock.release();
    }
  } else {
    // The middle of the root-level index.
    targetMidKey = blockKeys[rootCount / 2];
  }

  this.midKey.set(targetMidKey);
  return targetMidKey;
}
 
Example 12
Source Project: hbase   File: TestHFileBlockIndex.java    License: Apache License 2.0 4 votes vote down vote up
private void readIndex(boolean useTags) throws IOException {
  long fileSize = fs.getFileStatus(path).getLen();
  LOG.info("Size of {}: {} compression={}", path, fileSize, compr.toString());

  FSDataInputStream istream = fs.open(path);
  HFileContext meta = new HFileContextBuilder()
                      .withHBaseCheckSum(true)
                      .withIncludesMvcc(includesMemstoreTS)
                      .withIncludesTags(useTags)
                      .withCompression(compr)
                      .build();
  ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, path).build();
  HFileBlock.FSReader blockReader = new HFileBlock.FSReaderImpl(context, meta,
      ByteBuffAllocator.HEAP);

  BlockReaderWrapper brw = new BlockReaderWrapper(blockReader);
  HFileBlockIndex.BlockIndexReader indexReader =
      new HFileBlockIndex.CellBasedKeyBlockIndexReader(
          CellComparatorImpl.COMPARATOR, numLevels);

  indexReader.readRootIndex(blockReader.blockRange(rootIndexOffset,
      fileSize).nextBlockWithBlockType(BlockType.ROOT_INDEX), numRootEntries);

  long prevOffset = -1;
  int i = 0;
  int expectedHitCount = 0;
  int expectedMissCount = 0;
  LOG.info("Total number of keys: " + keys.size());
  for (byte[] key : keys) {
    assertTrue(key != null);
    assertTrue(indexReader != null);
    KeyValue.KeyOnlyKeyValue keyOnlyKey = new KeyValue.KeyOnlyKeyValue(key, 0, key.length);
    HFileBlock b =
        indexReader.seekToDataBlock(keyOnlyKey, null, true,
          true, false, null, brw);
    if (PrivateCellUtil.compare(CellComparatorImpl.COMPARATOR, keyOnlyKey, firstKeyInFile, 0,
      firstKeyInFile.length) < 0) {
      assertTrue(b == null);
      ++i;
      continue;
    }

    String keyStr = "key #" + i + ", " + Bytes.toStringBinary(key);

    assertTrue("seekToDataBlock failed for " + keyStr, b != null);

    if (prevOffset == b.getOffset()) {
      assertEquals(++expectedHitCount, brw.hitCount);
    } else {
      LOG.info("First key in a new block: " + keyStr + ", block offset: "
          + b.getOffset() + ")");
      assertTrue(b.getOffset() > prevOffset);
      assertEquals(++expectedMissCount, brw.missCount);
      prevOffset = b.getOffset();
    }
    ++i;
  }

  istream.close();
}
 
Example 13
Source Project: hbase   File: BufferedDataBlockEncoder.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Cell getKey() {
  byte[] key = new byte[current.keyLength];
  System.arraycopy(current.keyBuffer, 0, key, 0, current.keyLength);
  return new KeyValue.KeyOnlyKeyValue(key);
}