Java Code Examples for org.apache.hadoop.hbase.KeyValue#getLength()

The following examples show how to use org.apache.hadoop.hbase.KeyValue#getLength() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestProtobufUtil.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testToCell() {
  KeyValue kv1 =
      new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
  KeyValue kv2 =
      new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
  KeyValue kv3 =
      new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
  byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()];
  System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength());
  System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength());
  System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(),
    kv3.getLength());
  ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length);
  dbb.put(arr);
  ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength());
  CellProtos.Cell cell = ProtobufUtil.toCell(offheapKV);
  Cell newOffheapKV =
      ProtobufUtil.toCell(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY), cell);
  assertTrue(CellComparatorImpl.COMPARATOR.compare(offheapKV, newOffheapKV) == 0);
}
 
Example 2
Source File: TestKeyValueCodec.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testOne() throws IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  CountingOutputStream cos = new CountingOutputStream(baos);
  DataOutputStream dos = new DataOutputStream(cos);
  KeyValueCodec kvc = new KeyValueCodec();
  Codec.Encoder encoder = kvc.getEncoder(dos);
  final KeyValue kv =
    new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v"));
  final int length = kv.getLength() + Bytes.SIZEOF_INT;
  encoder.write(kv);
  encoder.flush();
  dos.close();
  long offset = cos.getCount();
  assertEquals(length, offset);
  CountingInputStream cis =
    new CountingInputStream(new ByteArrayInputStream(baos.toByteArray()));
  DataInputStream dis = new DataInputStream(cis);
  Codec.Decoder decoder = kvc.getDecoder(dis);
  assertTrue(decoder.advance()); // First read should pull in the KV
  // Second read should trip over the end-of-stream  marker and return false
  assertFalse(decoder.advance());
  dis.close();
  assertEquals(length, cis.getCount());
}
 
Example 3
Source File: MappedByteBufferSortedQueue.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private int sizeof(List<KeyValue> kvs) {
    int size = Bytes.SIZEOF_INT; // totalLen

    for (KeyValue kv : kvs) {
        size += kv.getLength();
        size += Bytes.SIZEOF_INT; // kv.getLength
    }

    return size;
}
 
Example 4
Source File: HFileAppender.java    From tajo with Apache License 2.0 5 votes vote down vote up
@Override
public void addTuple(Tuple tuple) throws IOException {
  Datum datum;

  byte[] rowkey = getRowKeyBytes(tuple);

  if (!first && !Bytes.equals(keyWritable.get(), 0, keyWritable.getLength(), rowkey, 0, rowkey.length)) {
    try {
      for (KeyValue kv : kvSet) {
        writer.write(keyWritable, kv);
        totalNumBytes += keyWritable.getLength() + kv.getLength();
      }
      kvSet.clear();
      // Statistical section
      if (tableStatsEnabled) {
        stats.incrementRow();
      }
    } catch (InterruptedException e) {
      LOG.error(e.getMessage(), e);
    }
  }

  first = false;

  keyWritable.set(rowkey);

  readKeyValues(tuple, rowkey);
  if (keyValues != null) {
    Collections.addAll(kvSet, keyValues);
  }
}
 
Example 5
Source File: PhoenixQueues.java    From phoenix with Apache License 2.0 5 votes vote down vote up
public static SizeAwareQueue<Tuple> newSizeBoundTupleQueue(long maxSizeBytes) {
    LinkedList<Tuple> results = Lists.newLinkedList();
    return new SizeBoundQueue<Tuple>(maxSizeBytes, results) {

        @Override
        public long sizeOf(Tuple e) {
            KeyValue kv = PhoenixKeyValueUtil.maybeCopyCell(e.getValue(0));
            return Bytes.SIZEOF_INT * 2 + kv.getLength();
        }

    };
}
 
Example 6
Source File: OrderedResultIterator.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private static long sizeof(List<KeyValue> kvs) {
  long size = Bytes.SIZEOF_INT; // totalLen

  for (KeyValue kv : kvs) {
      size += kv.getLength();
      size += Bytes.SIZEOF_INT; // kv.getLength
  }

  return size;
}
 
Example 7
Source File: MappedByteBufferSortedQueue.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
private int sizeof(List<KeyValue> kvs) {
    int size = Bytes.SIZEOF_INT; // totalLen

    for (KeyValue kv : kvs) {
        size += kv.getLength();
        size += Bytes.SIZEOF_INT; // kv.getLength
    }

    return size;
}
 
Example 8
Source File: SortMergeJoinPlan.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@Override
protected int sizeOf(Tuple e) {
    KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0));
    return Bytes.SIZEOF_INT * 2 + kv.getLength();
}
 
Example 9
Source File: StatisticsCollector.java    From phoenix with Apache License 2.0 4 votes vote down vote up
/**
 * Update the current statistics based on the latest batch of key-values from the underlying scanner
 * 
 * @param results
 *            next batch of {@link KeyValue}s
 */
public void collectStatistics(final List<Cell> results) {
    Map<ImmutableBytesPtr, Boolean> famMap = Maps.newHashMap();
    List<GuidePostsInfo> rowTracker = null;
    if(cachedGps == null) {
        rowTracker = 
                new ArrayList<GuidePostsInfo>();
    }
    for (Cell cell : results) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp());
        Pair<Long, GuidePostsInfo> gps;
        if (cachedGps == null) {
            ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(),
                    kv.getFamilyLength());
            gps = guidePostsMap.get(cfKey);
            if (gps == null) {
                gps = new Pair<Long, GuidePostsInfo>(0l, new GuidePostsInfo(0,
                        Collections.<byte[]> emptyList(), 0l));
                guidePostsMap.put(cfKey, gps);
            }
            if (famMap.get(cfKey) == null) {
                famMap.put(cfKey, true);
                rowTracker.add(gps.getSecond());
            }
        } else {
            gps = cachedGps;
        }
        int kvLength = kv.getLength();
        long byteCount = gps.getFirst() + kvLength;
        gps.setFirst(byteCount);
        if (byteCount >= guidepostDepth) {
            byte[] row = ByteUtil.copyKeyBytesIfNecessary(new ImmutableBytesWritable(kv.getRowArray(), kv
                    .getRowOffset(), kv.getRowLength()));
            if (gps.getSecond().addGuidePost(row, byteCount)) {
                gps.setFirst(0l);
            }
        }
    }
    if(cachedGps == null) {
        for (GuidePostsInfo s : rowTracker) {
            s.incrementRowCount();
        }
    } else {
        cachedGps.getSecond().incrementRowCount();
    }
}
 
Example 10
Source File: TestWALReaderOnSecureWAL.java    From hbase with Apache License 2.0 4 votes vote down vote up
private Path writeWAL(final WALFactory wals, final String tblName, boolean offheap)
    throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  String clsName = conf.get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
  conf.setClass(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, SecureWALCellCodec.class,
    WALCellCodec.class);
  try {
    TableName tableName = TableName.valueOf(tblName);
    NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
    scopes.put(tableName.getName(), 0);
    RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
    final int total = 10;
    final byte[] row = Bytes.toBytes("row");
    final byte[] family = Bytes.toBytes("family");
    final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);

    // Write the WAL
    WAL wal = wals.getWAL(regionInfo);
    for (int i = 0; i < total; i++) {
      WALEdit kvs = new WALEdit();
      KeyValue kv = new KeyValue(row, family, Bytes.toBytes(i), value);
      if (offheap) {
        ByteBuffer bb = ByteBuffer.allocateDirect(kv.getBuffer().length);
        bb.put(kv.getBuffer());
        ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(bb, 0, kv.getLength());
        kvs.add(offheapKV);
      } else {
        kvs.add(kv);
      }
      wal.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName,
        System.currentTimeMillis(), mvcc, scopes), kvs);
    }
    wal.sync();
    final Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);
    wal.shutdown();

    return walPath;
  } finally {
    // restore the cell codec class
    conf.set(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, clsName);
  }
}
 
Example 11
Source File: DataBlockEncodingTool.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Check statistics for given HFile for different data block encoders.
 * @param scanner Of file which will be compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @throws IOException thrown if scanner is invalid
 */
public void checkStatistics(final KeyValueScanner scanner, final int kvLimit)
    throws IOException {
  scanner.seek(KeyValue.LOWESTKEY);

  KeyValue currentKV;

  byte[] previousKey = null;
  byte[] currentKey;

  DataBlockEncoding[] encodings = DataBlockEncoding.values();

  ByteArrayOutputStream uncompressedOutputStream =
      new ByteArrayOutputStream();

  int j = 0;
  while ((currentKV = KeyValueUtil.ensureKeyValue(scanner.next())) != null && j < kvLimit) {
    // Iterates through key/value pairs
    j++;
    currentKey = currentKV.getKey();
    if (previousKey != null) {
      for (int i = 0; i < previousKey.length && i < currentKey.length &&
          previousKey[i] == currentKey[i]; ++i) {
        totalKeyRedundancyLength++;
      }
    }

    // Add tagsLen zero to cells don't include tags. Since the process of
    // scanner converts byte array to KV would abandon tagsLen part if tagsLen
    // is zero. But we still needs the tagsLen part to check if current cell
    // include tags. If USE_TAG is true, HFile contains cells with tags,
    // if the cell tagsLen equals 0, it means other cells may have tags.
    if (USE_TAG && currentKV.getTagsLength() == 0) {
      uncompressedOutputStream.write(currentKV.getBuffer(),
          currentKV.getOffset(), currentKV.getLength());
      // write tagsLen = 0.
      uncompressedOutputStream.write(Bytes.toBytes((short) 0));
    } else {
      uncompressedOutputStream.write(currentKV.getBuffer(),
          currentKV.getOffset(), currentKV.getLength());
    }

    if(includesMemstoreTS) {
      WritableUtils.writeVLong(
          new DataOutputStream(uncompressedOutputStream), currentKV.getSequenceId());
    }

    previousKey = currentKey;

    int kLen = currentKV.getKeyLength();
    int vLen = currentKV.getValueLength();
    int cfLen = currentKV.getFamilyLength(currentKV.getFamilyOffset());
    int restLen = currentKV.getLength() - kLen - vLen;

    totalKeyLength += kLen;
    totalValueLength += vLen;
    totalPrefixLength += restLen;
    totalCFLength += cfLen;
  }

  rawKVs = uncompressedOutputStream.toByteArray();
  for (DataBlockEncoding encoding : encodings) {
    if (encoding == DataBlockEncoding.NONE) {
      continue;
    }
    DataBlockEncoder d = encoding.getEncoder();
    HFileContext meta = new HFileContextBuilder()
        .withDataBlockEncoding(encoding)
        .withCompression(Compression.Algorithm.NONE)
        .withIncludesMvcc(includesMemstoreTS)
        .withIncludesTags(USE_TAG).build();
    codecs.add(new EncodedDataBlock(d, encoding, rawKVs, meta ));
  }
}
 
Example 12
Source File: TestHFileBlock.java    From hbase with Apache License 2.0 4 votes vote down vote up
static int writeTestKeyValues(HFileBlock.Writer hbw, int seed, boolean includesMemstoreTS,
    boolean useTag) throws IOException {
  List<KeyValue> keyValues = new ArrayList<>();
  Random randomizer = new Random(42L + seed); // just any fixed number

  // generate keyValues
  for (int i = 0; i < NUM_KEYVALUES; ++i) {
    byte[] row;
    long timestamp;
    byte[] family;
    byte[] qualifier;
    byte[] value;

    // generate it or repeat, it should compress well
    if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
      row = CellUtil.cloneRow(keyValues.get(randomizer.nextInt(keyValues.size())));
    } else {
      row = new byte[FIELD_LENGTH];
      randomizer.nextBytes(row);
    }
    if (0 == i) {
      family = new byte[FIELD_LENGTH];
      randomizer.nextBytes(family);
    } else {
      family = CellUtil.cloneFamily(keyValues.get(0));
    }
    if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
      qualifier = CellUtil.cloneQualifier(keyValues.get(randomizer.nextInt(keyValues.size())));
    } else {
      qualifier = new byte[FIELD_LENGTH];
      randomizer.nextBytes(qualifier);
    }
    if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
      value = CellUtil.cloneValue(keyValues.get(randomizer.nextInt(keyValues.size())));
    } else {
      value = new byte[FIELD_LENGTH];
      randomizer.nextBytes(value);
    }
    if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
      timestamp = keyValues.get(
          randomizer.nextInt(keyValues.size())).getTimestamp();
    } else {
      timestamp = randomizer.nextLong();
    }
    if (!useTag) {
      keyValues.add(new KeyValue(row, family, qualifier, timestamp, value));
    } else {
      keyValues.add(new KeyValue(row, family, qualifier, timestamp, value,
          new Tag[] { new ArrayBackedTag((byte) 1, Bytes.toBytes("myTagVal")) }));
    }
  }

  // sort it and write to stream
  int totalSize = 0;
  Collections.sort(keyValues, CellComparatorImpl.COMPARATOR);

  for (KeyValue kv : keyValues) {
    totalSize += kv.getLength();
    if (includesMemstoreTS) {
      long memstoreTS = randomizer.nextLong();
      kv.setSequenceId(memstoreTS);
      totalSize += WritableUtils.getVIntSize(memstoreTS);
    }
    hbw.write(kv);
  }
  return totalSize;
}
 
Example 13
Source File: BufferedTupleQueue.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@Override
protected long sizeOf(Tuple e) {
    KeyValue kv = PhoenixKeyValueUtil.maybeCopyCell(e.getValue(0));
    return Bytes.SIZEOF_INT * 2 + kv.getLength();
}