Java Code Examples for org.apache.hadoop.hbase.KeyValue#getLength()
The following examples show how to use
org.apache.hadoop.hbase.KeyValue#getLength() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestProtobufUtil.java From hbase with Apache License 2.0 | 6 votes |
@Test public void testToCell() { KeyValue kv1 = new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); KeyValue kv2 = new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); KeyValue kv3 = new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]); byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()]; System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength()); System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength()); System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(), kv3.getLength()); ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length); dbb.put(arr); ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength()); CellProtos.Cell cell = ProtobufUtil.toCell(offheapKV); Cell newOffheapKV = ProtobufUtil.toCell(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY), cell); assertTrue(CellComparatorImpl.COMPARATOR.compare(offheapKV, newOffheapKV) == 0); }
Example 2
Source File: TestKeyValueCodec.java From hbase with Apache License 2.0 | 6 votes |
@Test public void testOne() throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); CountingOutputStream cos = new CountingOutputStream(baos); DataOutputStream dos = new DataOutputStream(cos); KeyValueCodec kvc = new KeyValueCodec(); Codec.Encoder encoder = kvc.getEncoder(dos); final KeyValue kv = new KeyValue(Bytes.toBytes("r"), Bytes.toBytes("f"), Bytes.toBytes("q"), Bytes.toBytes("v")); final int length = kv.getLength() + Bytes.SIZEOF_INT; encoder.write(kv); encoder.flush(); dos.close(); long offset = cos.getCount(); assertEquals(length, offset); CountingInputStream cis = new CountingInputStream(new ByteArrayInputStream(baos.toByteArray())); DataInputStream dis = new DataInputStream(cis); Codec.Decoder decoder = kvc.getDecoder(dis); assertTrue(decoder.advance()); // First read should pull in the KV // Second read should trip over the end-of-stream marker and return false assertFalse(decoder.advance()); dis.close(); assertEquals(length, cis.getCount()); }
Example 3
Source File: MappedByteBufferSortedQueue.java From phoenix with Apache License 2.0 | 5 votes |
private int sizeof(List<KeyValue> kvs) { int size = Bytes.SIZEOF_INT; // totalLen for (KeyValue kv : kvs) { size += kv.getLength(); size += Bytes.SIZEOF_INT; // kv.getLength } return size; }
Example 4
Source File: HFileAppender.java From tajo with Apache License 2.0 | 5 votes |
@Override public void addTuple(Tuple tuple) throws IOException { Datum datum; byte[] rowkey = getRowKeyBytes(tuple); if (!first && !Bytes.equals(keyWritable.get(), 0, keyWritable.getLength(), rowkey, 0, rowkey.length)) { try { for (KeyValue kv : kvSet) { writer.write(keyWritable, kv); totalNumBytes += keyWritable.getLength() + kv.getLength(); } kvSet.clear(); // Statistical section if (tableStatsEnabled) { stats.incrementRow(); } } catch (InterruptedException e) { LOG.error(e.getMessage(), e); } } first = false; keyWritable.set(rowkey); readKeyValues(tuple, rowkey); if (keyValues != null) { Collections.addAll(kvSet, keyValues); } }
Example 5
Source File: PhoenixQueues.java From phoenix with Apache License 2.0 | 5 votes |
public static SizeAwareQueue<Tuple> newSizeBoundTupleQueue(long maxSizeBytes) { LinkedList<Tuple> results = Lists.newLinkedList(); return new SizeBoundQueue<Tuple>(maxSizeBytes, results) { @Override public long sizeOf(Tuple e) { KeyValue kv = PhoenixKeyValueUtil.maybeCopyCell(e.getValue(0)); return Bytes.SIZEOF_INT * 2 + kv.getLength(); } }; }
Example 6
Source File: OrderedResultIterator.java From phoenix with Apache License 2.0 | 5 votes |
private static long sizeof(List<KeyValue> kvs) { long size = Bytes.SIZEOF_INT; // totalLen for (KeyValue kv : kvs) { size += kv.getLength(); size += Bytes.SIZEOF_INT; // kv.getLength } return size; }
Example 7
Source File: MappedByteBufferSortedQueue.java From phoenix with BSD 3-Clause "New" or "Revised" License | 5 votes |
private int sizeof(List<KeyValue> kvs) { int size = Bytes.SIZEOF_INT; // totalLen for (KeyValue kv : kvs) { size += kv.getLength(); size += Bytes.SIZEOF_INT; // kv.getLength } return size; }
Example 8
Source File: SortMergeJoinPlan.java From phoenix with Apache License 2.0 | 4 votes |
@Override protected int sizeOf(Tuple e) { KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0)); return Bytes.SIZEOF_INT * 2 + kv.getLength(); }
Example 9
Source File: StatisticsCollector.java From phoenix with Apache License 2.0 | 4 votes |
/** * Update the current statistics based on the latest batch of key-values from the underlying scanner * * @param results * next batch of {@link KeyValue}s */ public void collectStatistics(final List<Cell> results) { Map<ImmutableBytesPtr, Boolean> famMap = Maps.newHashMap(); List<GuidePostsInfo> rowTracker = null; if(cachedGps == null) { rowTracker = new ArrayList<GuidePostsInfo>(); } for (Cell cell : results) { KeyValue kv = KeyValueUtil.ensureKeyValue(cell); maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp()); Pair<Long, GuidePostsInfo> gps; if (cachedGps == null) { ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength()); gps = guidePostsMap.get(cfKey); if (gps == null) { gps = new Pair<Long, GuidePostsInfo>(0l, new GuidePostsInfo(0, Collections.<byte[]> emptyList(), 0l)); guidePostsMap.put(cfKey, gps); } if (famMap.get(cfKey) == null) { famMap.put(cfKey, true); rowTracker.add(gps.getSecond()); } } else { gps = cachedGps; } int kvLength = kv.getLength(); long byteCount = gps.getFirst() + kvLength; gps.setFirst(byteCount); if (byteCount >= guidepostDepth) { byte[] row = ByteUtil.copyKeyBytesIfNecessary(new ImmutableBytesWritable(kv.getRowArray(), kv .getRowOffset(), kv.getRowLength())); if (gps.getSecond().addGuidePost(row, byteCount)) { gps.setFirst(0l); } } } if(cachedGps == null) { for (GuidePostsInfo s : rowTracker) { s.incrementRowCount(); } } else { cachedGps.getSecond().incrementRowCount(); } }
Example 10
Source File: TestWALReaderOnSecureWAL.java From hbase with Apache License 2.0 | 4 votes |
private Path writeWAL(final WALFactory wals, final String tblName, boolean offheap) throws IOException { Configuration conf = TEST_UTIL.getConfiguration(); String clsName = conf.get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName()); conf.setClass(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, SecureWALCellCodec.class, WALCellCodec.class); try { TableName tableName = TableName.valueOf(tblName); NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR); scopes.put(tableName.getName(), 0); RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build(); final int total = 10; final byte[] row = Bytes.toBytes("row"); final byte[] family = Bytes.toBytes("family"); final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1); // Write the WAL WAL wal = wals.getWAL(regionInfo); for (int i = 0; i < total; i++) { WALEdit kvs = new WALEdit(); KeyValue kv = new KeyValue(row, family, Bytes.toBytes(i), value); if (offheap) { ByteBuffer bb = ByteBuffer.allocateDirect(kv.getBuffer().length); bb.put(kv.getBuffer()); ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(bb, 0, kv.getLength()); kvs.add(offheapKV); } else { kvs.add(kv); } wal.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName, System.currentTimeMillis(), mvcc, scopes), kvs); } wal.sync(); final Path walPath = AbstractFSWALProvider.getCurrentFileName(wal); wal.shutdown(); return walPath; } finally { // restore the cell codec class conf.set(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, clsName); } }
Example 11
Source File: DataBlockEncodingTool.java From hbase with Apache License 2.0 | 4 votes |
/** * Check statistics for given HFile for different data block encoders. * @param scanner Of file which will be compressed. * @param kvLimit Maximal count of KeyValue which will be processed. * @throws IOException thrown if scanner is invalid */ public void checkStatistics(final KeyValueScanner scanner, final int kvLimit) throws IOException { scanner.seek(KeyValue.LOWESTKEY); KeyValue currentKV; byte[] previousKey = null; byte[] currentKey; DataBlockEncoding[] encodings = DataBlockEncoding.values(); ByteArrayOutputStream uncompressedOutputStream = new ByteArrayOutputStream(); int j = 0; while ((currentKV = KeyValueUtil.ensureKeyValue(scanner.next())) != null && j < kvLimit) { // Iterates through key/value pairs j++; currentKey = currentKV.getKey(); if (previousKey != null) { for (int i = 0; i < previousKey.length && i < currentKey.length && previousKey[i] == currentKey[i]; ++i) { totalKeyRedundancyLength++; } } // Add tagsLen zero to cells don't include tags. Since the process of // scanner converts byte array to KV would abandon tagsLen part if tagsLen // is zero. But we still needs the tagsLen part to check if current cell // include tags. If USE_TAG is true, HFile contains cells with tags, // if the cell tagsLen equals 0, it means other cells may have tags. if (USE_TAG && currentKV.getTagsLength() == 0) { uncompressedOutputStream.write(currentKV.getBuffer(), currentKV.getOffset(), currentKV.getLength()); // write tagsLen = 0. uncompressedOutputStream.write(Bytes.toBytes((short) 0)); } else { uncompressedOutputStream.write(currentKV.getBuffer(), currentKV.getOffset(), currentKV.getLength()); } if(includesMemstoreTS) { WritableUtils.writeVLong( new DataOutputStream(uncompressedOutputStream), currentKV.getSequenceId()); } previousKey = currentKey; int kLen = currentKV.getKeyLength(); int vLen = currentKV.getValueLength(); int cfLen = currentKV.getFamilyLength(currentKV.getFamilyOffset()); int restLen = currentKV.getLength() - kLen - vLen; totalKeyLength += kLen; totalValueLength += vLen; totalPrefixLength += restLen; totalCFLength += cfLen; } rawKVs = uncompressedOutputStream.toByteArray(); for (DataBlockEncoding encoding : encodings) { if (encoding == DataBlockEncoding.NONE) { continue; } DataBlockEncoder d = encoding.getEncoder(); HFileContext meta = new HFileContextBuilder() .withDataBlockEncoding(encoding) .withCompression(Compression.Algorithm.NONE) .withIncludesMvcc(includesMemstoreTS) .withIncludesTags(USE_TAG).build(); codecs.add(new EncodedDataBlock(d, encoding, rawKVs, meta )); } }
Example 12
Source File: TestHFileBlock.java From hbase with Apache License 2.0 | 4 votes |
static int writeTestKeyValues(HFileBlock.Writer hbw, int seed, boolean includesMemstoreTS, boolean useTag) throws IOException { List<KeyValue> keyValues = new ArrayList<>(); Random randomizer = new Random(42L + seed); // just any fixed number // generate keyValues for (int i = 0; i < NUM_KEYVALUES; ++i) { byte[] row; long timestamp; byte[] family; byte[] qualifier; byte[] value; // generate it or repeat, it should compress well if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) { row = CellUtil.cloneRow(keyValues.get(randomizer.nextInt(keyValues.size()))); } else { row = new byte[FIELD_LENGTH]; randomizer.nextBytes(row); } if (0 == i) { family = new byte[FIELD_LENGTH]; randomizer.nextBytes(family); } else { family = CellUtil.cloneFamily(keyValues.get(0)); } if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) { qualifier = CellUtil.cloneQualifier(keyValues.get(randomizer.nextInt(keyValues.size()))); } else { qualifier = new byte[FIELD_LENGTH]; randomizer.nextBytes(qualifier); } if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) { value = CellUtil.cloneValue(keyValues.get(randomizer.nextInt(keyValues.size()))); } else { value = new byte[FIELD_LENGTH]; randomizer.nextBytes(value); } if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) { timestamp = keyValues.get( randomizer.nextInt(keyValues.size())).getTimestamp(); } else { timestamp = randomizer.nextLong(); } if (!useTag) { keyValues.add(new KeyValue(row, family, qualifier, timestamp, value)); } else { keyValues.add(new KeyValue(row, family, qualifier, timestamp, value, new Tag[] { new ArrayBackedTag((byte) 1, Bytes.toBytes("myTagVal")) })); } } // sort it and write to stream int totalSize = 0; Collections.sort(keyValues, CellComparatorImpl.COMPARATOR); for (KeyValue kv : keyValues) { totalSize += kv.getLength(); if (includesMemstoreTS) { long memstoreTS = randomizer.nextLong(); kv.setSequenceId(memstoreTS); totalSize += WritableUtils.getVIntSize(memstoreTS); } hbw.write(kv); } return totalSize; }
Example 13
Source File: BufferedTupleQueue.java From phoenix with Apache License 2.0 | 4 votes |
@Override protected long sizeOf(Tuple e) { KeyValue kv = PhoenixKeyValueUtil.maybeCopyCell(e.getValue(0)); return Bytes.SIZEOF_INT * 2 + kv.getLength(); }