Java Code Examples for org.apache.kylin.common.KylinConfig#getHBaseScanMaxResultSize()
The following examples show how to use
org.apache.kylin.common.KylinConfig#getHBaseScanMaxResultSize() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CubeHBaseRPC.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) { final List<Pair<byte[], byte[]>> selectedColumns = makeHBaseColumns(selectedColBlocks); LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid); byte[] start = encoder.createBuf(); byte[] end = encoder.createBuf(); encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE); encoder.encode(pkStart, pkStart.getInfo().getPrimaryKey(), start); encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE); encoder.encode(pkEnd, pkEnd.getInfo().getPrimaryKey(), end); byte[] temp = new byte[end.length + 1];//append extra 0 to the end key to make it inclusive while scanning System.arraycopy(end, 0, temp, 0, end.length); end = temp; List<Pair<byte[], byte[]>> hbaseFuzzyKeys = translateFuzzyKeys(fuzzyKeys); KylinConfig config = cubeSeg.getCubeDesc().getConfig(); int hbaseCaching = config.getHBaseScanCacheRows(); int hbaseMaxResultSize = config.getHBaseScanMaxResultSize(); // if (isMemoryHungry(selectedColBlocks)) // hbaseCaching /= 10; return new RawScan(start, end, selectedColumns, hbaseFuzzyKeys, hbaseCaching, hbaseMaxResultSize); }
Example 2
Source File: CubeHBaseRPC.java From kylin with Apache License 2.0 | 6 votes |
private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) { final List<Pair<byte[], byte[]>> selectedColumns = makeHBaseColumns(selectedColBlocks); LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid); byte[] start = encoder.createBuf(); byte[] end = encoder.createBuf(); encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE); encoder.encode(pkStart, pkStart.getInfo().getPrimaryKey(), start); encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE); encoder.encode(pkEnd, pkEnd.getInfo().getPrimaryKey(), end); byte[] temp = new byte[end.length + 1];//append extra 0 to the end key to make it inclusive while scanning System.arraycopy(end, 0, temp, 0, end.length); end = temp; List<Pair<byte[], byte[]>> hbaseFuzzyKeys = translateFuzzyKeys(fuzzyKeys); KylinConfig config = cubeSeg.getCubeDesc().getConfig(); int hbaseCaching = config.getHBaseScanCacheRows(); int hbaseMaxResultSize = config.getHBaseScanMaxResultSize(); // if (isMemoryHungry(selectedColBlocks)) // hbaseCaching /= 10; return new RawScan(start, end, selectedColumns, hbaseFuzzyKeys, hbaseCaching, hbaseMaxResultSize); }
Example 3
Source File: CubeVisitServiceTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
private static RawScan mockFullScan(GTInfo gtInfo, KylinConfig kylinConfig) { final List<Pair<byte[], byte[]>> selectedColumns = Lists.newArrayList(); selectedColumns.add(new Pair<>(FAM, COL_M)); int headerLength = RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN; int bodyLength = 0; ImmutableBitSet primaryKey = gtInfo.getPrimaryKey(); for (int i = 0; i < primaryKey.trueBitCount(); i++) { bodyLength += gtInfo.getCodeSystem().getDimEnc(primaryKey.trueBitAt(i)).getLengthOfEncoding(); } //Mock start key byte[] start = new byte[headerLength + bodyLength]; BytesUtil.writeShort((short) 0, start, 0, RowConstants.ROWKEY_SHARDID_LEN); System.arraycopy(Bytes.toBytes(baseCuboid), 0, start, RowConstants.ROWKEY_SHARDID_LEN, RowConstants.ROWKEY_CUBOIDID_LEN); //Mock end key byte[] end = new byte[headerLength + bodyLength + 1]; for (int i = 0; i < end.length - 1; i++) { end[i] = RowConstants.ROWKEY_UPPER_BYTE; } BytesUtil.writeShort((short) 0, end, 0, RowConstants.ROWKEY_SHARDID_LEN); System.arraycopy(Bytes.toBytes(baseCuboid), 0, end, RowConstants.ROWKEY_SHARDID_LEN, RowConstants.ROWKEY_CUBOIDID_LEN); //Mock fuzzy key List<Pair<byte[], byte[]>> fuzzyKeys = Collections.emptyList(); return new RawScan(start, end, selectedColumns, fuzzyKeys, kylinConfig.getHBaseScanCacheRows(), kylinConfig.getHBaseScanMaxResultSize()); }
Example 4
Source File: CubeVisitServiceTest.java From kylin with Apache License 2.0 | 5 votes |
private static RawScan mockFullScan(GTInfo gtInfo, KylinConfig kylinConfig, List<Pair<byte[], byte[]>> selectedColumns) { int headerLength = RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN; int bodyLength = 0; ImmutableBitSet primaryKey = gtInfo.getPrimaryKey(); for (int i = 0; i < primaryKey.trueBitCount(); i++) { bodyLength += gtInfo.getCodeSystem().getDimEnc(primaryKey.trueBitAt(i)).getLengthOfEncoding(); } //Mock start key byte[] start = new byte[headerLength + bodyLength]; BytesUtil.writeShort((short) 0, start, 0, RowConstants.ROWKEY_SHARDID_LEN); System.arraycopy(Bytes.toBytes(baseCuboid), 0, start, RowConstants.ROWKEY_SHARDID_LEN, RowConstants.ROWKEY_CUBOIDID_LEN); //Mock end key byte[] end = new byte[headerLength + bodyLength + 1]; for (int i = 0; i < end.length - 1; i++) { end[i] = RowConstants.ROWKEY_UPPER_BYTE; } BytesUtil.writeShort((short) 0, end, 0, RowConstants.ROWKEY_SHARDID_LEN); System.arraycopy(Bytes.toBytes(baseCuboid), 0, end, RowConstants.ROWKEY_SHARDID_LEN, RowConstants.ROWKEY_CUBOIDID_LEN); //Mock fuzzy key List<Pair<byte[], byte[]>> fuzzyKeys = Collections.emptyList(); return new RawScan(start, end, selectedColumns, fuzzyKeys, kylinConfig.getHBaseScanCacheRows(), kylinConfig.getHBaseScanMaxResultSize()); }