Java Code Examples for org.apache.kylin.common.KylinConfig#getHBaseScanMaxResultSize()

The following examples show how to use org.apache.kylin.common.KylinConfig#getHBaseScanMaxResultSize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CubeHBaseRPC.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) {
    final List<Pair<byte[], byte[]>> selectedColumns = makeHBaseColumns(selectedColBlocks);

    LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid);
    byte[] start = encoder.createBuf();
    byte[] end = encoder.createBuf();

    encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE);
    encoder.encode(pkStart, pkStart.getInfo().getPrimaryKey(), start);

    encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE);
    encoder.encode(pkEnd, pkEnd.getInfo().getPrimaryKey(), end);
    byte[] temp = new byte[end.length + 1];//append extra 0 to the end key to make it inclusive while scanning
    System.arraycopy(end, 0, temp, 0, end.length);
    end = temp;

    List<Pair<byte[], byte[]>> hbaseFuzzyKeys = translateFuzzyKeys(fuzzyKeys);

    KylinConfig config = cubeSeg.getCubeDesc().getConfig();
    int hbaseCaching = config.getHBaseScanCacheRows();
    int hbaseMaxResultSize = config.getHBaseScanMaxResultSize();
    //        if (isMemoryHungry(selectedColBlocks))
    //            hbaseCaching /= 10;

    return new RawScan(start, end, selectedColumns, hbaseFuzzyKeys, hbaseCaching, hbaseMaxResultSize);
}
Example 2
Source File: CubeHBaseRPC.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private RawScan preparedHBaseScan(GTRecord pkStart, GTRecord pkEnd, List<GTRecord> fuzzyKeys, ImmutableBitSet selectedColBlocks) {
    final List<Pair<byte[], byte[]>> selectedColumns = makeHBaseColumns(selectedColBlocks);

    LazyRowKeyEncoder encoder = new LazyRowKeyEncoder(cubeSeg, cuboid);
    byte[] start = encoder.createBuf();
    byte[] end = encoder.createBuf();

    encoder.setBlankByte(RowConstants.ROWKEY_LOWER_BYTE);
    encoder.encode(pkStart, pkStart.getInfo().getPrimaryKey(), start);

    encoder.setBlankByte(RowConstants.ROWKEY_UPPER_BYTE);
    encoder.encode(pkEnd, pkEnd.getInfo().getPrimaryKey(), end);
    byte[] temp = new byte[end.length + 1];//append extra 0 to the end key to make it inclusive while scanning
    System.arraycopy(end, 0, temp, 0, end.length);
    end = temp;

    List<Pair<byte[], byte[]>> hbaseFuzzyKeys = translateFuzzyKeys(fuzzyKeys);

    KylinConfig config = cubeSeg.getCubeDesc().getConfig();
    int hbaseCaching = config.getHBaseScanCacheRows();
    int hbaseMaxResultSize = config.getHBaseScanMaxResultSize();
    //        if (isMemoryHungry(selectedColBlocks))
    //            hbaseCaching /= 10;

    return new RawScan(start, end, selectedColumns, hbaseFuzzyKeys, hbaseCaching, hbaseMaxResultSize);
}
Example 3
Source File: CubeVisitServiceTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private static RawScan mockFullScan(GTInfo gtInfo, KylinConfig kylinConfig) {
    final List<Pair<byte[], byte[]>> selectedColumns = Lists.newArrayList();
    selectedColumns.add(new Pair<>(FAM, COL_M));

    int headerLength = RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN;
    int bodyLength = 0;
    ImmutableBitSet primaryKey = gtInfo.getPrimaryKey();
    for (int i = 0; i < primaryKey.trueBitCount(); i++) {
        bodyLength += gtInfo.getCodeSystem().getDimEnc(primaryKey.trueBitAt(i)).getLengthOfEncoding();
    }
    //Mock start key
    byte[] start = new byte[headerLength + bodyLength];
    BytesUtil.writeShort((short) 0, start, 0, RowConstants.ROWKEY_SHARDID_LEN);
    System.arraycopy(Bytes.toBytes(baseCuboid), 0, start, RowConstants.ROWKEY_SHARDID_LEN,
            RowConstants.ROWKEY_CUBOIDID_LEN);

    //Mock end key
    byte[] end = new byte[headerLength + bodyLength + 1];
    for (int i = 0; i < end.length - 1; i++) {
        end[i] = RowConstants.ROWKEY_UPPER_BYTE;
    }
    BytesUtil.writeShort((short) 0, end, 0, RowConstants.ROWKEY_SHARDID_LEN);
    System.arraycopy(Bytes.toBytes(baseCuboid), 0, end, RowConstants.ROWKEY_SHARDID_LEN,
            RowConstants.ROWKEY_CUBOIDID_LEN);

    //Mock fuzzy key
    List<Pair<byte[], byte[]>> fuzzyKeys = Collections.emptyList();

    return new RawScan(start, end, selectedColumns, fuzzyKeys, kylinConfig.getHBaseScanCacheRows(),
            kylinConfig.getHBaseScanMaxResultSize());
}
Example 4
Source File: CubeVisitServiceTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private static RawScan mockFullScan(GTInfo gtInfo, KylinConfig kylinConfig,
        List<Pair<byte[], byte[]>> selectedColumns) {

    int headerLength = RowConstants.ROWKEY_SHARD_AND_CUBOID_LEN;
    int bodyLength = 0;
    ImmutableBitSet primaryKey = gtInfo.getPrimaryKey();
    for (int i = 0; i < primaryKey.trueBitCount(); i++) {
        bodyLength += gtInfo.getCodeSystem().getDimEnc(primaryKey.trueBitAt(i)).getLengthOfEncoding();
    }
    //Mock start key
    byte[] start = new byte[headerLength + bodyLength];
    BytesUtil.writeShort((short) 0, start, 0, RowConstants.ROWKEY_SHARDID_LEN);
    System.arraycopy(Bytes.toBytes(baseCuboid), 0, start, RowConstants.ROWKEY_SHARDID_LEN,
            RowConstants.ROWKEY_CUBOIDID_LEN);

    //Mock end key
    byte[] end = new byte[headerLength + bodyLength + 1];
    for (int i = 0; i < end.length - 1; i++) {
        end[i] = RowConstants.ROWKEY_UPPER_BYTE;
    }
    BytesUtil.writeShort((short) 0, end, 0, RowConstants.ROWKEY_SHARDID_LEN);
    System.arraycopy(Bytes.toBytes(baseCuboid), 0, end, RowConstants.ROWKEY_SHARDID_LEN,
            RowConstants.ROWKEY_CUBOIDID_LEN);

    //Mock fuzzy key
    List<Pair<byte[], byte[]>> fuzzyKeys = Collections.emptyList();

    return new RawScan(start, end, selectedColumns, fuzzyKeys, kylinConfig.getHBaseScanCacheRows(),
            kylinConfig.getHBaseScanMaxResultSize());
}