Java Code Examples for org.apache.kylin.cube.model.HBaseColumnDesc#getMeasures()

The following examples show how to use org.apache.kylin.cube.model.HBaseColumnDesc#getMeasures() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KeyValueCreator.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

        cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
        qBytes = Bytes.toBytes(colDesc.getQualifier());
        timestamp = 0; // use 0 for timestamp

        refIndex = colDesc.getMeasureIndex();
        refMeasures = colDesc.getMeasures();

        codec = new BufferedMeasureCodec(refMeasures);
        colValues = new Object[refMeasures.length];

        isFullCopy = true;
        List<MeasureDesc> measures = cubeDesc.getMeasures();
        for (int i = 0; i < measures.size(); i++) {
            if (refIndex.length <= i || refIndex[i] != i)
                isFullCopy = false;
        }
    }
Example 2
Source File: KeyValueCreator.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

        cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
        qBytes = Bytes.toBytes(colDesc.getQualifier());
        timestamp = 0; // use 0 for timestamp

        refIndex = colDesc.getMeasureIndex();
        refMeasures = colDesc.getMeasures();

        codec = new BufferedMeasureCodec(refMeasures);
        colValues = new Object[refMeasures.length];

        isFullCopy = true;
        List<MeasureDesc> measures = cubeDesc.getMeasures();
        for (int i = 0; i < measures.size(); i++) {
            if (refIndex.length <= i || refIndex[i] != i)
                isFullCopy = false;
        }
    }
Example 3
Source File: RowValueDecoder.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.hbaseColumnFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
    this.hbaseColumnQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
    this.projectionIndex = new BitSet();
    this.measures = hbaseColumn.getMeasures();
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}
Example 4
Source File: RowValueDecoderTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    Long count = new Long(2);
    Long item_count = new Long(100);
    ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count });

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasure(aggrFunc);
        rowValueDecoder.setProjectIndex(index);
    }

    rowValueDecoder.decodeAndConvertJavaObj(valueBytes);
    Object[] measureValues = rowValueDecoder.getValues();
    //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer
    assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues));
}
Example 5
Source File: RowValueDecoderTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Test(expected = IllegalArgumentException.class)
public void testError() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    Long item_count = new Long(100);
    codec.encode(new Object[] { sum, min, max, count, item_count });

}
Example 6
Source File: RowValueDecoder.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.hbaseColumnFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
    this.hbaseColumnQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
    this.projectionIndex = new BitSet();
    this.measures = hbaseColumn.getMeasures();
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}
Example 7
Source File: RowValueDecoderTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    Long count = new Long(2);
    Long item_count = new Long(100);
    ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count });

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasure(aggrFunc);
        rowValueDecoder.setProjectIndex(index);
    }

    rowValueDecoder.decodeAndConvertJavaObj(valueBytes);
    Object[] measureValues = rowValueDecoder.getValues();
    //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer
    assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues));
}
Example 8
Source File: RowValueDecoderTest.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Test(expected = IllegalArgumentException.class)
public void testError() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    Long item_count = new Long(100);
    codec.encode(new Object[] { sum, min, max, count, item_count });

}
Example 9
Source File: RowValueDecoder.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.projectionIndex = new BitSet();
    this.names = new ArrayList<String>();
    this.measures = hbaseColumn.getMeasures();
    for (MeasureDesc measure : measures) {
        this.names.add(measure.getFunction().getRewriteFieldName());
    }
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}
Example 10
Source File: RowValueDecoderTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0];

    MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
    codec.encode(new Object[] { sum, min, max, count }, buf);

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasureIndex(aggrFunc);
        rowValueDecoder.setIndex(index);
    }

    rowValueDecoder.decode(valueBytes);
    List<String> measureNames = rowValueDecoder.getNames();
    Object[] measureValues = rowValueDecoder.getValues();

    assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString());
    assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues));
}
Example 11
Source File: ObserverAggregators.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private static HCol buildHCol(HBaseColumnDesc desc) {
    byte[] family = Bytes.toBytes(desc.getColumnFamilyName());
    byte[] qualifier = Bytes.toBytes(desc.getQualifier());
    MeasureDesc[] measures = desc.getMeasures();

    String[] funcNames = new String[measures.length];
    String[] dataTypes = new String[measures.length];

    for (int i = 0; i < measures.length; i++) {
        funcNames[i] = measures[i].getFunction().getExpression();
        dataTypes[i] = measures[i].getFunction().getReturnType();
    }

    return new HCol(family, qualifier, funcNames, dataTypes);
}
Example 12
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
private List<RowValueDecoder> translateAggregation(HBaseMappingDesc hbaseMapping, Collection<FunctionDesc> metrics, //
        StorageContext context) {
    Map<HBaseColumnDesc, RowValueDecoder> codecMap = Maps.newHashMap();
    for (FunctionDesc aggrFunc : metrics) {
        Collection<HBaseColumnDesc> hbCols = hbaseMapping.findHBaseColumnByFunction(aggrFunc);
        if (hbCols.isEmpty()) {
            throw new IllegalStateException("can't find HBaseColumnDesc for function " + aggrFunc.getFullExpression());
        }
        HBaseColumnDesc bestHBCol = null;
        int bestIndex = -1;
        for (HBaseColumnDesc hbCol : hbCols) {
            bestHBCol = hbCol;
            bestIndex = hbCol.findMeasureIndex(aggrFunc);
            MeasureDesc measure = hbCol.getMeasures()[bestIndex];
            // criteria for holistic measure: Exact Aggregation && Exact Cuboid
            if (measure.isHolisticCountDistinct() && context.isExactAggregation()) {
                logger.info("Holistic count distinct chosen for " + aggrFunc);
                break;
            }
        }

        RowValueDecoder codec = codecMap.get(bestHBCol);
        if (codec == null) {
            codec = new RowValueDecoder(bestHBCol);
            codecMap.put(bestHBCol, codec);
        }
        codec.setIndex(bestIndex);
    }
    return new ArrayList<RowValueDecoder>(codecMap.values());
}