Java Code Examples for org.apache.kylin.cube.model.HBaseColumnDesc

The following examples show how to use org.apache.kylin.cube.model.HBaseColumnDesc. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: kylin-on-parquet-v2   Source File: KeyValueCreator.java    License: Apache License 2.0 6 votes vote down vote up
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

        cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
        qBytes = Bytes.toBytes(colDesc.getQualifier());
        timestamp = 0; // use 0 for timestamp

        refIndex = colDesc.getMeasureIndex();
        refMeasures = colDesc.getMeasures();

        codec = new BufferedMeasureCodec(refMeasures);
        colValues = new Object[refMeasures.length];

        isFullCopy = true;
        List<MeasureDesc> measures = cubeDesc.getMeasures();
        for (int i = 0; i < measures.size(); i++) {
            if (refIndex.length <= i || refIndex[i] != i)
                isFullCopy = false;
        }
    }
 
Example 2
Source Project: kylin-on-parquet-v2   Source File: CubeHFileMapper.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void doSetup(Context context) throws IOException {
    super.bindCurrentConfiguration(context.getConfiguration());
    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    CubeManager cubeMgr = CubeManager.getInstance(config);
    cubeDesc = cubeMgr.getCube(cubeName).getDescriptor();

    inputCodec = new MeasureCodec(cubeDesc.getMeasures());
    inputMeasures = new Object[cubeDesc.getMeasures().size()];
    keyValueCreators = Lists.newArrayList();

    for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
            keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
        }
    }
}
 
Example 3
Source Project: kylin-on-parquet-v2   Source File: CubeHBaseRPC.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * prune untouched hbase columns
 */
protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) {
    List<Pair<byte[], byte[]>> result = Lists.newArrayList();

    int colBlkIndex = 1;
    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        byte[] byteFamily = Bytes.toBytes(familyDesc.getName());
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier());
                result.add(Pair.newPair(byteFamily, byteQualifier));
            }
            colBlkIndex++;
        }
    }

    return result;
}
 
Example 4
Source Project: kylin-on-parquet-v2   Source File: CubeHBaseRPC.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * for each selected hbase column, it might contain values of multiple GT columns.
 * The mapping should be passed down to storage
 */
protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) {

    List<List<Integer>> ret = Lists.newArrayList();

    int colBlkIndex = 1;
    int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount();

    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                int[] metricIndexes = hbaseColDesc.getMeasureIndex();
                Integer[] gtIndexes = new Integer[metricIndexes.length];
                for (int i = 0; i < gtIndexes.length; i++) {
                    gtIndexes[i] = metricIndexes[i] + metricOffset;
                }
                ret.add(Arrays.asList(gtIndexes));
            }
            colBlkIndex++;
        }
    }

    Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1);
    return ret;
}
 
Example 5
Source Project: kylin   Source File: KeyValueCreator.java    License: Apache License 2.0 6 votes vote down vote up
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

        cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
        qBytes = Bytes.toBytes(colDesc.getQualifier());
        timestamp = 0; // use 0 for timestamp

        refIndex = colDesc.getMeasureIndex();
        refMeasures = colDesc.getMeasures();

        codec = new BufferedMeasureCodec(refMeasures);
        colValues = new Object[refMeasures.length];

        isFullCopy = true;
        List<MeasureDesc> measures = cubeDesc.getMeasures();
        for (int i = 0; i < measures.size(); i++) {
            if (refIndex.length <= i || refIndex[i] != i)
                isFullCopy = false;
        }
    }
 
Example 6
Source Project: kylin   Source File: CubeHFileMapper.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void doSetup(Context context) throws IOException {
    super.bindCurrentConfiguration(context.getConfiguration());
    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    CubeManager cubeMgr = CubeManager.getInstance(config);
    cubeDesc = cubeMgr.getCube(cubeName).getDescriptor();

    inputCodec = new MeasureCodec(cubeDesc.getMeasures());
    inputMeasures = new Object[cubeDesc.getMeasures().size()];
    keyValueCreators = Lists.newArrayList();

    for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
            keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
        }
    }
}
 
Example 7
Source Project: kylin   Source File: CubeHBaseRPC.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * prune untouched hbase columns
 */
protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) {
    List<Pair<byte[], byte[]>> result = Lists.newArrayList();

    int colBlkIndex = 1;
    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        byte[] byteFamily = Bytes.toBytes(familyDesc.getName());
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier());
                result.add(Pair.newPair(byteFamily, byteQualifier));
            }
            colBlkIndex++;
        }
    }

    return result;
}
 
Example 8
Source Project: kylin   Source File: CubeHBaseRPC.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * for each selected hbase column, it might contain values of multiple GT columns.
 * The mapping should be passed down to storage
 */
protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) {

    List<List<Integer>> ret = Lists.newArrayList();

    int colBlkIndex = 1;
    int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount();

    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                int[] metricIndexes = hbaseColDesc.getMeasureIndex();
                Integer[] gtIndexes = new Integer[metricIndexes.length];
                for (int i = 0; i < gtIndexes.length; i++) {
                    gtIndexes[i] = metricIndexes[i] + metricOffset;
                }
                ret.add(Arrays.asList(gtIndexes));
            }
            colBlkIndex++;
        }
    }

    Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1);
    return ret;
}
 
Example 9
Source Project: Kylin   Source File: CubeHFileMapper.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void setup(Context context) throws IOException {
    super.publishConfiguration(context.getConfiguration());
    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());

    CubeManager cubeMgr = CubeManager.getInstance(config);
    cubeDesc = cubeMgr.getCube(cubeName).getDescriptor();

    inputCodec = new MeasureCodec(cubeDesc.getMeasures());
    inputMeasures = new Object[cubeDesc.getMeasures().size()];
    keyValueCreators = Lists.newArrayList();

    for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
            keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
        }
    }
}
 
Example 10
Source Project: Kylin   Source File: CubeHFileMapper.java    License: Apache License 2.0 6 votes vote down vote up
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

            cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
            qBytes = Bytes.toBytes(colDesc.getQualifier());
            timestamp = System.currentTimeMillis();

            List<MeasureDesc> measures = cubeDesc.getMeasures();
            String[] measureNames = getMeasureNames(cubeDesc);
            String[] refs = colDesc.getMeasureRefs();

            refIndex = new int[refs.length];
            refMeasures = new MeasureDesc[refs.length];
            for (int i = 0; i < refs.length; i++) {
                refIndex[i] = indexOf(measureNames, refs[i]);
                refMeasures[i] = measures.get(refIndex[i]);
            }

            codec = new MeasureCodec(refMeasures);
            colValues = new Object[refs.length];

            isFullCopy = true;
            for (int i = 0; i < measures.size(); i++) {
                if (refIndex.length <= i || refIndex[i] != i)
                    isFullCopy = false;
            }
        }
 
Example 11
Source Project: Kylin   Source File: CubeDesc.java    License: Apache License 2.0 6 votes vote down vote up
private void initMeasureReferenceToColumnFamily() {
    if (measures == null || measures.size() == 0)
        return;

    Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>();
    for (MeasureDesc m : measures)
        measureCache.put(m.getName(), m);

    for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc c : cf.getColumns()) {
            MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length];
            for (int i = 0; i < c.getMeasureRefs().length; i++) {
                measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]);
            }
            c.setMeasures(measureDescs);
            c.setColumnFamilyName(cf.getName());
        }
    }
}
 
Example 12
Source Project: kylin-on-parquet-v2   Source File: RowValueDecoder.java    License: Apache License 2.0 5 votes vote down vote up
public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.hbaseColumnFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
    this.hbaseColumnQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
    this.projectionIndex = new BitSet();
    this.measures = hbaseColumn.getMeasures();
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}
 
Example 13
Source Project: kylin-on-parquet-v2   Source File: RowValueDecoderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    Long count = new Long(2);
    Long item_count = new Long(100);
    ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count });

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasure(aggrFunc);
        rowValueDecoder.setProjectIndex(index);
    }

    rowValueDecoder.decodeAndConvertJavaObj(valueBytes);
    Object[] measureValues = rowValueDecoder.getValues();
    //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer
    assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues));
}
 
Example 14
Source Project: kylin-on-parquet-v2   Source File: RowValueDecoderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalArgumentException.class)
public void testError() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    Long item_count = new Long(100);
    codec.encode(new Object[] { sum, min, max, count, item_count });

}
 
Example 15
Source Project: kylin-on-parquet-v2   Source File: CubeDescTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBadInit14() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    String measureInfoForTransCnt = measureForTransCnt.toString();
    thrown.expectMessage(
            "measure (" + measureInfoForTransCnt + ") does not exist in column family, or measure duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}
 
Example 16
Source Project: kylin-on-parquet-v2   Source File: CubeDescTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBadInit15() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(
            new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TRANS_CNT", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}
 
Example 17
Source Project: kylin-on-parquet-v2   Source File: CubeController.java    License: Apache License 2.0 5 votes vote down vote up
private void validateColumnFamily(CubeDesc cubeDesc) {
    Set<String> columnFamilyMetricsSet = Sets.newHashSet();
    for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) {
            for (String columnName : hBaseColumnDesc.getMeasureRefs()) {
                columnFamilyMetricsSet.add(columnName);
            }
        }
    }
    for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
        if (!columnFamilyMetricsSet.contains(measureDesc.getName())) {
            throw new BadRequestException("column family lack measure:" + measureDesc.getName());
        }
    }
    if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) {
        throw new BadRequestException(
                "the number of input measure and the number of measure defined in cubedesc are not consistent");
    }

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        Object[] encodingConf = DimensionEncoding.parseEncodingConf(rowKeyColDesc.getEncoding());
        String encodingName = (String) encodingConf[0];
        String[] encodingArgs = (String[]) encodingConf[1];

        if (!DimensionEncodingFactory.isValidEncoding(encodingName, encodingArgs,
                rowKeyColDesc.getEncodingVersion())) {
            throw new BadRequestException("Illegal row key column desc: " + rowKeyColDesc);
        }
    }
}
 
Example 18
Source Project: kylin   Source File: RowValueDecoder.java    License: Apache License 2.0 5 votes vote down vote up
public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.hbaseColumnFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
    this.hbaseColumnQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
    this.projectionIndex = new BitSet();
    this.measures = hbaseColumn.getMeasures();
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}
 
Example 19
Source Project: kylin   Source File: RowValueDecoderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    Long count = new Long(2);
    Long item_count = new Long(100);
    ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count });

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasure(aggrFunc);
        rowValueDecoder.setProjectIndex(index);
    }

    rowValueDecoder.decodeAndConvertJavaObj(valueBytes);
    Object[] measureValues = rowValueDecoder.getValues();
    //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer
    assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues));
}
 
Example 20
Source Project: kylin   Source File: RowValueDecoderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalArgumentException.class)
public void testError() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    Long item_count = new Long(100);
    codec.encode(new Object[] { sum, min, max, count, item_count });

}
 
Example 21
Source Project: kylin   Source File: CubeDescTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBadInit14() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    String measureInfoForTransCnt = measureForTransCnt.toString();
    thrown.expectMessage(
            "measure (" + measureInfoForTransCnt + ") does not exist in column family, or measure duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}
 
Example 22
Source Project: kylin   Source File: CubeDescTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBadInit15() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(
            new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TRANS_CNT", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}
 
Example 23
Source Project: kylin   Source File: CubeController.java    License: Apache License 2.0 5 votes vote down vote up
private void validateColumnFamily(CubeDesc cubeDesc) {
    Set<String> columnFamilyMetricsSet = Sets.newHashSet();
    for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) {
            for (String columnName : hBaseColumnDesc.getMeasureRefs()) {
                columnFamilyMetricsSet.add(columnName);
            }
        }
    }
    for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
        if (!columnFamilyMetricsSet.contains(measureDesc.getName())) {
            throw new BadRequestException("column family lack measure:" + measureDesc.getName());
        }
    }
    if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) {
        throw new BadRequestException(
                "the number of input measure and the number of measure defined in cubedesc are not consistent");
    }

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        Object[] encodingConf = DimensionEncoding.parseEncodingConf(rowKeyColDesc.getEncoding());
        String encodingName = (String) encodingConf[0];
        String[] encodingArgs = (String[]) encodingConf[1];

        if (!DimensionEncodingFactory.isValidEncoding(encodingName, encodingArgs,
                rowKeyColDesc.getEncodingVersion())) {
            throw new BadRequestException("Illegal row key column desc: " + rowKeyColDesc);
        }
    }
}
 
Example 24
Source Project: Kylin   Source File: RowValueDecoder.java    License: Apache License 2.0 5 votes vote down vote up
public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.projectionIndex = new BitSet();
    this.names = new ArrayList<String>();
    this.measures = hbaseColumn.getMeasures();
    for (MeasureDesc measure : measures) {
        this.names.add(measure.getFunction().getRewriteFieldName());
    }
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}
 
Example 25
Source Project: Kylin   Source File: RowValueDecoderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0];

    MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
    codec.encode(new Object[] { sum, min, max, count }, buf);

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasureIndex(aggrFunc);
        rowValueDecoder.setIndex(index);
    }

    rowValueDecoder.decode(valueBytes);
    List<String> measureNames = rowValueDecoder.getNames();
    Object[] measureValues = rowValueDecoder.getValues();

    assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString());
    assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues));
}
 
Example 26
Source Project: Kylin   Source File: CubeSegmentTupleIterator.java    License: Apache License 2.0 5 votes vote down vote up
private Scan buildScan(HBaseKeyRange keyRange) {
    Scan scan = new Scan();
    scan.setCaching(SCAN_CACHE);
    scan.setCacheBlocks(true);
    scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
    for (RowValueDecoder valueDecoder : this.rowValueDecoders) {
        HBaseColumnDesc hbaseColumn = valueDecoder.getHBaseColumn();
        byte[] byteFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
        byte[] byteQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
        scan.addColumn(byteFamily, byteQualifier);
    }
    scan.setStartRow(keyRange.getStartKey());
    scan.setStopRow(keyRange.getStopKey());
    return scan;
}
 
Example 27
Source Project: Kylin   Source File: CubeSegmentTupleIterator.java    License: Apache License 2.0 5 votes vote down vote up
private void translateResult(Result res, Tuple tuple) throws IOException {
    // groups
    byte[] rowkey = res.getRow();
    rowKeyDecoder.decode(rowkey);
    List<TblColRef> columns = rowKeyDecoder.getColumns();
    List<String> dimensionNames = rowKeyDecoder.getNames(context.getAliasMap());
    List<String> dimensionValues = rowKeyDecoder.getValues();
    for (int i = 0; i < dimensionNames.size(); i++) {
        TblColRef column = columns.get(i);
        if (!tuple.hasColumn(column)) {
            continue;
        }
        tuple.setDimensionValue(dimensionNames.get(i), dimensionValues.get(i));
    }

    // derived
    for (IDerivedColumnFiller filler : tupleInfo.getDerivedColumnFillers()) {
        filler.fillDerivedColumns(dimensionValues, tuple);
    }

    // aggregations
    for (RowValueDecoder rowValueDecoder : this.rowValueDecoders) {
        HBaseColumnDesc hbaseColumn = rowValueDecoder.getHBaseColumn();
        String columnFamily = hbaseColumn.getColumnFamilyName();
        String qualifier = hbaseColumn.getQualifier();
        // FIXME: avoidable bytes array creation, why not use res.getValueAsByteBuffer directly?
        byte[] valueBytes = res.getValue(Bytes.toBytes(columnFamily), Bytes.toBytes(qualifier));
        rowValueDecoder.decode(valueBytes);
        List<String> measureNames = rowValueDecoder.getNames();
        Object[] measureValues = rowValueDecoder.getValues();
        BitSet projectionIndex = rowValueDecoder.getProjectionIndex();
        for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) {
            tuple.setMeasureValue(measureNames.get(i), measureValues[i]);
        }
    }
}
 
Example 28
Source Project: Kylin   Source File: ObserverAggregators.java    License: Apache License 2.0 5 votes vote down vote up
private static HCol buildHCol(HBaseColumnDesc desc) {
    byte[] family = Bytes.toBytes(desc.getColumnFamilyName());
    byte[] qualifier = Bytes.toBytes(desc.getQualifier());
    MeasureDesc[] measures = desc.getMeasures();

    String[] funcNames = new String[measures.length];
    String[] dataTypes = new String[measures.length];

    for (int i = 0; i < measures.length; i++) {
        funcNames[i] = measures[i].getFunction().getExpression();
        dataTypes[i] = measures[i].getFunction().getReturnType();
    }

    return new HCol(family, qualifier, funcNames, dataTypes);
}
 
Example 29
Source Project: Kylin   Source File: CubeStorageEngine.java    License: Apache License 2.0 5 votes vote down vote up
private List<RowValueDecoder> translateAggregation(HBaseMappingDesc hbaseMapping, Collection<FunctionDesc> metrics, //
        StorageContext context) {
    Map<HBaseColumnDesc, RowValueDecoder> codecMap = Maps.newHashMap();
    for (FunctionDesc aggrFunc : metrics) {
        Collection<HBaseColumnDesc> hbCols = hbaseMapping.findHBaseColumnByFunction(aggrFunc);
        if (hbCols.isEmpty()) {
            throw new IllegalStateException("can't find HBaseColumnDesc for function " + aggrFunc.getFullExpression());
        }
        HBaseColumnDesc bestHBCol = null;
        int bestIndex = -1;
        for (HBaseColumnDesc hbCol : hbCols) {
            bestHBCol = hbCol;
            bestIndex = hbCol.findMeasureIndex(aggrFunc);
            MeasureDesc measure = hbCol.getMeasures()[bestIndex];
            // criteria for holistic measure: Exact Aggregation && Exact Cuboid
            if (measure.isHolisticCountDistinct() && context.isExactAggregation()) {
                logger.info("Holistic count distinct chosen for " + aggrFunc);
                break;
            }
        }

        RowValueDecoder codec = codecMap.get(bestHBCol);
        if (codec == null) {
            codec = new RowValueDecoder(bestHBCol);
            codecMap.put(bestHBCol, codec);
        }
        codec.setIndex(bestIndex);
    }
    return new ArrayList<RowValueDecoder>(codecMap.values());
}
 
Example 30
Source Project: kylin-on-parquet-v2   Source File: RowValueDecoder.java    License: Apache License 2.0 4 votes vote down vote up
public HBaseColumnDesc getHBaseColumn() {
    return hbaseColumn;
}