org.apache.kylin.cube.model.HBaseColumnDesc Java Examples

The following examples show how to use org.apache.kylin.cube.model.HBaseColumnDesc. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CubeHFileMapper.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Override
protected void doSetup(Context context) throws IOException {
    super.bindCurrentConfiguration(context.getConfiguration());
    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    CubeManager cubeMgr = CubeManager.getInstance(config);
    cubeDesc = cubeMgr.getCube(cubeName).getDescriptor();

    inputCodec = new MeasureCodec(cubeDesc.getMeasures());
    inputMeasures = new Object[cubeDesc.getMeasures().size()];
    keyValueCreators = Lists.newArrayList();

    for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
            keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
        }
    }
}
 
Example #2
Source File: CubeDesc.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private void initMeasureReferenceToColumnFamily() {
    if (measures == null || measures.size() == 0)
        return;

    Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>();
    for (MeasureDesc m : measures)
        measureCache.put(m.getName(), m);

    for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc c : cf.getColumns()) {
            MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length];
            for (int i = 0; i < c.getMeasureRefs().length; i++) {
                measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]);
            }
            c.setMeasures(measureDescs);
            c.setColumnFamilyName(cf.getName());
        }
    }
}
 
Example #3
Source File: CubeHFileMapper.java    From Kylin with Apache License 2.0 6 votes vote down vote up
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

            cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
            qBytes = Bytes.toBytes(colDesc.getQualifier());
            timestamp = System.currentTimeMillis();

            List<MeasureDesc> measures = cubeDesc.getMeasures();
            String[] measureNames = getMeasureNames(cubeDesc);
            String[] refs = colDesc.getMeasureRefs();

            refIndex = new int[refs.length];
            refMeasures = new MeasureDesc[refs.length];
            for (int i = 0; i < refs.length; i++) {
                refIndex[i] = indexOf(measureNames, refs[i]);
                refMeasures[i] = measures.get(refIndex[i]);
            }

            codec = new MeasureCodec(refMeasures);
            colValues = new Object[refs.length];

            isFullCopy = true;
            for (int i = 0; i < measures.size(); i++) {
                if (refIndex.length <= i || refIndex[i] != i)
                    isFullCopy = false;
            }
        }
 
Example #4
Source File: CubeHFileMapper.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Override
protected void setup(Context context) throws IOException {
    super.publishConfiguration(context.getConfiguration());
    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(context.getConfiguration());

    CubeManager cubeMgr = CubeManager.getInstance(config);
    cubeDesc = cubeMgr.getCube(cubeName).getDescriptor();

    inputCodec = new MeasureCodec(cubeDesc.getMeasures());
    inputMeasures = new Object[cubeDesc.getMeasures().size()];
    keyValueCreators = Lists.newArrayList();

    for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
            keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
        }
    }
}
 
Example #5
Source File: CubeHBaseRPC.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * for each selected hbase column, it might contain values of multiple GT columns.
 * The mapping should be passed down to storage
 */
protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) {

    List<List<Integer>> ret = Lists.newArrayList();

    int colBlkIndex = 1;
    int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount();

    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                int[] metricIndexes = hbaseColDesc.getMeasureIndex();
                Integer[] gtIndexes = new Integer[metricIndexes.length];
                for (int i = 0; i < gtIndexes.length; i++) {
                    gtIndexes[i] = metricIndexes[i] + metricOffset;
                }
                ret.add(Arrays.asList(gtIndexes));
            }
            colBlkIndex++;
        }
    }

    Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1);
    return ret;
}
 
Example #6
Source File: CubeHBaseRPC.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * prune untouched hbase columns
 */
protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) {
    List<Pair<byte[], byte[]>> result = Lists.newArrayList();

    int colBlkIndex = 1;
    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        byte[] byteFamily = Bytes.toBytes(familyDesc.getName());
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier());
                result.add(Pair.newPair(byteFamily, byteQualifier));
            }
            colBlkIndex++;
        }
    }

    return result;
}
 
Example #7
Source File: CubeHFileMapper.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
protected void doSetup(Context context) throws IOException {
    super.bindCurrentConfiguration(context.getConfiguration());
    cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    CubeManager cubeMgr = CubeManager.getInstance(config);
    cubeDesc = cubeMgr.getCube(cubeName).getDescriptor();

    inputCodec = new MeasureCodec(cubeDesc.getMeasures());
    inputMeasures = new Object[cubeDesc.getMeasures().size()];
    keyValueCreators = Lists.newArrayList();

    for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc colDesc : cfDesc.getColumns()) {
            keyValueCreators.add(new KeyValueCreator(cubeDesc, colDesc));
        }
    }
}
 
Example #8
Source File: KeyValueCreator.java    From kylin with Apache License 2.0 6 votes vote down vote up
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

        cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
        qBytes = Bytes.toBytes(colDesc.getQualifier());
        timestamp = 0; // use 0 for timestamp

        refIndex = colDesc.getMeasureIndex();
        refMeasures = colDesc.getMeasures();

        codec = new BufferedMeasureCodec(refMeasures);
        colValues = new Object[refMeasures.length];

        isFullCopy = true;
        List<MeasureDesc> measures = cubeDesc.getMeasures();
        for (int i = 0; i < measures.size(); i++) {
            if (refIndex.length <= i || refIndex[i] != i)
                isFullCopy = false;
        }
    }
 
Example #9
Source File: CubeHBaseRPC.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
/**
 * for each selected hbase column, it might contain values of multiple GT columns.
 * The mapping should be passed down to storage
 */
protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) {

    List<List<Integer>> ret = Lists.newArrayList();

    int colBlkIndex = 1;
    int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount();

    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                int[] metricIndexes = hbaseColDesc.getMeasureIndex();
                Integer[] gtIndexes = new Integer[metricIndexes.length];
                for (int i = 0; i < gtIndexes.length; i++) {
                    gtIndexes[i] = metricIndexes[i] + metricOffset;
                }
                ret.add(Arrays.asList(gtIndexes));
            }
            colBlkIndex++;
        }
    }

    Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1);
    return ret;
}
 
Example #10
Source File: CubeHBaseRPC.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
/**
 * prune untouched hbase columns
 */
protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) {
    List<Pair<byte[], byte[]>> result = Lists.newArrayList();

    int colBlkIndex = 1;
    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        byte[] byteFamily = Bytes.toBytes(familyDesc.getName());
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier());
                result.add(Pair.newPair(byteFamily, byteQualifier));
            }
            colBlkIndex++;
        }
    }

    return result;
}
 
Example #11
Source File: KeyValueCreator.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

        cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
        qBytes = Bytes.toBytes(colDesc.getQualifier());
        timestamp = 0; // use 0 for timestamp

        refIndex = colDesc.getMeasureIndex();
        refMeasures = colDesc.getMeasures();

        codec = new BufferedMeasureCodec(refMeasures);
        colValues = new Object[refMeasures.length];

        isFullCopy = true;
        List<MeasureDesc> measures = cubeDesc.getMeasures();
        for (int i = 0; i < measures.size(); i++) {
            if (refIndex.length <= i || refIndex[i] != i)
                isFullCopy = false;
        }
    }
 
Example #12
Source File: CubeDescTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testBadInit15() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(
            new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TRANS_CNT", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}
 
Example #13
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private List<RowValueDecoder> translateAggregation(HBaseMappingDesc hbaseMapping, Collection<FunctionDesc> metrics, //
        StorageContext context) {
    Map<HBaseColumnDesc, RowValueDecoder> codecMap = Maps.newHashMap();
    for (FunctionDesc aggrFunc : metrics) {
        Collection<HBaseColumnDesc> hbCols = hbaseMapping.findHBaseColumnByFunction(aggrFunc);
        if (hbCols.isEmpty()) {
            throw new IllegalStateException("can't find HBaseColumnDesc for function " + aggrFunc.getFullExpression());
        }
        HBaseColumnDesc bestHBCol = null;
        int bestIndex = -1;
        for (HBaseColumnDesc hbCol : hbCols) {
            bestHBCol = hbCol;
            bestIndex = hbCol.findMeasureIndex(aggrFunc);
            MeasureDesc measure = hbCol.getMeasures()[bestIndex];
            // criteria for holistic measure: Exact Aggregation && Exact Cuboid
            if (measure.isHolisticCountDistinct() && context.isExactAggregation()) {
                logger.info("Holistic count distinct chosen for " + aggrFunc);
                break;
            }
        }

        RowValueDecoder codec = codecMap.get(bestHBCol);
        if (codec == null) {
            codec = new RowValueDecoder(bestHBCol);
            codecMap.put(bestHBCol, codec);
        }
        codec.setIndex(bestIndex);
    }
    return new ArrayList<RowValueDecoder>(codecMap.values());
}
 
Example #14
Source File: ObserverAggregators.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private static HCol buildHCol(HBaseColumnDesc desc) {
    byte[] family = Bytes.toBytes(desc.getColumnFamilyName());
    byte[] qualifier = Bytes.toBytes(desc.getQualifier());
    MeasureDesc[] measures = desc.getMeasures();

    String[] funcNames = new String[measures.length];
    String[] dataTypes = new String[measures.length];

    for (int i = 0; i < measures.length; i++) {
        funcNames[i] = measures[i].getFunction().getExpression();
        dataTypes[i] = measures[i].getFunction().getReturnType();
    }

    return new HCol(family, qualifier, funcNames, dataTypes);
}
 
Example #15
Source File: CubeSegmentTupleIterator.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private void translateResult(Result res, Tuple tuple) throws IOException {
    // groups
    byte[] rowkey = res.getRow();
    rowKeyDecoder.decode(rowkey);
    List<TblColRef> columns = rowKeyDecoder.getColumns();
    List<String> dimensionNames = rowKeyDecoder.getNames(context.getAliasMap());
    List<String> dimensionValues = rowKeyDecoder.getValues();
    for (int i = 0; i < dimensionNames.size(); i++) {
        TblColRef column = columns.get(i);
        if (!tuple.hasColumn(column)) {
            continue;
        }
        tuple.setDimensionValue(dimensionNames.get(i), dimensionValues.get(i));
    }

    // derived
    for (IDerivedColumnFiller filler : tupleInfo.getDerivedColumnFillers()) {
        filler.fillDerivedColumns(dimensionValues, tuple);
    }

    // aggregations
    for (RowValueDecoder rowValueDecoder : this.rowValueDecoders) {
        HBaseColumnDesc hbaseColumn = rowValueDecoder.getHBaseColumn();
        String columnFamily = hbaseColumn.getColumnFamilyName();
        String qualifier = hbaseColumn.getQualifier();
        // FIXME: avoidable bytes array creation, why not use res.getValueAsByteBuffer directly?
        byte[] valueBytes = res.getValue(Bytes.toBytes(columnFamily), Bytes.toBytes(qualifier));
        rowValueDecoder.decode(valueBytes);
        List<String> measureNames = rowValueDecoder.getNames();
        Object[] measureValues = rowValueDecoder.getValues();
        BitSet projectionIndex = rowValueDecoder.getProjectionIndex();
        for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) {
            tuple.setMeasureValue(measureNames.get(i), measureValues[i]);
        }
    }
}
 
Example #16
Source File: CubeSegmentTupleIterator.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private Scan buildScan(HBaseKeyRange keyRange) {
    Scan scan = new Scan();
    scan.setCaching(SCAN_CACHE);
    scan.setCacheBlocks(true);
    scan.setAttribute(Scan.SCAN_ATTRIBUTES_METRICS_ENABLE, Bytes.toBytes(Boolean.TRUE));
    for (RowValueDecoder valueDecoder : this.rowValueDecoders) {
        HBaseColumnDesc hbaseColumn = valueDecoder.getHBaseColumn();
        byte[] byteFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
        byte[] byteQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
        scan.addColumn(byteFamily, byteQualifier);
    }
    scan.setStartRow(keyRange.getStartKey());
    scan.setStopRow(keyRange.getStopKey());
    return scan;
}
 
Example #17
Source File: RowValueDecoderTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0];

    MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
    codec.encode(new Object[] { sum, min, max, count }, buf);

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasureIndex(aggrFunc);
        rowValueDecoder.setIndex(index);
    }

    rowValueDecoder.decode(valueBytes);
    List<String> measureNames = rowValueDecoder.getNames();
    Object[] measureValues = rowValueDecoder.getValues();

    assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString());
    assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues));
}
 
Example #18
Source File: RowValueDecoder.java    From Kylin with Apache License 2.0 5 votes vote down vote up
public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.projectionIndex = new BitSet();
    this.names = new ArrayList<String>();
    this.measures = hbaseColumn.getMeasures();
    for (MeasureDesc measure : measures) {
        this.names.add(measure.getFunction().getRewriteFieldName());
    }
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}
 
Example #19
Source File: RowValueDecoder.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.hbaseColumnFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
    this.hbaseColumnQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
    this.projectionIndex = new BitSet();
    this.measures = hbaseColumn.getMeasures();
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}
 
Example #20
Source File: CubeController.java    From kylin with Apache License 2.0 5 votes vote down vote up
private void validateColumnFamily(CubeDesc cubeDesc) {
    Set<String> columnFamilyMetricsSet = Sets.newHashSet();
    for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) {
            for (String columnName : hBaseColumnDesc.getMeasureRefs()) {
                columnFamilyMetricsSet.add(columnName);
            }
        }
    }
    for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
        if (!columnFamilyMetricsSet.contains(measureDesc.getName())) {
            throw new BadRequestException("column family lack measure:" + measureDesc.getName());
        }
    }
    if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) {
        throw new BadRequestException(
                "the number of input measure and the number of measure defined in cubedesc are not consistent");
    }

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        Object[] encodingConf = DimensionEncoding.parseEncodingConf(rowKeyColDesc.getEncoding());
        String encodingName = (String) encodingConf[0];
        String[] encodingArgs = (String[]) encodingConf[1];

        if (!DimensionEncodingFactory.isValidEncoding(encodingName, encodingArgs,
                rowKeyColDesc.getEncodingVersion())) {
            throw new BadRequestException("Illegal row key column desc: " + rowKeyColDesc);
        }
    }
}
 
Example #21
Source File: CubeDescTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testBadInit14() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    String measureInfoForTransCnt = measureForTransCnt.toString();
    thrown.expectMessage(
            "measure (" + measureInfoForTransCnt + ") does not exist in column family, or measure duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}
 
Example #22
Source File: CubeDescTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testBadInit14() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    String measureInfoForTransCnt = measureForTransCnt.toString();
    thrown.expectMessage(
            "measure (" + measureInfoForTransCnt + ") does not exist in column family, or measure duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}
 
Example #23
Source File: RowValueDecoderTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalArgumentException.class)
public void testError() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    Long item_count = new Long(100);
    codec.encode(new Object[] { sum, min, max, count, item_count });

}
 
Example #24
Source File: RowValueDecoderTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    Long count = new Long(2);
    Long item_count = new Long(100);
    ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count });

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasure(aggrFunc);
        rowValueDecoder.setProjectIndex(index);
    }

    rowValueDecoder.decodeAndConvertJavaObj(valueBytes);
    Object[] measureValues = rowValueDecoder.getValues();
    //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer
    assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues));
}
 
Example #25
Source File: RowValueDecoder.java    From kylin with Apache License 2.0 5 votes vote down vote up
public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.hbaseColumnFamily = Bytes.toBytes(hbaseColumn.getColumnFamilyName());
    this.hbaseColumnQualifier = Bytes.toBytes(hbaseColumn.getQualifier());
    this.projectionIndex = new BitSet();
    this.measures = hbaseColumn.getMeasures();
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}
 
Example #26
Source File: RowValueDecoderTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    Long count = new Long(2);
    Long item_count = new Long(100);
    ByteBuffer buf = codec.encode(new Object[] { sum, min, max, count, item_count });

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasure(aggrFunc);
        rowValueDecoder.setProjectIndex(index);
    }

    rowValueDecoder.decodeAndConvertJavaObj(valueBytes);
    Object[] measureValues = rowValueDecoder.getValues();
    //BigDecimal.ROUND_HALF_EVEN in BigDecimalSerializer
    assertEquals("[333.1235, 333.1111, 333.2000, 2, 100]", Arrays.toString(measureValues));
}
 
Example #27
Source File: RowValueDecoderTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test(expected = IllegalArgumentException.class)
public void testError() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0];

    BufferedMeasureCodec codec = new BufferedMeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("11111111111111111111333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    Long item_count = new Long(100);
    codec.encode(new Object[] { sum, min, max, count, item_count });

}
 
Example #28
Source File: CubeController.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private void validateColumnFamily(CubeDesc cubeDesc) {
    Set<String> columnFamilyMetricsSet = Sets.newHashSet();
    for (HBaseColumnFamilyDesc hBaseColumnFamilyDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc hBaseColumnDesc : hBaseColumnFamilyDesc.getColumns()) {
            for (String columnName : hBaseColumnDesc.getMeasureRefs()) {
                columnFamilyMetricsSet.add(columnName);
            }
        }
    }
    for (MeasureDesc measureDesc : cubeDesc.getMeasures()) {
        if (!columnFamilyMetricsSet.contains(measureDesc.getName())) {
            throw new BadRequestException("column family lack measure:" + measureDesc.getName());
        }
    }
    if (cubeDesc.getMeasures().size() != columnFamilyMetricsSet.size()) {
        throw new BadRequestException(
                "the number of input measure and the number of measure defined in cubedesc are not consistent");
    }

    for (RowKeyColDesc rowKeyColDesc : cubeDesc.getRowkey().getRowKeyColumns()) {
        Object[] encodingConf = DimensionEncoding.parseEncodingConf(rowKeyColDesc.getEncoding());
        String encodingName = (String) encodingConf[0];
        String[] encodingArgs = (String[]) encodingConf[1];

        if (!DimensionEncodingFactory.isValidEncoding(encodingName, encodingArgs,
                rowKeyColDesc.getEncodingVersion())) {
            throw new BadRequestException("Illegal row key column desc: " + rowKeyColDesc);
        }
    }
}
 
Example #29
Source File: CubeDescTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testBadInit15() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(
            new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TRANS_CNT", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}
 
Example #30
Source File: RowValueDecoder.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up
public HBaseColumnDesc getHBaseColumn() {
    return hbaseColumn;
}