org.apache.kylin.metadata.model.MeasureDesc Java Examples

The following examples show how to use org.apache.kylin.metadata.model.MeasureDesc. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RawMeasureType.java    From kylin with Apache License 2.0 6 votes vote down vote up
public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, final MeasureDesc measureDesc) {
    //is raw query
    if (!digest.isRawQuery)
        return null;

    TblColRef rawColumn = getRawColumn(measureDesc.getFunction());
    if (!digest.allColumns.isEmpty() && !digest.allColumns.contains(rawColumn)) {
        return null;
    }

    unmatchedAggregations.remove(measureDesc.getFunction());

    //contain one raw measure : cost * 0.9
    return new CapabilityResult.CapabilityInfluence() {
        @Override
        public double suggestCostMultiplier() {
            return 0.9;
        }

        @Override
        public MeasureDesc getInvolvedMeasure() {
            return measureDesc;
        }
    };
}
 
Example #2
Source File: CubeDesc.java    From kylin with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
private void initMeasureColumns() {
    if (measures == null || measures.isEmpty()) {
        return;
    }

    for (MeasureDesc m : measures) {
        m.setName(m.getName().toUpperCase(Locale.ROOT));

        if (m.getDependentMeasureRef() != null) {
            m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase(Locale.ROOT));
        }

        FunctionDesc func = m.getFunction();
        func.init(model);
        allColumns.addAll(func.getParameter().getColRefs());

        if (ExtendedColumnMeasureType.FUNC_EXTENDED_COLUMN.equalsIgnoreCase(m.getFunction().getExpression())) {
            FunctionDesc functionDesc = m.getFunction();

            List<TblColRef> hosts = ExtendedColumnMeasureType.getExtendedColumnHosts(functionDesc);
            TblColRef extendedColumn = ExtendedColumnMeasureType.getExtendedColumn(functionDesc);
            initExtendedColumnMap(hosts.toArray(new TblColRef[hosts.size()]), extendedColumn);
        }
    }
}
 
Example #3
Source File: OLAPAggregateRel.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
FunctionDesc findInMeasures(FunctionDesc aggFunc, List<MeasureDesc> measures) {
    for (MeasureDesc m : measures) {
        if (aggFunc.equals(m.getFunction())) {
            return m.getFunction();
        }
    }

    // no count(col) measure found, use count(1) to replace it.
    if (aggFunc.isCount()) {
        FunctionDesc func = findCountConstantFunc(measures);
        if (func != null)
            return func;
    }

    return aggFunc;
}
 
Example #4
Source File: MeasureAggregators.java    From kylin with Apache License 2.0 6 votes vote down vote up
public MeasureAggregators(MeasureDesc... measureDescs) {
    descLength = measureDescs.length;
    aggs = new MeasureAggregator[descLength];

    Map<String, Integer> measureIndexMap = new HashMap<String, Integer>();
    for (int i = 0; i < descLength; i++) {
        FunctionDesc func = measureDescs[i].getFunction();
        aggs[i] = func.getMeasureType().newAggregator();
        measureIndexMap.put(measureDescs[i].getName(), i);
    }
    // fill back dependent aggregator
    for (int i = 0; i < descLength; i++) {
        String depMsrRef = measureDescs[i].getDependentMeasureRef();
        if (depMsrRef != null) {
            int index = measureIndexMap.get(depMsrRef);
            aggs[i].setDependentAggregator(aggs[index]);
        }
    }
}
 
Example #5
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private void setThreshold(Collection<TblColRef> dimensions, List<RowValueDecoder> valueDecoders, StorageContext context) {
    if (RowValueDecoder.hasMemHungryCountDistinct(valueDecoders) == false) {
        return;
    }

    int rowSizeEst = dimensions.size() * 3;
    for (RowValueDecoder decoder : valueDecoders) {
        MeasureDesc[] measures = decoder.getMeasures();
        BitSet projectionIndex = decoder.getProjectionIndex();
        for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) {
            FunctionDesc func = measures[i].getFunction();
            rowSizeEst += func.getReturnDataType().getSpaceEstimate();
        }
    }

    long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst;
    context.setThreshold((int) rowEst);
}
 
Example #6
Source File: ITStorageTest.java    From kylin with Apache License 2.0 6 votes vote down vote up
private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
    int count = 0;
    ITupleIterator iterator = null;
    try {
        SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", /*allCol*/ Collections.<TblColRef> emptySet(), /*join*/ null, //
                groups, /*subqueryJoinParticipants*/ Sets.<TblColRef> newHashSet(), //
                /*dynamicGroupByColumns*/ Collections.<TblColRef, TupleExpression> emptyMap(), //
                /*groupByExpression*/ false, //
                /*metricCol*/ Collections.<TblColRef> emptySet(), aggregations, /*aggrSqlCalls*/ Collections.<SQLCall> emptyList(), //
                /*dynamicAggregations*/ Collections.<DynamicFunctionDesc> emptyList(), //
                /*runtimeDimensionColumns*/ Collections.<TblColRef> emptySet(), //
                /*runtimeMetricColumns*/ Collections.<TblColRef> emptySet(), //
                /*filter col*/ Collections.<TblColRef> emptySet(), filter, null, //
                /*sortCol*/ new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false, false, false, new HashSet<MeasureDesc>());
        iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations));
        while (iterator.hasNext()) {
            ITuple tuple = iterator.next();
            System.out.println("Tuple = " + tuple);
            count++;
        }
    } finally {
        if (iterator != null)
            iterator.close();
    }
    return count;
}
 
Example #7
Source File: CubeCapabilityChecker.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions,
        Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube,
        CapabilityResult result) {
    CubeDesc cubeDesc = cube.getDescriptor();
    List<String> influencingMeasures = Lists.newArrayList();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        //            if (unmatchedDimensions.isEmpty() && unmatchedAggregations.isEmpty())
        //                break;

        MeasureType<?> measureType = measure.getFunction().getMeasureType();
        if (measureType instanceof BasicMeasureType)
            continue;

        CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations,
                digest, measure);
        if (inf != null) {
            result.influences.add(inf);
            influencingMeasures.add(measure.getName() + "@" + measureType.getClass());
        }
    }
    if (influencingMeasures.size() != 0)
        logger.info("Cube {} CapabilityInfluences: {}", cube.getCanonicalName(),
                StringUtils.join(influencingMeasures, ","));
}
 
Example #8
Source File: OLAPSortRel.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Override
public void implementRewrite(RewriteImplementor implementor) {
    implementor.visitChild(this, getChild());

    for (RelFieldCollation fieldCollation : this.collation.getFieldCollations()) {
        int index = fieldCollation.getFieldIndex();
        StorageContext.OrderEnum order = getOrderEnum(fieldCollation.getDirection());
        OLAPRel olapChild = (OLAPRel) this.getChild();
        TblColRef orderCol = olapChild.getColumnRowType().getAllColumns().get(index);
        MeasureDesc measure = findMeasure(orderCol);
        if (measure != null) {
            this.context.storageContext.addSort(measure, order);
        }
        this.context.storageContext.markSort();
    }

    this.rowType = this.deriveRowType();
    this.columnRowType = buildColumnRowType();
}
 
Example #9
Source File: TopNMeasureTypeTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void test() {

    CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc");

    MeasureDesc topSellerMeasure = null;

    for (MeasureDesc measureDesc : desc.getMeasures()) {
        if (measureDesc.getName().equals("TOP_SELLER")) {
            topSellerMeasure = measureDesc;
            break;
        }
    }
    TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType());

    topSellerMeasure.getFunction().getConfiguration().clear();
    List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());

    assertTrue(colsNeedDict != null && colsNeedDict.size() == 1);

    TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1);
    topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6");
    colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());

    assertTrue(colsNeedDict.size() == 0);
}
 
Example #10
Source File: CubeDesc.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private void initMeasureReferenceToColumnFamily() {
    if (measures == null || measures.size() == 0)
        return;

    Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>();
    for (MeasureDesc m : measures)
        measureCache.put(m.getName(), m);

    for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc c : cf.getColumns()) {
            MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length];
            for (int i = 0; i < c.getMeasureRefs().length; i++) {
                measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]);
            }
            c.setMeasures(measureDescs);
            c.setColumnFamilyName(cf.getName());
        }
    }
}
 
Example #11
Source File: CubeDesc.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
private void initMeasureColumns() {
    if (measures == null || measures.isEmpty()) {
        return;
    }

    for (MeasureDesc m : measures) {
        m.setName(m.getName().toUpperCase(Locale.ROOT));

        if (m.getDependentMeasureRef() != null) {
            m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase(Locale.ROOT));
        }

        FunctionDesc func = m.getFunction();
        func.init(model);
        allColumns.addAll(func.getParameter().getColRefs());

        if (ExtendedColumnMeasureType.FUNC_EXTENDED_COLUMN.equalsIgnoreCase(m.getFunction().getExpression())) {
            FunctionDesc functionDesc = m.getFunction();

            List<TblColRef> hosts = ExtendedColumnMeasureType.getExtendedColumnHosts(functionDesc);
            TblColRef extendedColumn = ExtendedColumnMeasureType.getExtendedColumn(functionDesc);
            initExtendedColumnMap(hosts.toArray(new TblColRef[hosts.size()]), extendedColumn);
        }
    }
}
 
Example #12
Source File: CubeDesc.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private void initMeasureReferenceToColumnFamily() {
    if (measures == null || measures.size() == 0)
        return;

    Map<String, MeasureDesc> measureCache = new HashMap<String, MeasureDesc>();
    for (MeasureDesc m : measures)
        measureCache.put(m.getName(), m);

    for (HBaseColumnFamilyDesc cf : getHBaseMapping().getColumnFamily()) {
        for (HBaseColumnDesc c : cf.getColumns()) {
            MeasureDesc[] measureDescs = new MeasureDesc[c.getMeasureRefs().length];
            for (int i = 0; i < c.getMeasureRefs().length; i++) {
                measureDescs[i] = measureCache.get(c.getMeasureRefs()[i]);
            }
            c.setMeasures(measureDescs);
            c.setColumnFamilyName(cf.getName());
        }
    }
}
 
Example #13
Source File: KeyValueCreator.java    From kylin with Apache License 2.0 6 votes vote down vote up
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

        cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
        qBytes = Bytes.toBytes(colDesc.getQualifier());
        timestamp = 0; // use 0 for timestamp

        refIndex = colDesc.getMeasureIndex();
        refMeasures = colDesc.getMeasures();

        codec = new BufferedMeasureCodec(refMeasures);
        colValues = new Object[refMeasures.length];

        isFullCopy = true;
        List<MeasureDesc> measures = cubeDesc.getMeasures();
        for (int i = 0; i < measures.size(); i++) {
            if (refIndex.length <= i || refIndex[i] != i)
                isFullCopy = false;
        }
    }
 
Example #14
Source File: OLAPAggregateRel.java    From kylin with Apache License 2.0 6 votes vote down vote up
FunctionDesc findInMeasures(FunctionDesc aggFunc, List<MeasureDesc> measures) {
    for (MeasureDesc m : measures) {
        if (aggFunc.equals(m.getFunction())) {
            return m.getFunction();
        }
    }

    // no count(col) measure found, use count(1) to replace it.
    if (aggFunc.isCount()) {
        FunctionDesc func = findCountConstantFunc(measures);
        if (func != null)
            return func;
    }

    return aggFunc;
}
 
Example #15
Source File: BaseCuboidBuilder.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private void checkMrDictClolumn(){
    Set<String> mrDictColumnSet = new HashSet<>();
    if (kylinConfig.getMrHiveDictColumns() != null) {
        Collections.addAll(mrDictColumnSet, kylinConfig.getMrHiveDictColumns());
    }

    for (MeasureDesc measure : measureDescList) {
        if (measure.getFunction().getExpression().equalsIgnoreCase(FunctionDesc.FUNC_COUNT_DISTINCT)) {
            FunctionDesc functionDesc = measure.getFunction();
            TblColRef colRef = functionDesc.getParameter().getColRefs().get(0);
            if (mrDictColumnSet.contains(JoinedFlatTable.colName(colRef, true))) {
                functionDesc.setMrDict(true);
                logger.info("setMrDict for {}", colRef);
                measure.setFunction(functionDesc);
            }
        }
    }
}
 
Example #16
Source File: FragmentCuboidReaderPerfTest.java    From kylin with Apache License 2.0 6 votes vote down vote up
public Pair<List<TblColRef>, List<MeasureDesc>> getReadDimensionsAndMetrics() {
        Set<TblColRef> dimensionsSet = testHelper.simulateDimensions("STREAMING_V2_TABLE.MINUTE_START");
        List<TblColRef> dimensions = new ArrayList<>(dimensionsSet);
        FunctionDesc metricFunc = testHelper.simulateMetric("STREAMING_V2_TABLE.GMV", "SUM", "decimal(19,6)");
        List<MeasureDesc> allMetrics = cubeInstance.getDescriptor().getMeasures();
        List<MeasureDesc> targetMetrics = Lists.newArrayList();
        for (MeasureDesc metric : allMetrics) {
                                    if (metric.getFunction().equals(metricFunc)) {
                                        targetMetrics.add(metric);
                                    }
            if (metric.getName().equalsIgnoreCase("_COUNT_")) {
//                targetMetrics.add(metric);
            }
        }
        return new Pair<>(dimensions, targetMetrics);
    }
 
Example #17
Source File: MeasureAggregators.java    From Kylin with Apache License 2.0 6 votes vote down vote up
public MeasureAggregators(MeasureDesc... measureDescs) {
    descs = measureDescs;
    aggs = new MeasureAggregator[descs.length];

    Map<String, Integer> measureIndexMap = new HashMap<String, Integer>();
    for (int i = 0; i < descs.length; i++) {
        FunctionDesc func = descs[i].getFunction();
        aggs[i] = MeasureAggregator.create(func.getExpression(), func.getReturnType());
        measureIndexMap.put(descs[i].getName(), i);
    }
    // fill back dependent aggregator
    for (int i = 0; i < descs.length; i++) {
        String depMsrRef = descs[i].getDependentMeasureRef();
        if (depMsrRef != null) {
            int index = measureIndexMap.get(depMsrRef);
            aggs[i].setDependentAggregator(aggs[index]);
        }
    }
}
 
Example #18
Source File: CubeDesc.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private void initMeasureColumns(Map<String, TableDesc> tables) {
    if (measures == null || measures.isEmpty()) {
        return;
    }

    TableDesc factTable = tables.get(getFactTable());
    for (MeasureDesc m : measures) {
        m.setName(m.getName().toUpperCase());

        if (m.getDependentMeasureRef() != null) {
            m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase());
        }
        
        FunctionDesc f = m.getFunction();
        f.setExpression(f.getExpression().toUpperCase());
        f.setReturnDataType(DataType.getInstance(f.getReturnType()));

        ParameterDesc p = f.getParameter();
        p.normalizeColumnValue();

        if (p.isColumnType()) {
            ArrayList<TblColRef> colRefs = Lists.newArrayList();
            for (String cName : p.getValue().split("\\s*,\\s*")) {
                ColumnDesc sourceColumn = factTable.findColumnByName(cName);
                TblColRef colRef = new TblColRef(sourceColumn);
                colRefs.add(colRef);
                allColumns.add(colRef);
            }
            if (colRefs.isEmpty() == false)
                p.setColRefs(colRefs);
        }
        
        // verify holistic count distinct as a dependent measure
        if (m.isHolisticCountDistinct() && StringUtils.isBlank(m.getDependentMeasureRef())) {
            throw new IllegalStateException(m + " is a holistic count distinct but it has no DependentMeasureRef defined!");
        }
    }
}
 
Example #19
Source File: ExtendedColumnMeasureType.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Override
public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions,
        Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, final MeasureDesc measureDesc) {
    TblColRef extendedCol = getExtendedColumn(measureDesc.getFunction());

    if (!unmatchedDimensions.contains(extendedCol)) {
        return null;
    }

    if (digest.filterColumns.contains(extendedCol)) {
        return null;
    }

    unmatchedDimensions.remove(extendedCol);

    return new CapabilityResult.CapabilityInfluence() {
        @Override
        public double suggestCostMultiplier() {
            return 0.9;
        }

        @Override
        public MeasureDesc getInvolvedMeasure() {
            return measureDesc;
        }
    };
}
 
Example #20
Source File: RowValueDecoderTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testDecode() throws Exception {
    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    HBaseColumnDesc hbaseCol = cubeDesc.getHBaseMapping().getColumnFamily()[0].getColumns()[0];

    MeasureCodec codec = new MeasureCodec(hbaseCol.getMeasures());
    BigDecimal sum = new BigDecimal("333.1234567");
    BigDecimal min = new BigDecimal("333.1111111");
    BigDecimal max = new BigDecimal("333.1999999");
    LongWritable count = new LongWritable(2);
    ByteBuffer buf = ByteBuffer.allocate(RowConstants.ROWVALUE_BUFFER_SIZE);
    codec.encode(new Object[] { sum, min, max, count }, buf);

    buf.flip();
    byte[] valueBytes = new byte[buf.limit()];
    System.arraycopy(buf.array(), 0, valueBytes, 0, buf.limit());

    RowValueDecoder rowValueDecoder = new RowValueDecoder(hbaseCol);
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc aggrFunc = measure.getFunction();
        int index = hbaseCol.findMeasureIndex(aggrFunc);
        rowValueDecoder.setIndex(index);
    }

    rowValueDecoder.decode(valueBytes);
    List<String> measureNames = rowValueDecoder.getNames();
    Object[] measureValues = rowValueDecoder.getValues();

    assertEquals("[PRICE, MIN_PRICE_, MAX_PRICE_, COUNT__]", measureNames.toString());
    assertEquals("[333.1234567, 333.1111111, 333.1999999, 2]", Arrays.toString(measureValues));
}
 
Example #21
Source File: FragmentCuboidReaderPerfTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
public void scanPerformance() throws Exception {
    Pair<List<TblColRef>, List<MeasureDesc>> readDimAndMetrics = getReadDimensionsAndMetrics();

    for (int i = 1; i < 5; i++) {
        scan(i, readDimAndMetrics.getFirst(), readDimAndMetrics.getSecond().toArray(new MeasureDesc[0]));
    }
}
 
Example #22
Source File: CubeDesc.java    From kylin with Apache License 2.0 5 votes vote down vote up
public boolean hasMemoryHungryMeasures() {
    for (MeasureDesc measure : measures) {
        if (measure.getFunction().getMeasureType().isMemoryHungry()) {
            return true;
        }
    }
    return false;
}
 
Example #23
Source File: LongIngester.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Override
public Long valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
    if (values.length > 1)
        throw new IllegalArgumentException();

    if (values[0] == null || values[0].length() == 0)
        return new Long(0L);
    else
        return Long.valueOf(values[0]);
}
 
Example #24
Source File: GTCubeStorageQueryBase.java    From kylin with Apache License 2.0 5 votes vote down vote up
private TupleFilter checkHavingCanPushDown(TupleFilter havingFilter, Set<TblColRef> groupsD,
        List<FunctionDesc> aggregations, Set<FunctionDesc> metrics) {
    // must have only one segment
    Segments<CubeSegment> readySegs = cubeInstance.getSegments(SegmentStatusEnum.READY);
    if (readySegs.size() != 1) {
        logger.info("Can not push down having filter, must have only one segment");
        return null;
    }
    // sharded-by column must on group by
    CubeDesc desc = cubeInstance.getDescriptor();
    Set<TblColRef> shardBy = desc.getShardByColumns();
    if (groupsD == null || shardBy.isEmpty() || !groupsD.containsAll(shardBy))
        return null;

    // OK, push down
    logger.info("Push down having filter {}", havingFilter);

    // convert columns in the filter
    Set<TblColRef> aggrOutCols = new HashSet<>();
    TupleFilter.collectColumns(havingFilter, aggrOutCols);

    for (TblColRef aggrOutCol : aggrOutCols) {
        int aggrIdxOnSql = aggrOutCol.getColumnDesc().getZeroBasedIndex(); // aggr index marked in OLAPAggregateRel
        FunctionDesc aggrFunc = aggregations.get(aggrIdxOnSql);

        // calculate the index of this aggr among all the metrics that is sending to storage
        int aggrIdxAmongMetrics = 0;
        for (MeasureDesc m : cubeDesc.getMeasures()) {
            if (aggrFunc.equals(m.getFunction()))
                break;
            if (metrics.contains(m.getFunction()))
                aggrIdxAmongMetrics++;
        }
        aggrOutCol.getColumnDesc().setId("" + (aggrIdxAmongMetrics + 1));
    }
    return havingFilter;
}
 
Example #25
Source File: OLAPTable.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private List<ColumnDesc> listSourceColumns() {
    ProjectManager mgr = ProjectManager.getInstance(olapSchema.getConfig());

    List<ColumnDesc> tableColumns = mgr.listExposedColumns(olapSchema.getProjectName(), sourceTable, exposeMore);

    List<ColumnDesc> metricColumns = Lists.newArrayList();
    List<MeasureDesc> countMeasures = mgr.listEffectiveRewriteMeasures(olapSchema.getProjectName(),
            sourceTable.getIdentity());
    HashSet<String> metFields = new HashSet<String>();
    for (MeasureDesc m : countMeasures) {

        FunctionDesc func = m.getFunction();
        String fieldName = func.getRewriteFieldName();
        if (!metFields.contains(fieldName)) {
            metFields.add(fieldName);
            ColumnDesc fakeCountCol = func.newFakeRewriteColumn(sourceTable);
            metricColumns.add(fakeCountCol);
        }
    }

    Collections.sort(tableColumns, new Comparator<ColumnDesc>() {
        @Override
        public int compare(ColumnDesc o1, ColumnDesc o2) {
            return o1.getZeroBasedIndex() - o2.getZeroBasedIndex();
        }
    });
    return Lists.newArrayList(Iterables.concat(tableColumns, metricColumns));
}
 
Example #26
Source File: CubeDescTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testBadInit15() throws Exception {
    thrown.expect(IllegalStateException.class);
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    MeasureDesc measureForTransCnt = cubeDesc.getMeasures().get(3);
    Assert.assertEquals(measureForTransCnt.getName(), "TRANS_CNT");
    thrown.expectMessage("measure (" + measureForTransCnt.getName() + ") duplicates");
    HBaseColumnDesc colDesc = new HBaseColumnDesc();
    colDesc.setQualifier("M");
    colDesc.setMeasureRefs(
            new String[] { "GMV_SUM", "GMV_MIN", "GMV_MAX", "TRANS_CNT", "TRANS_CNT", "ITEM_COUNT_SUM" });
    cubeDesc.getHbaseMapping().getColumnFamily()[0].getColumns()[0] = colDesc;
    cubeDesc.initMeasureReferenceToColumnFamily();
}
 
Example #27
Source File: FragmentCuboidReaderPerfTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
private void readRow(int time, List<TblColRef> dimensions, MeasureDesc[] metrics) throws IOException {
    System.out.println("start " + time + " read, " + dimensions.size() + " dimensions," + metrics.length
            + " measures");
    TblColRef[] dimArray = dimensions.toArray(new TblColRef[dimensions.size()]);
    Random rand = new Random();
    int randReadNum = 100;
    int[][] readRows = new int[fragments.length][randReadNum];
    List<FragmentCuboidReader> fragmentCuboidReaders = Lists.newArrayList();
    for (int i = 0; i < fragments.length; i++) {
        FragmentMetaInfo fragmentMetaInfo = fragments[i].getMetaInfo();
        FragmentData fragmentData = new FragmentData(fragmentMetaInfo, fragments[i].getDataFile());
        Map<TblColRef, Dictionary<String>> dictionaryMap = fragmentData
                .getDimensionDictionaries(parsedStreamingCubeInfo.dimensionsUseDictEncoding);
        DimensionEncoding[] dimensionEncodings = ParsedStreamingCubeInfo.getDimensionEncodings(
                parsedStreamingCubeInfo.cubeDesc, dimArray, dictionaryMap);
        FragmentCuboidReader fragmentCuboidReader = new FragmentCuboidReader(parsedStreamingCubeInfo.cubeDesc,
                fragmentData, fragmentMetaInfo.getBasicCuboidMetaInfo(), dimArray, metrics, dimensionEncodings);
        fragmentCuboidReaders.add(fragmentCuboidReader);
        for (int j = 0; j < randReadNum; j++) {
            readRows[i][j] = rand.nextInt((int) fragmentMetaInfo.getNumberOfRows());
        }
    }
    Stopwatch sw = Stopwatch.createUnstarted();
    sw.start();
    int rowNum = 0;

    for (int i = 0; i < fragments.length; i++) {
        for (int j = 0; j < readRows.length; j++) {
            fragmentCuboidReaders.get(i).read(readRows[i][j]);
        }
    }
    sw.stop();
    long takeTime = sw.elapsed(MILLISECONDS);
    System.out.println(time + " scan finished, total rows:" + rowNum);
    System.out.println(time + " scan took:" + takeTime + ",rowsPerSec:" + (rowNum / takeTime) * 1000);
}
 
Example #28
Source File: LongIngester.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Override
public Long valueOf(String[] values, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
    if (values.length > 1)
        throw new IllegalArgumentException();

    if (values[0] == null || values[0].length() == 0)
        return new Long(0L);
    else
        return Long.valueOf(values[0]);
}
 
Example #29
Source File: RowValueDecoder.java    From Kylin with Apache License 2.0 5 votes vote down vote up
public RowValueDecoder(HBaseColumnDesc hbaseColumn) {
    this.hbaseColumn = hbaseColumn;
    this.projectionIndex = new BitSet();
    this.names = new ArrayList<String>();
    this.measures = hbaseColumn.getMeasures();
    for (MeasureDesc measure : measures) {
        this.names.add(measure.getFunction().getRewriteFieldName());
    }
    this.codec = new MeasureCodec(measures);
    this.values = new Object[measures.length];
}
 
Example #30
Source File: CubeDesc.java    From Kylin with Apache License 2.0 5 votes vote down vote up
public boolean hasHolisticCountDistinctMeasures() {
    for (MeasureDesc measure : measures) {
        if (measure.getFunction().isHolisticCountDistinct()) {
            return true;
        }
    }
    return false;
}