Java Code Examples for org.apache.kylin.metadata.realization.SQLDigest

The following examples show how to use org.apache.kylin.metadata.realization.SQLDigest. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: kylin-on-parquet-v2   Source File: OLAPEnumerator.java    License: Apache License 2.0 6 votes vote down vote up
private ITupleIterator queryStorage() {

        logger.debug("query storage...");
        // bind dynamic variables
        olapContext.bindVariable(optiqContext);

        // If olapContext is cached, then inherit it.
        if (!olapContext.isBorrowedContext) {
            olapContext.resetSQLDigest();
        }
        SQLDigest sqlDigest = olapContext.getSQLDigest();

        // query storage engine
        IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
        ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
                olapContext.returnTupleInfo);
        if (logger.isDebugEnabled()) {
            logger.debug("return TupleIterator...");
        }

        return iterator;
    }
 
Example 2
Source Project: kylin-on-parquet-v2   Source File: OLAPSortRel.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void implementOLAP(OLAPImplementor implementor) {
    implementor.fixSharedOlapTableScan(this);
    implementor.visitChild(getInput(), this);

    this.context = implementor.getContext();
    this.columnRowType = buildColumnRowType();

    for (RelFieldCollation fieldCollation : this.collation.getFieldCollations()) {
        int index = fieldCollation.getFieldIndex();
        SQLDigest.OrderEnum order = getOrderEnum(fieldCollation.getDirection());
        OLAPRel olapChild = (OLAPRel) this.getInput();
        TblColRef orderCol = olapChild.getColumnRowType().getAllColumns().get(index);
        this.context.addSort(orderCol, order);
        this.context.storageContext.markSort();

    }
}
 
Example 3
@Override
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
    for (MeasureDesc measureDesc : measureDescs) {
        if (!sqlDigest.involvedMeasure.contains(measureDesc)) {
            continue;
        }
        FunctionDesc extendColumnFunc = measureDesc.getFunction();
        List<TblColRef> hosts = getExtendedColumnHosts(extendColumnFunc);
        TblColRef extended = getExtendedColumn(extendColumnFunc);

        if (!sqlDigest.groupbyColumns.contains(extended)) {
            continue;
        }

        sqlDigest.aggregations.add(extendColumnFunc);
        sqlDigest.groupbyColumns.remove(extended);
        sqlDigest.groupbyColumns.addAll(hosts);
        sqlDigest.metricColumns.add(extended);
    }
}
 
Example 4
Source Project: kylin-on-parquet-v2   Source File: TopNMeasureType.java    License: Apache License 2.0 6 votes vote down vote up
private boolean totallyMatchTopN(SQLDigest digest) {
    if (!checkSortAndOrder(digest.sortColumns, digest.sortOrders)) {
        return false;
    }

    TblColRef sortColumn = digest.sortColumns.get(0);

    // first sort column must be sum()
    if (digest.groupbyColumns.contains(sortColumn)) {
        return false;
    }

    // first order must be desc
    if (!DESCENDING.equals(digest.sortOrders.get(0))) {
        return false;
    }

    if (!digest.hasLimit) {
        return false;
    }

    return true;
}
 
Example 5
Source Project: kylin-on-parquet-v2   Source File: RawMeasureType.java    License: Apache License 2.0 6 votes vote down vote up
public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, final MeasureDesc measureDesc) {
    //is raw query
    if (!digest.isRawQuery)
        return null;

    TblColRef rawColumn = getRawColumn(measureDesc.getFunction());
    if (!digest.allColumns.isEmpty() && !digest.allColumns.contains(rawColumn)) {
        return null;
    }

    unmatchedAggregations.remove(measureDesc.getFunction());

    //contain one raw measure : cost * 0.9
    return new CapabilityResult.CapabilityInfluence() {
        @Override
        public double suggestCostMultiplier() {
            return 0.9;
        }

        @Override
        public MeasureDesc getInvolvedMeasure() {
            return measureDesc;
        }
    };
}
 
Example 6
Source Project: kylin-on-parquet-v2   Source File: RawMeasureType.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {

    if (sqlDigest.isRawQuery) {
        for (MeasureDesc measureDesc : measureDescs) {
            if (!sqlDigest.involvedMeasure.contains(measureDesc)) {
                continue;
            }
            TblColRef col = this.getRawColumn(measureDesc.getFunction());
            ParameterDesc colParameter = ParameterDesc.newInstance(col);
            FunctionDesc rawFunc = FunctionDesc.newInstance("RAW", colParameter, null);

            if (sqlDigest.allColumns.contains(col)) {
                if (measureDesc.getFunction().equals(rawFunc)) {
                    FunctionDesc sumFunc = FunctionDesc.newInstance("SUM", colParameter, null);
                    sqlDigest.aggregations.remove(sumFunc);
                    sqlDigest.aggregations.add(rawFunc);
                    logger.info("Add RAW measure on column " + col);
                }
                if (!sqlDigest.metricColumns.contains(col)) {
                    sqlDigest.metricColumns.add(col);
                }
            }
        }
    }
}
 
Example 7
Source Project: kylin-on-parquet-v2   Source File: ITStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
    int count = 0;
    ITupleIterator iterator = null;
    try {
        SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", /*allCol*/ Collections.<TblColRef> emptySet(), /*join*/ null, //
                groups, /*subqueryJoinParticipants*/ Sets.<TblColRef> newHashSet(), //
                /*dynamicGroupByColumns*/ Collections.<TblColRef, TupleExpression> emptyMap(), //
                /*groupByExpression*/ false, //
                /*metricCol*/ Collections.<TblColRef> emptySet(), aggregations, /*aggrSqlCalls*/ Collections.<SQLCall> emptyList(), //
                /*dynamicAggregations*/ Collections.<DynamicFunctionDesc> emptyList(), //
                /*runtimeDimensionColumns*/ Collections.<TblColRef> emptySet(), //
                /*runtimeMetricColumns*/ Collections.<TblColRef> emptySet(), //
                /*filter col*/ Collections.<TblColRef> emptySet(), filter, null, //
                /*sortCol*/ new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false, false, false, new HashSet<MeasureDesc>());
        iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations));
        while (iterator.hasNext()) {
            ITuple tuple = iterator.next();
            System.out.println("Tuple = " + tuple);
            count++;
        }
    } finally {
        if (iterator != null)
            iterator.close();
    }
    return count;
}
 
Example 8
Source Project: kylin-on-parquet-v2   Source File: HybridInstance.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CapabilityResult isCapable(SQLDigest digest) {
    CapabilityResult result = new CapabilityResult();
    result.cost = Integer.MAX_VALUE;

    for (IRealization realization : getRealizations()) {
        CapabilityResult child = realization.isCapable(digest);
        if (child.capable) {
            result.capable = true;
            result.cost = Math.min(result.cost, child.cost);
            result.influences.addAll(child.influences);
        } else {
            result.incapableCause = child.incapableCause;
        }
    }

    if (result.cost > 0)
        result.cost--; // let hybrid win its children

    return result;
}
 
Example 9
public SequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid,
        Set<TblColRef> selectedDimensions, List<TblColRef> rtGroups, Set<TblColRef> groups, //
        Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context, SQLDigest sqlDigest) {
    this.context = context;
    this.scanners = scanners;

    Set<TblColRef> selectedDims = Sets.newHashSet(selectedDimensions);
    selectedDims.addAll(rtGroups);

    segmentCubeTupleIterators = Lists.newArrayList();
    for (CubeSegmentScanner scanner : scanners) {
        segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDims, selectedMetrics, returnTupleInfo, context));
    }

    if (context.mergeSortPartitionResults() && !sqlDigest.isRawQuery) {
        //query with limit
        logger.info("Using SortedIteratorMergerWithLimit to merge segment results");
        Iterator<Iterator<ITuple>> transformed = (Iterator<Iterator<ITuple>>) (Iterator<?>) segmentCubeTupleIterators.iterator();
        tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, groups, returnTupleInfo)).getIterator();
    } else {
        //normal case
        logger.info("Using Iterators.concat to merge segment results");
        tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
    }
}
 
Example 10
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);

    List<CubeSegmentScanner> scanners = Lists.newArrayList();
    SegmentPruner segPruner = new SegmentPruner(sqlDigest.filter);
    for (CubeSegment cubeSeg : segPruner.listSegmentsForQuery(cubeInstance)) {
        CubeSegmentScanner scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), //
                request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), //
                request.getMetrics(), request.getDynFuncs(), //
                request.getFilter(), request.getHavingFilter(), request.getContext());
        if (!scanner.isSegmentSkipped())
            scanners.add(scanner);
    }

    if (scanners.isEmpty())
        return ITupleIterator.EMPTY_TUPLE_ITERATOR;

    return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
            request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo,
            request.getContext(), sqlDigest);
}
 
Example 11
protected void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions,
        Collection<FunctionDesc> metrics) {
    for (FunctionDesc func : sqlDigest.aggregations) {
        if (!func.isDimensionAsMetric() && !FunctionDesc.FUNC_GROUPING.equalsIgnoreCase(func.getExpression())) {
            // use the FunctionDesc from cube desc as much as possible, that has more info such as HLLC precision
            metrics.add(findAggrFuncFromCubeDesc(func));
        }
    }

    for (TblColRef column : sqlDigest.allColumns) {
        // skip measure columns
        if ((sqlDigest.metricColumns.contains(column) || sqlDigest.rtMetricColumns.contains(column))
                && !(sqlDigest.groupbyColumns.contains(column) || sqlDigest.filterColumns.contains(column)
                        || sqlDigest.rtDimensionColumns.contains(column))) {
            continue;
        }

        dimensions.add(column);
    }
}
 
Example 12
protected void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
    Map<String, List<MeasureDesc>> map = Maps.newHashMap();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        MeasureType<?> measureType = measure.getFunction().getMeasureType();

        String key = measureType.getClass().getCanonicalName();
        List<MeasureDesc> temp = null;
        if ((temp = map.get(key)) != null) {
            temp.add(measure);
        } else {
            map.put(key, Lists.<MeasureDesc> newArrayList(measure));
        }
    }

    for (List<MeasureDesc> sublist : map.values()) {
        sublist.get(0).getFunction().getMeasureType().adjustSqlDigest(sublist, sqlDigest);
    }
}
 
Example 13
Source Project: Kylin   Source File: StorageTest.java    License: Apache License 2.0 6 votes vote down vote up
private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
    int count = 0;
    ITupleIterator iterator = null;
    try {
        SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
        iterator = storageEngine.search(context, sqlDigest);
        while (iterator.hasNext()) {
            ITuple tuple = iterator.next();
            System.out.println("Tuple = " + tuple);
            count++;
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (iterator != null) {
            iterator.close();
        }
    }
    return count;
}
 
Example 14
Source Project: kylin-on-parquet-v2   Source File: CubeCapabilityChecker.java    License: Apache License 2.0 6 votes vote down vote up
private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions,
        Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube,
        CapabilityResult result) {
    CubeDesc cubeDesc = cube.getDescriptor();
    List<String> influencingMeasures = Lists.newArrayList();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        //            if (unmatchedDimensions.isEmpty() && unmatchedAggregations.isEmpty())
        //                break;

        MeasureType<?> measureType = measure.getFunction().getMeasureType();
        if (measureType instanceof BasicMeasureType)
            continue;

        CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations,
                digest, measure);
        if (inf != null) {
            result.influences.add(inf);
            influencingMeasures.add(measure.getName() + "@" + measureType.getClass());
        }
    }
    if (influencingMeasures.size() != 0)
        logger.info("Cube {} CapabilityInfluences: {}", cube.getCanonicalName(),
                StringUtils.join(influencingMeasures, ","));
}
 
Example 15
Source Project: kylin-on-parquet-v2   Source File: CubeInstance.java    License: Apache License 2.0 6 votes vote down vote up
public int getCost(SQLDigest digest) {
    int calculatedCost = cost;

    //the number of dimensions is not as accurate as number of row key cols
    calculatedCost += getRowKeyColumnCount() * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;

    for (JoinTableDesc joinTable : this.getModel().getJoinTables()) {
        // more tables, more cost
        if (joinTable.getJoin().isInnerJoin()) {
            // inner join cost is bigger than left join, as it will filter some records
            calculatedCost += COST_WEIGHT_INNER_JOIN;
        }
    }

    return calculatedCost;
}
 
Example 16
Source Project: kylin   Source File: OLAPEnumerator.java    License: Apache License 2.0 6 votes vote down vote up
private ITupleIterator queryStorage() {

        logger.debug("query storage...");
        // bind dynamic variables
        olapContext.bindVariable(optiqContext);

        // If olapContext is cached, then inherit it.
        if (!olapContext.isBorrowedContext) {
            olapContext.resetSQLDigest();
        }
        SQLDigest sqlDigest = olapContext.getSQLDigest();

        // query storage engine
        IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
        ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
                olapContext.returnTupleInfo);
        if (logger.isDebugEnabled()) {
            logger.debug("return TupleIterator...");
        }

        return iterator;
    }
 
Example 17
Source Project: Kylin   Source File: CubeStorageEngine.java    License: Apache License 2.0 6 votes vote down vote up
private void buildDimensionsAndMetrics(Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics, SQLDigest sqlDigest) {

        for (FunctionDesc func : sqlDigest.aggregations) {
            if (!func.isDimensionAsMetric()) {
                metrics.add(func);
            }
        }

        for (TblColRef column : sqlDigest.allColumns) {
            // skip measure columns
            if (sqlDigest.metricColumns.contains(column)) {
                continue;
            }
            dimensions.add(column);
        }
    }
 
Example 18
Source Project: kylin   Source File: OLAPSortRel.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void implementOLAP(OLAPImplementor implementor) {
    implementor.fixSharedOlapTableScan(this);
    implementor.visitChild(getInput(), this);

    this.context = implementor.getContext();
    this.columnRowType = buildColumnRowType();

    for (RelFieldCollation fieldCollation : this.collation.getFieldCollations()) {
        int index = fieldCollation.getFieldIndex();
        SQLDigest.OrderEnum order = getOrderEnum(fieldCollation.getDirection());
        OLAPRel olapChild = (OLAPRel) this.getInput();
        TblColRef orderCol = olapChild.getColumnRowType().getAllColumns().get(index);
        this.context.addSort(orderCol, order);
        this.context.storageContext.markSort();

    }
}
 
Example 19
Source Project: kylin   Source File: ExtendedColumnMeasureType.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
    for (MeasureDesc measureDesc : measureDescs) {
        if (!sqlDigest.involvedMeasure.contains(measureDesc)) {
            continue;
        }
        FunctionDesc extendColumnFunc = measureDesc.getFunction();
        List<TblColRef> hosts = getExtendedColumnHosts(extendColumnFunc);
        TblColRef extended = getExtendedColumn(extendColumnFunc);

        if (!sqlDigest.groupbyColumns.contains(extended)) {
            continue;
        }

        sqlDigest.aggregations.add(extendColumnFunc);
        sqlDigest.groupbyColumns.remove(extended);
        sqlDigest.groupbyColumns.addAll(hosts);
        sqlDigest.metricColumns.add(extended);
    }
}
 
Example 20
Source Project: kylin   Source File: TopNMeasureType.java    License: Apache License 2.0 6 votes vote down vote up
private boolean totallyMatchTopN(SQLDigest digest) {
    if (!checkSortAndOrder(digest.sortColumns, digest.sortOrders)) {
        return false;
    }

    TblColRef sortColumn = digest.sortColumns.get(0);

    // first sort column must be sum()
    if (digest.groupbyColumns.contains(sortColumn)) {
        return false;
    }

    // first order must be desc
    if (!DESCENDING.equals(digest.sortOrders.get(0))) {
        return false;
    }

    if (!digest.hasLimit) {
        return false;
    }

    return true;
}
 
Example 21
Source Project: kylin   Source File: RawMeasureType.java    License: Apache License 2.0 6 votes vote down vote up
public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, final MeasureDesc measureDesc) {
    //is raw query
    if (!digest.isRawQuery)
        return null;

    TblColRef rawColumn = getRawColumn(measureDesc.getFunction());
    if (!digest.allColumns.isEmpty() && !digest.allColumns.contains(rawColumn)) {
        return null;
    }

    unmatchedAggregations.remove(measureDesc.getFunction());

    //contain one raw measure : cost * 0.9
    return new CapabilityResult.CapabilityInfluence() {
        @Override
        public double suggestCostMultiplier() {
            return 0.9;
        }

        @Override
        public MeasureDesc getInvolvedMeasure() {
            return measureDesc;
        }
    };
}
 
Example 22
Source Project: kylin   Source File: HybridInstance.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CapabilityResult isCapable(SQLDigest digest) {
    CapabilityResult result = new CapabilityResult();
    result.cost = Integer.MAX_VALUE;

    for (IRealization realization : getRealizations()) {
        CapabilityResult child = realization.isCapable(digest);
        if (child.capable) {
            result.capable = true;
            result.cost = Math.min(result.cost, child.cost);
            result.influences.addAll(child.influences);
        } else {
            result.incapableCause = child.incapableCause;
        }
    }

    if (result.cost > 0)
        result.cost--; // let hybrid win its children

    return result;
}
 
Example 23
Source Project: kylin   Source File: SequentialCubeTupleIterator.java    License: Apache License 2.0 6 votes vote down vote up
public SequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid,
        Set<TblColRef> selectedDimensions, List<TblColRef> rtGroups, Set<TblColRef> groups, //
        Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context, SQLDigest sqlDigest) {
    this.context = context;
    this.scanners = scanners;

    Set<TblColRef> selectedDims = Sets.newHashSet(selectedDimensions);
    selectedDims.addAll(rtGroups);

    segmentCubeTupleIterators = Lists.newArrayList();
    for (CubeSegmentScanner scanner : scanners) {
        segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDims, selectedMetrics, returnTupleInfo, context));
    }

    if (context.mergeSortPartitionResults() && !sqlDigest.isRawQuery) {
        //query with limit
        logger.info("Using SortedIteratorMergerWithLimit to merge segment results");
        Iterator<Iterator<ITuple>> transformed = (Iterator<Iterator<ITuple>>) (Iterator<?>) segmentCubeTupleIterators.iterator();
        tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, groups, returnTupleInfo)).getIterator();
    } else {
        //normal case
        logger.info("Using Iterators.concat to merge segment results");
        tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
    }
}
 
Example 24
Source Project: kylin   Source File: GTCubeStorageQueryBase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);

    List<CubeSegmentScanner> scanners = Lists.newArrayList();
    SegmentPruner segPruner = new SegmentPruner(sqlDigest.filter);
    for (CubeSegment cubeSeg : segPruner.listSegmentsForQuery(cubeInstance)) {
        CubeSegmentScanner scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), //
                request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), //
                request.getMetrics(), request.getDynFuncs(), //
                request.getFilter(), request.getHavingFilter(), request.getContext());
        if (!scanner.isSegmentSkipped())
            scanners.add(scanner);
    }

    if (scanners.isEmpty())
        return ITupleIterator.EMPTY_TUPLE_ITERATOR;

    return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
            request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo,
            request.getContext(), sqlDigest);
}
 
Example 25
Source Project: kylin   Source File: GTCubeStorageQueryBase.java    License: Apache License 2.0 6 votes vote down vote up
protected void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions,
        Collection<FunctionDesc> metrics) {
    for (FunctionDesc func : sqlDigest.aggregations) {
        if (!func.isDimensionAsMetric() && !FunctionDesc.FUNC_GROUPING.equalsIgnoreCase(func.getExpression())) {
            // use the FunctionDesc from cube desc as much as possible, that has more info such as HLLC precision
            metrics.add(findAggrFuncFromCubeDesc(func));
        }
    }

    for (TblColRef column : sqlDigest.allColumns) {
        // skip measure columns
        if ((sqlDigest.metricColumns.contains(column) || sqlDigest.rtMetricColumns.contains(column))
                && !(sqlDigest.groupbyColumns.contains(column) || sqlDigest.filterColumns.contains(column)
                        || sqlDigest.rtDimensionColumns.contains(column))) {
            continue;
        }

        dimensions.add(column);
    }
}
 
Example 26
Source Project: kylin   Source File: GTCubeStorageQueryBase.java    License: Apache License 2.0 6 votes vote down vote up
protected void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
    Map<String, List<MeasureDesc>> map = Maps.newHashMap();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        MeasureType<?> measureType = measure.getFunction().getMeasureType();

        String key = measureType.getClass().getCanonicalName();
        List<MeasureDesc> temp = null;
        if ((temp = map.get(key)) != null) {
            temp.add(measure);
        } else {
            map.put(key, Lists.<MeasureDesc> newArrayList(measure));
        }
    }

    for (List<MeasureDesc> sublist : map.values()) {
        sublist.get(0).getFunction().getMeasureType().adjustSqlDigest(sublist, sqlDigest);
    }
}
 
Example 27
Source Project: kylin   Source File: CubeCapabilityChecker.java    License: Apache License 2.0 6 votes vote down vote up
private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions,
        Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube,
        CapabilityResult result) {
    CubeDesc cubeDesc = cube.getDescriptor();
    List<String> influencingMeasures = Lists.newArrayList();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        //            if (unmatchedDimensions.isEmpty() && unmatchedAggregations.isEmpty())
        //                break;

        MeasureType<?> measureType = measure.getFunction().getMeasureType();
        if (measureType instanceof BasicMeasureType)
            continue;

        CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations,
                digest, measure);
        if (inf != null) {
            result.influences.add(inf);
            influencingMeasures.add(measure.getName() + "@" + measureType.getClass());
        }
    }
    if (influencingMeasures.size() != 0)
        logger.info("Cube {} CapabilityInfluences: {}", cube.getCanonicalName(),
                StringUtils.join(influencingMeasures, ","));
}
 
Example 28
Source Project: kylin   Source File: CubeInstance.java    License: Apache License 2.0 6 votes vote down vote up
public int getCost(SQLDigest digest) {
    int calculatedCost = cost;

    //the number of dimensions is not as accurate as number of row key cols
    calculatedCost += getRowKeyColumnCount() * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;

    for (JoinTableDesc joinTable : this.getModel().getJoinTables()) {
        // more tables, more cost
        if (joinTable.getJoin().isInnerJoin()) {
            // inner join cost is bigger than left join, as it will filter some records
            calculatedCost += COST_WEIGHT_INNER_JOIN;
        }
    }

    return calculatedCost;
}
 
Example 29
Source Project: kylin-on-parquet-v2   Source File: OLAPSortRel.java    License: Apache License 2.0 5 votes vote down vote up
SQLDigest.OrderEnum getOrderEnum(RelFieldCollation.Direction direction) {
    if (direction == RelFieldCollation.Direction.DESCENDING) {
        return SQLDigest.OrderEnum.DESCENDING;
    } else {
        return SQLDigest.OrderEnum.ASCENDING;
    }
}
 
Example 30
Source Project: kylin-on-parquet-v2   Source File: BitmapMeasureType.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
    for (SQLCall call : sqlDigest.aggrSqlCalls) {
        if (FUNC_INTERSECT_COUNT_DISTINCT.equals(call.function)
                || FUNC_INTERSECT_VALUE.equals(call.function)) {
            TblColRef col = (TblColRef) call.args.get(1);
            if (!sqlDigest.groupbyColumns.contains(col))
                sqlDigest.groupbyColumns.add(col);
        }
    }
}