org.apache.kylin.metadata.realization.SQLDigest Java Examples

The following examples show how to use org.apache.kylin.metadata.realization.SQLDigest. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HybridInstance.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
public CapabilityResult isCapable(SQLDigest digest) {
    CapabilityResult result = new CapabilityResult();
    result.cost = Integer.MAX_VALUE;

    for (IRealization realization : getRealizations()) {
        CapabilityResult child = realization.isCapable(digest);
        if (child.capable) {
            result.capable = true;
            result.cost = Math.min(result.cost, child.cost);
            result.influences.addAll(child.influences);
        } else {
            result.incapableCause = child.incapableCause;
        }
    }

    if (result.cost > 0)
        result.cost--; // let hybrid win its children

    return result;
}
 
Example #2
Source File: RawMeasureType.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, final MeasureDesc measureDesc) {
    //is raw query
    if (!digest.isRawQuery)
        return null;

    TblColRef rawColumn = getRawColumn(measureDesc.getFunction());
    if (!digest.allColumns.isEmpty() && !digest.allColumns.contains(rawColumn)) {
        return null;
    }

    unmatchedAggregations.remove(measureDesc.getFunction());

    //contain one raw measure : cost * 0.9
    return new CapabilityResult.CapabilityInfluence() {
        @Override
        public double suggestCostMultiplier() {
            return 0.9;
        }

        @Override
        public MeasureDesc getInvolvedMeasure() {
            return measureDesc;
        }
    };
}
 
Example #3
Source File: TopNMeasureType.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private boolean totallyMatchTopN(SQLDigest digest) {
    if (!checkSortAndOrder(digest.sortColumns, digest.sortOrders)) {
        return false;
    }

    TblColRef sortColumn = digest.sortColumns.get(0);

    // first sort column must be sum()
    if (digest.groupbyColumns.contains(sortColumn)) {
        return false;
    }

    // first order must be desc
    if (!DESCENDING.equals(digest.sortOrders.get(0))) {
        return false;
    }

    if (!digest.hasLimit) {
        return false;
    }

    return true;
}
 
Example #4
Source File: ExtendedColumnMeasureType.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Override
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
    for (MeasureDesc measureDesc : measureDescs) {
        if (!sqlDigest.involvedMeasure.contains(measureDesc)) {
            continue;
        }
        FunctionDesc extendColumnFunc = measureDesc.getFunction();
        List<TblColRef> hosts = getExtendedColumnHosts(extendColumnFunc);
        TblColRef extended = getExtendedColumn(extendColumnFunc);

        if (!sqlDigest.groupbyColumns.contains(extended)) {
            continue;
        }

        sqlDigest.aggregations.add(extendColumnFunc);
        sqlDigest.groupbyColumns.remove(extended);
        sqlDigest.groupbyColumns.addAll(hosts);
        sqlDigest.metricColumns.add(extended);
    }
}
 
Example #5
Source File: CubeCapabilityChecker.java    From kylin with Apache License 2.0 6 votes vote down vote up
private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions,
        Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube,
        CapabilityResult result) {
    CubeDesc cubeDesc = cube.getDescriptor();
    List<String> influencingMeasures = Lists.newArrayList();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        //            if (unmatchedDimensions.isEmpty() && unmatchedAggregations.isEmpty())
        //                break;

        MeasureType<?> measureType = measure.getFunction().getMeasureType();
        if (measureType instanceof BasicMeasureType)
            continue;

        CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations,
                digest, measure);
        if (inf != null) {
            result.influences.add(inf);
            influencingMeasures.add(measure.getName() + "@" + measureType.getClass());
        }
    }
    if (influencingMeasures.size() != 0)
        logger.info("Cube {} CapabilityInfluences: {}", cube.getCanonicalName(),
                StringUtils.join(influencingMeasures, ","));
}
 
Example #6
Source File: CubeInstance.java    From kylin with Apache License 2.0 6 votes vote down vote up
public int getCost(SQLDigest digest) {
    int calculatedCost = cost;

    //the number of dimensions is not as accurate as number of row key cols
    calculatedCost += getRowKeyColumnCount() * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;

    for (JoinTableDesc joinTable : this.getModel().getJoinTables()) {
        // more tables, more cost
        if (joinTable.getJoin().isInnerJoin()) {
            // inner join cost is bigger than left join, as it will filter some records
            calculatedCost += COST_WEIGHT_INNER_JOIN;
        }
    }

    return calculatedCost;
}
 
Example #7
Source File: OLAPSortRel.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Override
public void implementOLAP(OLAPImplementor implementor) {
    implementor.fixSharedOlapTableScan(this);
    implementor.visitChild(getInput(), this);

    this.context = implementor.getContext();
    this.columnRowType = buildColumnRowType();

    for (RelFieldCollation fieldCollation : this.collation.getFieldCollations()) {
        int index = fieldCollation.getFieldIndex();
        SQLDigest.OrderEnum order = getOrderEnum(fieldCollation.getDirection());
        OLAPRel olapChild = (OLAPRel) this.getInput();
        TblColRef orderCol = olapChild.getColumnRowType().getAllColumns().get(index);
        this.context.addSort(orderCol, order);
        this.context.storageContext.markSort();

    }
}
 
Example #8
Source File: TopNMeasureType.java    From kylin with Apache License 2.0 6 votes vote down vote up
private boolean totallyMatchTopN(SQLDigest digest) {
    if (!checkSortAndOrder(digest.sortColumns, digest.sortOrders)) {
        return false;
    }

    TblColRef sortColumn = digest.sortColumns.get(0);

    // first sort column must be sum()
    if (digest.groupbyColumns.contains(sortColumn)) {
        return false;
    }

    // first order must be desc
    if (!DESCENDING.equals(digest.sortOrders.get(0))) {
        return false;
    }

    if (!digest.hasLimit) {
        return false;
    }

    return true;
}
 
Example #9
Source File: RawMeasureType.java    From kylin with Apache License 2.0 6 votes vote down vote up
public CapabilityResult.CapabilityInfluence influenceCapabilityCheck(Collection<TblColRef> unmatchedDimensions, Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, final MeasureDesc measureDesc) {
    //is raw query
    if (!digest.isRawQuery)
        return null;

    TblColRef rawColumn = getRawColumn(measureDesc.getFunction());
    if (!digest.allColumns.isEmpty() && !digest.allColumns.contains(rawColumn)) {
        return null;
    }

    unmatchedAggregations.remove(measureDesc.getFunction());

    //contain one raw measure : cost * 0.9
    return new CapabilityResult.CapabilityInfluence() {
        @Override
        public double suggestCostMultiplier() {
            return 0.9;
        }

        @Override
        public MeasureDesc getInvolvedMeasure() {
            return measureDesc;
        }
    };
}
 
Example #10
Source File: RawMeasureType.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Override
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {

    if (sqlDigest.isRawQuery) {
        for (MeasureDesc measureDesc : measureDescs) {
            if (!sqlDigest.involvedMeasure.contains(measureDesc)) {
                continue;
            }
            TblColRef col = this.getRawColumn(measureDesc.getFunction());
            ParameterDesc colParameter = ParameterDesc.newInstance(col);
            FunctionDesc rawFunc = FunctionDesc.newInstance("RAW", colParameter, null);

            if (sqlDigest.allColumns.contains(col)) {
                if (measureDesc.getFunction().equals(rawFunc)) {
                    FunctionDesc sumFunc = FunctionDesc.newInstance("SUM", colParameter, null);
                    sqlDigest.aggregations.remove(sumFunc);
                    sqlDigest.aggregations.add(rawFunc);
                    logger.info("Add RAW measure on column " + col);
                }
                if (!sqlDigest.metricColumns.contains(col)) {
                    sqlDigest.metricColumns.add(col);
                }
            }
        }
    }
}
 
Example #11
Source File: ITStorageTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
    int count = 0;
    ITupleIterator iterator = null;
    try {
        SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", /*allCol*/ Collections.<TblColRef> emptySet(), /*join*/ null, //
                groups, /*subqueryJoinParticipants*/ Sets.<TblColRef> newHashSet(), //
                /*dynamicGroupByColumns*/ Collections.<TblColRef, TupleExpression> emptyMap(), //
                /*groupByExpression*/ false, //
                /*metricCol*/ Collections.<TblColRef> emptySet(), aggregations, /*aggrSqlCalls*/ Collections.<SQLCall> emptyList(), //
                /*dynamicAggregations*/ Collections.<DynamicFunctionDesc> emptyList(), //
                /*runtimeDimensionColumns*/ Collections.<TblColRef> emptySet(), //
                /*runtimeMetricColumns*/ Collections.<TblColRef> emptySet(), //
                /*filter col*/ Collections.<TblColRef> emptySet(), filter, null, //
                /*sortCol*/ new ArrayList<TblColRef>(), new ArrayList<SQLDigest.OrderEnum>(), false, false, false, new HashSet<MeasureDesc>());
        iterator = storageEngine.search(context, sqlDigest, mockup.newTupleInfo(groups, aggregations));
        while (iterator.hasNext()) {
            ITuple tuple = iterator.next();
            System.out.println("Tuple = " + tuple);
            count++;
        }
    } finally {
        if (iterator != null)
            iterator.close();
    }
    return count;
}
 
Example #12
Source File: HybridInstance.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Override
public CapabilityResult isCapable(SQLDigest digest) {
    CapabilityResult result = new CapabilityResult();
    result.cost = Integer.MAX_VALUE;

    for (IRealization realization : getRealizations()) {
        CapabilityResult child = realization.isCapable(digest);
        if (child.capable) {
            result.capable = true;
            result.cost = Math.min(result.cost, child.cost);
            result.influences.addAll(child.influences);
        } else {
            result.incapableCause = child.incapableCause;
        }
    }

    if (result.cost > 0)
        result.cost--; // let hybrid win its children

    return result;
}
 
Example #13
Source File: GTCubeStorageQueryBase.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);

    List<CubeSegmentScanner> scanners = Lists.newArrayList();
    SegmentPruner segPruner = new SegmentPruner(sqlDigest.filter);
    for (CubeSegment cubeSeg : segPruner.listSegmentsForQuery(cubeInstance)) {
        CubeSegmentScanner scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), //
                request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), //
                request.getMetrics(), request.getDynFuncs(), //
                request.getFilter(), request.getHavingFilter(), request.getContext());
        if (!scanner.isSegmentSkipped())
            scanners.add(scanner);
    }

    if (scanners.isEmpty())
        return ITupleIterator.EMPTY_TUPLE_ITERATOR;

    return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
            request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo,
            request.getContext(), sqlDigest);
}
 
Example #14
Source File: GTCubeStorageQueryBase.java    From kylin with Apache License 2.0 6 votes vote down vote up
protected void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
    Map<String, List<MeasureDesc>> map = Maps.newHashMap();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        MeasureType<?> measureType = measure.getFunction().getMeasureType();

        String key = measureType.getClass().getCanonicalName();
        List<MeasureDesc> temp = null;
        if ((temp = map.get(key)) != null) {
            temp.add(measure);
        } else {
            map.put(key, Lists.<MeasureDesc> newArrayList(measure));
        }
    }

    for (List<MeasureDesc> sublist : map.values()) {
        sublist.get(0).getFunction().getMeasureType().adjustSqlDigest(sublist, sqlDigest);
    }
}
 
Example #15
Source File: GTCubeStorageQueryBase.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
protected void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions,
        Collection<FunctionDesc> metrics) {
    for (FunctionDesc func : sqlDigest.aggregations) {
        if (!func.isDimensionAsMetric() && !FunctionDesc.FUNC_GROUPING.equalsIgnoreCase(func.getExpression())) {
            // use the FunctionDesc from cube desc as much as possible, that has more info such as HLLC precision
            metrics.add(findAggrFuncFromCubeDesc(func));
        }
    }

    for (TblColRef column : sqlDigest.allColumns) {
        // skip measure columns
        if ((sqlDigest.metricColumns.contains(column) || sqlDigest.rtMetricColumns.contains(column))
                && !(sqlDigest.groupbyColumns.contains(column) || sqlDigest.filterColumns.contains(column)
                        || sqlDigest.rtDimensionColumns.contains(column))) {
            continue;
        }

        dimensions.add(column);
    }
}
 
Example #16
Source File: GTCubeStorageQueryBase.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
protected void notifyBeforeStorageQuery(SQLDigest sqlDigest) {
    Map<String, List<MeasureDesc>> map = Maps.newHashMap();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        MeasureType<?> measureType = measure.getFunction().getMeasureType();

        String key = measureType.getClass().getCanonicalName();
        List<MeasureDesc> temp = null;
        if ((temp = map.get(key)) != null) {
            temp.add(measure);
        } else {
            map.put(key, Lists.<MeasureDesc> newArrayList(measure));
        }
    }

    for (List<MeasureDesc> sublist : map.values()) {
        sublist.get(0).getFunction().getMeasureType().adjustSqlDigest(sublist, sqlDigest);
    }
}
 
Example #17
Source File: StorageTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private int search(List<TblColRef> groups, List<FunctionDesc> aggregations, TupleFilter filter, StorageContext context) {
    int count = 0;
    ITupleIterator iterator = null;
    try {
        SQLDigest sqlDigest = new SQLDigest("default.test_kylin_fact", filter, null, Collections.<TblColRef> emptySet(), groups, Collections.<TblColRef> emptySet(), Collections.<TblColRef> emptySet(), aggregations);
        iterator = storageEngine.search(context, sqlDigest);
        while (iterator.hasNext()) {
            ITuple tuple = iterator.next();
            System.out.println("Tuple = " + tuple);
            count++;
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (iterator != null) {
            iterator.close();
        }
    }
    return count;
}
 
Example #18
Source File: CubeCapabilityChecker.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private static void tryCustomMeasureTypes(Collection<TblColRef> unmatchedDimensions,
        Collection<FunctionDesc> unmatchedAggregations, SQLDigest digest, CubeInstance cube,
        CapabilityResult result) {
    CubeDesc cubeDesc = cube.getDescriptor();
    List<String> influencingMeasures = Lists.newArrayList();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        //            if (unmatchedDimensions.isEmpty() && unmatchedAggregations.isEmpty())
        //                break;

        MeasureType<?> measureType = measure.getFunction().getMeasureType();
        if (measureType instanceof BasicMeasureType)
            continue;

        CapabilityInfluence inf = measureType.influenceCapabilityCheck(unmatchedDimensions, unmatchedAggregations,
                digest, measure);
        if (inf != null) {
            result.influences.add(inf);
            influencingMeasures.add(measure.getName() + "@" + measureType.getClass());
        }
    }
    if (influencingMeasures.size() != 0)
        logger.info("Cube {} CapabilityInfluences: {}", cube.getCanonicalName(),
                StringUtils.join(influencingMeasures, ","));
}
 
Example #19
Source File: CubeInstance.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public int getCost(SQLDigest digest) {
    int calculatedCost = cost;

    //the number of dimensions is not as accurate as number of row key cols
    calculatedCost += getRowKeyColumnCount() * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;

    for (JoinTableDesc joinTable : this.getModel().getJoinTables()) {
        // more tables, more cost
        if (joinTable.getJoin().isInnerJoin()) {
            // inner join cost is bigger than left join, as it will filter some records
            calculatedCost += COST_WEIGHT_INNER_JOIN;
        }
    }

    return calculatedCost;
}
 
Example #20
Source File: GTCubeStorageQueryBase.java    From kylin with Apache License 2.0 6 votes vote down vote up
protected void buildDimensionsAndMetrics(SQLDigest sqlDigest, Collection<TblColRef> dimensions,
        Collection<FunctionDesc> metrics) {
    for (FunctionDesc func : sqlDigest.aggregations) {
        if (!func.isDimensionAsMetric() && !FunctionDesc.FUNC_GROUPING.equalsIgnoreCase(func.getExpression())) {
            // use the FunctionDesc from cube desc as much as possible, that has more info such as HLLC precision
            metrics.add(findAggrFuncFromCubeDesc(func));
        }
    }

    for (TblColRef column : sqlDigest.allColumns) {
        // skip measure columns
        if ((sqlDigest.metricColumns.contains(column) || sqlDigest.rtMetricColumns.contains(column))
                && !(sqlDigest.groupbyColumns.contains(column) || sqlDigest.filterColumns.contains(column)
                        || sqlDigest.rtDimensionColumns.contains(column))) {
            continue;
        }

        dimensions.add(column);
    }
}
 
Example #21
Source File: OLAPEnumerator.java    From kylin with Apache License 2.0 6 votes vote down vote up
private ITupleIterator queryStorage() {

        logger.debug("query storage...");
        // bind dynamic variables
        olapContext.bindVariable(optiqContext);

        // If olapContext is cached, then inherit it.
        if (!olapContext.isBorrowedContext) {
            olapContext.resetSQLDigest();
        }
        SQLDigest sqlDigest = olapContext.getSQLDigest();

        // query storage engine
        IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
        ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
                olapContext.returnTupleInfo);
        if (logger.isDebugEnabled()) {
            logger.debug("return TupleIterator...");
        }

        return iterator;
    }
 
Example #22
Source File: ExtendedColumnMeasureType.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
public void adjustSqlDigest(List<MeasureDesc> measureDescs, SQLDigest sqlDigest) {
    for (MeasureDesc measureDesc : measureDescs) {
        if (!sqlDigest.involvedMeasure.contains(measureDesc)) {
            continue;
        }
        FunctionDesc extendColumnFunc = measureDesc.getFunction();
        List<TblColRef> hosts = getExtendedColumnHosts(extendColumnFunc);
        TblColRef extended = getExtendedColumn(extendColumnFunc);

        if (!sqlDigest.groupbyColumns.contains(extended)) {
            continue;
        }

        sqlDigest.aggregations.add(extendColumnFunc);
        sqlDigest.groupbyColumns.remove(extended);
        sqlDigest.groupbyColumns.addAll(hosts);
        sqlDigest.metricColumns.add(extended);
    }
}
 
Example #23
Source File: OLAPEnumerator.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private ITupleIterator queryStorage() {

        logger.debug("query storage...");
        // bind dynamic variables
        olapContext.bindVariable(optiqContext);

        // If olapContext is cached, then inherit it.
        if (!olapContext.isBorrowedContext) {
            olapContext.resetSQLDigest();
        }
        SQLDigest sqlDigest = olapContext.getSQLDigest();

        // query storage engine
        IStorageQuery storageEngine = StorageFactory.createQuery(olapContext.realization);
        ITupleIterator iterator = storageEngine.search(olapContext.storageContext, sqlDigest,
                olapContext.returnTupleInfo);
        if (logger.isDebugEnabled()) {
            logger.debug("return TupleIterator...");
        }

        return iterator;
    }
 
Example #24
Source File: GTCubeStorageQueryBase.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
public ITupleIterator search(StorageContext context, SQLDigest sqlDigest, TupleInfo returnTupleInfo) {
    GTCubeStorageQueryRequest request = getStorageQueryRequest(context, sqlDigest, returnTupleInfo);

    List<CubeSegmentScanner> scanners = Lists.newArrayList();
    SegmentPruner segPruner = new SegmentPruner(sqlDigest.filter);
    for (CubeSegment cubeSeg : segPruner.listSegmentsForQuery(cubeInstance)) {
        CubeSegmentScanner scanner = new CubeSegmentScanner(cubeSeg, request.getCuboid(), request.getDimensions(), //
                request.getGroups(), request.getDynGroups(), request.getDynGroupExprs(), //
                request.getMetrics(), request.getDynFuncs(), //
                request.getFilter(), request.getHavingFilter(), request.getContext());
        if (!scanner.isSegmentSkipped())
            scanners.add(scanner);
    }

    if (scanners.isEmpty())
        return ITupleIterator.EMPTY_TUPLE_ITERATOR;

    return new SequentialCubeTupleIterator(scanners, request.getCuboid(), request.getDimensions(),
            request.getDynGroups(), request.getGroups(), request.getMetrics(), returnTupleInfo,
            request.getContext(), sqlDigest);
}
 
Example #25
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private void buildDimensionsAndMetrics(Collection<TblColRef> dimensions, Collection<FunctionDesc> metrics, SQLDigest sqlDigest) {

        for (FunctionDesc func : sqlDigest.aggregations) {
            if (!func.isDimensionAsMetric()) {
                metrics.add(func);
            }
        }

        for (TblColRef column : sqlDigest.allColumns) {
            // skip measure columns
            if (sqlDigest.metricColumns.contains(column)) {
                continue;
            }
            dimensions.add(column);
        }
    }
 
Example #26
Source File: OLAPSortRel.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
public void implementOLAP(OLAPImplementor implementor) {
    implementor.fixSharedOlapTableScan(this);
    implementor.visitChild(getInput(), this);

    this.context = implementor.getContext();
    this.columnRowType = buildColumnRowType();

    for (RelFieldCollation fieldCollation : this.collation.getFieldCollations()) {
        int index = fieldCollation.getFieldIndex();
        SQLDigest.OrderEnum order = getOrderEnum(fieldCollation.getDirection());
        OLAPRel olapChild = (OLAPRel) this.getInput();
        TblColRef orderCol = olapChild.getColumnRowType().getAllColumns().get(index);
        this.context.addSort(orderCol, order);
        this.context.storageContext.markSort();

    }
}
 
Example #27
Source File: SequentialCubeTupleIterator.java    From kylin with Apache License 2.0 6 votes vote down vote up
public SequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid,
        Set<TblColRef> selectedDimensions, List<TblColRef> rtGroups, Set<TblColRef> groups, //
        Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context, SQLDigest sqlDigest) {
    this.context = context;
    this.scanners = scanners;

    Set<TblColRef> selectedDims = Sets.newHashSet(selectedDimensions);
    selectedDims.addAll(rtGroups);

    segmentCubeTupleIterators = Lists.newArrayList();
    for (CubeSegmentScanner scanner : scanners) {
        segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDims, selectedMetrics, returnTupleInfo, context));
    }

    if (context.mergeSortPartitionResults() && !sqlDigest.isRawQuery) {
        //query with limit
        logger.info("Using SortedIteratorMergerWithLimit to merge segment results");
        Iterator<Iterator<ITuple>> transformed = (Iterator<Iterator<ITuple>>) (Iterator<?>) segmentCubeTupleIterators.iterator();
        tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, groups, returnTupleInfo)).getIterator();
    } else {
        //normal case
        logger.info("Using Iterators.concat to merge segment results");
        tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
    }
}
 
Example #28
Source File: SequentialCubeTupleIterator.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public SequentialCubeTupleIterator(List<CubeSegmentScanner> scanners, Cuboid cuboid,
        Set<TblColRef> selectedDimensions, List<TblColRef> rtGroups, Set<TblColRef> groups, //
        Set<FunctionDesc> selectedMetrics, TupleInfo returnTupleInfo, StorageContext context, SQLDigest sqlDigest) {
    this.context = context;
    this.scanners = scanners;

    Set<TblColRef> selectedDims = Sets.newHashSet(selectedDimensions);
    selectedDims.addAll(rtGroups);

    segmentCubeTupleIterators = Lists.newArrayList();
    for (CubeSegmentScanner scanner : scanners) {
        segmentCubeTupleIterators.add(new SegmentCubeTupleIterator(scanner, cuboid, selectedDims, selectedMetrics, returnTupleInfo, context));
    }

    if (context.mergeSortPartitionResults() && !sqlDigest.isRawQuery) {
        //query with limit
        logger.info("Using SortedIteratorMergerWithLimit to merge segment results");
        Iterator<Iterator<ITuple>> transformed = (Iterator<Iterator<ITuple>>) (Iterator<?>) segmentCubeTupleIterators.iterator();
        tupleIterator = new SortedIteratorMergerWithLimit<ITuple>(transformed, context.getFinalPushDownLimit(), getTupleDimensionComparator(cuboid, groups, returnTupleInfo)).getIterator();
    } else {
        //normal case
        logger.info("Using Iterators.concat to merge segment results");
        tupleIterator = Iterators.concat(segmentCubeTupleIterators.iterator());
    }
}
 
Example #29
Source File: CubeCapabilityChecker.java    From Kylin with Apache License 2.0 5 votes vote down vote up
public static boolean check(CubeInstance cube, SQLDigest digest, boolean allowWeekMatch) {

        // retrieve members from olapContext
        Collection<TblColRef> dimensionColumns = CubeDimensionDeriver.getDimensionColumns(digest.groupbyColumns, digest.filterColumns);
        Collection<FunctionDesc> functions = digest.aggregations;
        Collection<TblColRef> metricsColumns = digest.metricColumns;
        Collection<JoinDesc> joins = digest.joinDescs;

        // match dimensions & aggregations & joins

        boolean isOnline = cube.isReady();

        boolean matchDimensions = isMatchedWithDimensions(dimensionColumns, cube);
        boolean matchAggregation = isMatchedWithAggregations(functions, cube);
        boolean matchJoin = isMatchedWithJoins(joins, cube);

        // Some cubes are not "perfectly" match, but still save them in case of usage
        if (allowWeekMatch && isOnline && matchDimensions && !matchAggregation && matchJoin) {
            // sometimes metrics are indeed dimensions
            // e.g. select min(cal_dt) from ..., where cal_dt is actually a dimension
            if (isWeaklyMatchedWithAggregations(functions, metricsColumns, cube)) {
                logger.info("Weakly matched cube found " + cube.getName());
                return true;
            }
        }

        if (!isOnline || !matchDimensions || !matchAggregation || !matchJoin) {
            logger.info("Exclude cube " + cube.getName() + " because " + " isOnlne=" + isOnline + ",matchDimensions=" + matchDimensions + ",matchAggregation=" + matchAggregation + ",matchJoin=" + matchJoin);
            return false;
        }

        return true;
    }
 
Example #30
Source File: HybridStorageQuery.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Override
public ITupleIterator search(final StorageContext context, final SQLDigest sqlDigest, final TupleInfo returnTupleInfo) {
    List<ITupleIterator> tupleIterators = Lists.newArrayList();
    for (int i = 0; i < realizations.length; i++) {
        if (realizations[i].isReady() && realizations[i].isCapable(sqlDigest).capable) {
            ITupleIterator dataIterator = storageEngines[i].search(context, sqlDigest, returnTupleInfo);
            tupleIterators.add(dataIterator);
        }
    }
    // combine tuple iterator
    return new CompoundTupleIterator(tupleIterators);
}