Java Code Examples for org.apache.kylin.metadata.model.FunctionDesc#isCountDistinct()

public static String createMeasureStatement(List<MeasureDesc> measureList) {
    StringBuilder sql = new StringBuilder();

    for (MeasureDesc measureDesc : measureList) {
        FunctionDesc functionDesc = measureDesc.getFunction();
        if (functionDesc.isSum() || functionDesc.isMax() || functionDesc.isMin()) {
            sql.append("," + functionDesc.getExpression() + "(" + functionDesc.getParameter().getValue() + ")\n");
            break;
        } else if (functionDesc.isCountDistinct()) {
            sql.append(",COUNT" + "(DISTINCT " + functionDesc.getParameter().getValue() + ")\n");
            break;
        }
    }

    return sql.toString();
}
 

private static boolean isWeaklyMatchedWithAggregations(Collection<FunctionDesc> aggregations, Collection<TblColRef> metricColumns, CubeInstance cube) {
    CubeDesc cubeDesc = cube.getDescriptor();
    Collection<FunctionDesc> cubeFuncs = cubeDesc.listAllFunctions();

    boolean matched = true;
    for (FunctionDesc functionDesc : aggregations) {
        if (cubeFuncs.contains(functionDesc))
            continue;

        // only inverted-index cube does not have count, and let calcite handle in this case
        if (functionDesc.isCount())
            continue;

        if (functionDesc.isCountDistinct()) // calcite can not handle distinct count
            matched = false;

        TblColRef col = functionDesc.selectTblColRef(metricColumns, cubeDesc.getFactTable());
        if (col == null || !cubeDesc.listDimensionColumnsIncludingDerived().contains(col)) {
            matched = false;
        }
    }
    return matched;
}
 

private void validateReturnType(ValidateContext context, CubeDesc cube, FunctionDesc funcDesc) {

        String func = funcDesc.getExpression();
        DataType rtype = funcDesc.getReturnDataType();

        if (funcDesc.isCount()) {
            if (rtype.isIntegerFamily() == false) {
                context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of " + DataType.INTEGER_FAMILY);
            }
        } else if (funcDesc.isCountDistinct()) {
            if (rtype.isHLLC() == false && funcDesc.isHolisticCountDistinct() == false) {
                context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be hllc(10), hllc(12) etc.");
            }
        } else if (funcDesc.isMax() || funcDesc.isMin() || funcDesc.isSum()) {
            if (rtype.isNumberFamily() == false) {
                context.addResult(ResultLevel.ERROR, "Return type for function " + func + " must be one of " + DataType.NUMBER_FAMILY);
            }
        } else {
            if (StringUtils.equalsIgnoreCase(KylinConfig.getInstanceFromEnv().getProperty(KEY_IGNORE_UNKNOWN_FUNC, "false"), "false")) {
                context.addResult(ResultLevel.ERROR, "Unrecognized function: [" + func + "]");
            }
        }

    }
 

private AggregateCall rewriteAggregateCall(AggregateCall aggCall, FunctionDesc func) {

        // rebuild parameters
        List<Integer> newArgList = new ArrayList<Integer>(1);
        String fieldName = func.getRewriteFieldName();
        RelDataTypeField field = getChild().getRowType().getField(fieldName, true);
        newArgList.add(field.getIndex());

        // rebuild function
        RelDataType fieldType = aggCall.getType();
        Aggregation newAgg = aggCall.getAggregation();
        if (func.isCountDistinct()) {
            newAgg = createHyperLogLogAggFunction(fieldType);
        } else if (func.isCount()) {
            newAgg = new SqlSumEmptyIsZeroAggFunction(fieldType);
        }

        // rebuild aggregate call
        AggregateCall newAggCall = new AggregateCall(newAgg, false, newArgList, fieldType, newAgg.getName());

        // To make sure specified type matches the inferReturnType, or otherwise
        // there will be assertion failure in optiq
        // The problem is BIGINT != BIGINT NOT NULL
        // Details see https://github.scm.corp.ebay.com/Kylin/Kylin/issues/323
        SqlAggFunction aggFunction = (SqlAggFunction) newAggCall.getAggregation();
        AggCallBinding callBinding = newAggCall.createBinding(this);
        RelDataType inferReturnType = aggFunction.inferReturnType(callBinding);

        return new AggregateCall(newAgg, false, newArgList, inferReturnType, newAgg.getName());
    }
 

public boolean hasMemHungryCountDistinct() {
    for (int i = projectionIndex.nextSetBit(0); i >= 0; i = projectionIndex.nextSetBit(i + 1)) {
        FunctionDesc func = measures[i].getFunction();
        if (func.isCountDistinct() && !func.isHolisticCountDistinct()) {
            return true;
        }
    }
    return false;
}
 

/**
 * measure comes from query engine, does not contain enough information
 *
 * @param measures
 * @param columns
 */
private void rewriteMeasureParameters(List<FunctionDesc> measures, List<TblColRef> columns) {
    for (FunctionDesc functionDesc : measures) {
        if (functionDesc.isCount()) {
            functionDesc.setReturnType("bigint");
            functionDesc.setReturnDataType(DataType.getInstance(functionDesc.getReturnType()));
        } else {
            boolean updated = false;
            for (TblColRef column : columns) {
                if (column.isSameAs(factTableName, functionDesc.getParameter().getValue())) {
                    if (functionDesc.isCountDistinct()) {
                        //TODO: default precision might need be configurable
                        String iiDefaultHLLC = "hllc10";
                        functionDesc.setReturnType(iiDefaultHLLC);
                        functionDesc.setReturnDataType(DataType.getInstance(iiDefaultHLLC));
                    } else {
                        functionDesc.setReturnType(column.getColumn().getType().toString());
                        functionDesc.setReturnDataType(DataType.getInstance(functionDesc.getReturnType()));
                    }
                    functionDesc.getParameter().setColRefs(ImmutableList.of(column));
                    updated = true;
                    break;
                }
            }
            if (!updated) {
                throw new RuntimeException("Func " + functionDesc + " is not related to any column in fact table " + factTableName);
            }
        }
    }
}
 

public static EndpointAggregators fromFunctions(TableRecordInfo tableInfo, List<FunctionDesc> metrics) {
    String[] funcNames = new String[metrics.size()];
    String[] dataTypes = new String[metrics.size()];
    MetricInfo[] metricInfos = new MetricInfo[metrics.size()];

    for (int i = 0; i < metrics.size(); i++) {
        FunctionDesc functionDesc = metrics.get(i);

        //TODO: what if funcionDesc's type is different from tablDesc? cause scale difference
        funcNames[i] = functionDesc.getExpression();
        dataTypes[i] = functionDesc.getReturnType();

        if (functionDesc.isCount()) {
            metricInfos[i] = new MetricInfo(MetricType.Count);
        } else if (functionDesc.isDimensionAsMetric()) {
            metricInfos[i] = new MetricInfo(MetricType.DimensionAsMetric);
        } else {
            int index = tableInfo.findFactTableColumn(functionDesc.getParameter().getValue());
            if (index < 0) {
                throw new IllegalStateException("Column " + functionDesc.getParameter().getValue() + " is not found in II");
            }

            if (functionDesc.isCountDistinct()) {
                metricInfos[i] = new MetricInfo(MetricType.DistinctCount, index, functionDesc.getReturnDataType().getPrecision());
            } else {
                metricInfos[i] = new MetricInfo(MetricType.Normal, index);
            }
        }
    }

    return new EndpointAggregators(funcNames, dataTypes, metricInfos, tableInfo.getDigest());
}