Java Code Examples for org.apache.kylin.metadata.model.PartitionDesc#getPartitionDateColumnRef()

/**
 * Use the segment start time as the map key, the time unit depends on the partition columns
 * If the partition_time_column is null, the unit is day;
 *                            otherwise, the unit is second
 */
private String getSegmentStartTime(CubeSegment segment) {
    long startTime = segment.getTSRange().start.v;
    DataModelDesc model = segment.getModel();
    PartitionDesc partitionDesc = model.getPartitionDesc();
    if (partitionDesc == null || !partitionDesc.isPartitioned()) {
        return "0";
    } else if (partitionDesc.partitionColumnIsTimeMillis()) {
        return "" + startTime;
    } else if (partitionDesc.getPartitionTimeColumnRef() != null) {
        return "" + startTime / 1000L;
    } else if (partitionDesc.getPartitionDateColumnRef() != null) {
        return "" + startTime / 86400000L;
    }
    return "0";
}
 

private static TblColRef getPartitionCol(IRealization realization) {
    String modelName = realization.getModel().getName();
    DataModelDesc dataModelDesc = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getDataModelDesc(modelName);
    PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
    Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
    TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef();
    Preconditions.checkArgument(partitionColRef != null,
            "getPartitionDateColumnRef for " + realization + " is null");
    return partitionColRef;
}
 

private static TblColRef getPartitionCol(IRealization realization) {
    String modelName = realization.getModel().getName();
    DataModelDesc dataModelDesc = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getDataModelDesc(modelName);
    PartitionDesc partitionDesc = dataModelDesc.getPartitionDesc();
    Preconditions.checkArgument(partitionDesc != null, "PartitionDesc for " + realization + " is null!");
    TblColRef partitionColRef = partitionDesc.getPartitionDateColumnRef();
    Preconditions.checkArgument(partitionColRef != null,
            "getPartitionDateColumnRef for " + realization + " is null");
    return partitionColRef;
}
 

private boolean isExactAggregation(StorageContext context, Cuboid cuboid, Collection<TblColRef> groups,
        Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation,
        Collection<FunctionDesc> functionDescs, List<SQLDigest.SQLCall> aggrSQLCalls, boolean groupByExpression) {
    if (context.isNeedStorageAggregation()) {
        logger.info("exactAggregation is false because need storage aggregation");
        return false;
    }

    if (cuboid.requirePostAggregation()) {
        logger.info("exactAggregation is false because cuboid {}=>{}", cuboid.getInputID(), cuboid.getId());
        return false;
    }

    // derived aggregation is bad, unless expanded columns are already in group by
    if (!groups.containsAll(derivedPostAggregation)) {
        logger.info("exactAggregation is false because derived column require post aggregation: {}",
                derivedPostAggregation);
        return false;
    }

    // other columns (from filter) is bad, unless they are ensured to have single value
    if (!singleValuesD.containsAll(othersD)) {
        logger.info("exactAggregation is false because some column not on group by: {} (single value column: {})",
                othersD, singleValuesD);
        return false;
    }

    //for DimensionAsMetric like max(cal_dt), the dimension column maybe not in real group by
    for (FunctionDesc functionDesc : functionDescs) {
        if (functionDesc.isDimensionAsMetric()) {
            logger.info("exactAggregation is false because has DimensionAsMetric");
            return false;
        }
    }
    for (SQLDigest.SQLCall aggrSQLCall : aggrSQLCalls) {
        if (aggrSQLCall.function.equals(BitmapMeasureType.FUNC_INTERSECT_COUNT_DISTINCT)
        || aggrSQLCall.function.equals(BitmapMeasureType.FUNC_INTERSECT_VALUE)) {
            logger.info("exactAggregation is false because has INTERSECT_COUNT OR INTERSECT_VALUE");
            return false;
        }
    }

    // for partitioned cube, the partition column must belong to group by or has single value
    PartitionDesc partDesc = cuboid.getCubeDesc().getModel().getPartitionDesc();
    if (partDesc.isPartitioned()) {
        TblColRef col = partDesc.getPartitionDateColumnRef();
        if (!groups.contains(col) && !singleValuesD.contains(col)) {
            logger.info("exactAggregation is false because cube is partitioned and {} is not on group by", col);
            return false;
        }
    }

    // for group by expression like: group by seller_id/100. seller_id_1(200) get 2, seller_id_2(201) also get 2, so can't aggregate exactly
    if (groupByExpression) {
        logger.info("exactAggregation is false because group by expression");
        return false;
    }

    logger.info("exactAggregation is true, cuboid id is {}", cuboid.getId());
    return true;
}
 

private boolean isExactAggregation(StorageContext context, Cuboid cuboid, Collection<TblColRef> groups,
        Set<TblColRef> othersD, Set<TblColRef> singleValuesD, Set<TblColRef> derivedPostAggregation,
        Collection<FunctionDesc> functionDescs, List<SQLDigest.SQLCall> aggrSQLCalls, boolean groupByExpression) {
    if (context.isNeedStorageAggregation()) {
        logger.info("exactAggregation is false because need storage aggregation");
        return false;
    }

    if (cuboid.requirePostAggregation()) {
        logger.info("exactAggregation is false because cuboid {}=>{}", cuboid.getInputID(), cuboid.getId());
        return false;
    }

    // derived aggregation is bad, unless expanded columns are already in group by
    if (!groups.containsAll(derivedPostAggregation)) {
        logger.info("exactAggregation is false because derived column require post aggregation: {}",
                derivedPostAggregation);
        return false;
    }

    // other columns (from filter) is bad, unless they are ensured to have single value
    if (!singleValuesD.containsAll(othersD)) {
        logger.info("exactAggregation is false because some column not on group by: {} (single value column: {})",
                othersD, singleValuesD);
        return false;
    }

    //for DimensionAsMetric like max(cal_dt), the dimension column maybe not in real group by
    for (FunctionDesc functionDesc : functionDescs) {
        if (functionDesc.isDimensionAsMetric()) {
            logger.info("exactAggregation is false because has DimensionAsMetric");
            return false;
        }
    }
    for (SQLDigest.SQLCall aggrSQLCall : aggrSQLCalls) {
        if (aggrSQLCall.function.equals(BitmapMeasureType.FUNC_INTERSECT_COUNT_DISTINCT)
        || aggrSQLCall.function.equals(BitmapMeasureType.FUNC_INTERSECT_VALUE)) {
            logger.info("exactAggregation is false because has INTERSECT_COUNT OR INTERSECT_VALUE");
            return false;
        }
    }

    // for partitioned cube, the partition column must belong to group by or has single value
    PartitionDesc partDesc = cuboid.getCubeDesc().getModel().getPartitionDesc();
    if (partDesc.isPartitioned()) {
        TblColRef col = partDesc.getPartitionDateColumnRef();
        if (!groups.contains(col) && !singleValuesD.contains(col)) {
            logger.info("exactAggregation is false because cube is partitioned and {} is not on group by", col);
            return false;
        }
    }

    // for group by expression like: group by seller_id/100. seller_id_1(200) get 2, seller_id_2(201) also get 2, so can't aggregate exactly
    if (groupByExpression) {
        logger.info("exactAggregation is false because group by expression");
        return false;
    }

    logger.info("exactAggregation is true, cuboid id is {}", cuboid.getId());
    return true;
}
 

public static DataModelDesc dealWithMappingForModel(DataModelDesc other,
        Map<String, TableSchemaUpdateMapping> mappings) {
    // For filter condition, not support
    if (!Strings.isNullOrEmpty(other.getFilterCondition())) {
        throw new UnsupportedOperationException("Cannot deal with filter condition " + other.getFilterCondition());
    }

    DataModelDesc copy = DataModelDesc.getCopyOf(other);
    copy.setLastModified(other.getLastModified());

    // mapping for root fact table identity
    TableSchemaUpdateMapping rootMapping = getTableSchemaUpdateMapping(mappings, other.getRootFactTableName());
    if (rootMapping != null) {
        TableDesc rootFactTable = other.getRootFactTable().getTableDesc();
        copy.setRootFactTableName(
                rootMapping.getTableIdentity(rootFactTable.getDatabase(), rootFactTable.getName()));
    }

    // mapping for joins
    JoinTableDesc[] joinTables = other.getJoinTables();
    JoinTableDesc[] joinTablesCopy = new JoinTableDesc[joinTables.length];
    for (int i = 0; i < joinTables.length; i++) {
        JoinTableDesc joinTable = joinTables[i];
        joinTablesCopy[i] = JoinTableDesc.getCopyOf(joinTable);
        String tableIdentity = joinTable.getTable();
        TableSchemaUpdateMapping mapping = getTableSchemaUpdateMapping(mappings, tableIdentity);
        if (mapping != null && mapping.isTableIdentityChanged()) {
            joinTablesCopy[i].setTable(mapping.getTableIdentity(tableIdentity));
        }
    }
    copy.setJoinTables(joinTablesCopy);

    // mapping for partition columns
    PartitionDesc partDesc = other.getPartitionDesc();
    PartitionDesc partCopy = PartitionDesc.getCopyOf(partDesc);
    if (partDesc.getPartitionDateColumnRef() != null) {
        partCopy.setPartitionDateColumn(
                replacePartitionCol(partDesc.getPartitionDateColumnRef().getCanonicalName(), mappings));
    }
    if (partDesc.getPartitionTimeColumnRef() != null) {
        partCopy.setPartitionTimeColumn(
                replacePartitionCol(partDesc.getPartitionTimeColumnRef().getCanonicalName(), mappings));
    }
    copy.setPartitionDesc(partCopy);

    return copy;
}