org.apache.kylin.cube.model.RowKeyDesc Java Examples

public static boolean isValid(CubeDesc cube, long cuboidID) {
    RowKeyDesc rowkey = cube.getRowkey();

    if (cuboidID < 0) {
        throw new IllegalArgumentException("Cuboid " + cuboidID + " should be greater than 0");
    }

    if (checkBaseCuboid(rowkey, cuboidID)) {
        return true;
    }

    if (checkMandatoryColumns(rowkey, cuboidID) == false) {
        return false;
    }

    if (checkAggregationGroup(rowkey, cuboidID) == false) {
        return false;
    }

    if (checkHierarchy(rowkey, cuboidID) == false) {
        return false;
    }

    return true;
}
 

private static int mathCalcCuboidCount_aggrGroup(RowKeyDesc rowkey, AggrGroupMask aggrGroupMask, boolean hasTail) {
    long groupMask = aggrGroupMask.groupMask;
    int n = mathCalcCuboidCount_combination(rowkey, groupMask);
    n -= 2; // exclude group all 1 and all 0

    long nonUniqueMask = groupMask & (~aggrGroupMask.uniqueMask);
    if (nonUniqueMask > 0) {
        // exclude duplicates caused by non-unique columns
        // FIXME this assumes non-unique masks consolidates in ONE following group which maybe not be true
        n -= mathCalcCuboidCount_combination(rowkey, nonUniqueMask) - 1; // exclude all 0
    }

    if (hasTail) {
        n *= 2; // tail being 1 and 0
        n += 2; // +1 for group all 1 and tail 0; +1 for group all 0 and tail 1
    }

    return n;
}
 

private static int mathCalcCuboidCount_combination(RowKeyDesc rowkey, long colMask) {
    if (colMask == 0) // no column selected
        return 0;

    int count = 1;

    for (HierarchyMask hierMask : rowkey.getHierarchyMasks()) {
        long hierBits = colMask & hierMask.fullMask;
        if (hierBits != 0) {
            count *= Long.bitCount(hierBits) + 1; // +1 is for all-zero case
            colMask &= ~hierBits;
        }
    }

    count *= Math.pow(2, Long.bitCount(colMask));

    return count;
}
 

private void generateZeroTailBase(long cuboid, Collection<Long> result) {
    RowKeyDesc rowkey = cubeDef.getRowkey();

    long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask();

    for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
        if ((cuboidWithoutMandatory & mask.groupMask) == mask.groupMask && (cuboidWithoutMandatory & mask.leftoverMask) == mask.leftoverMask) {
            long zeroTail = rowkey.getMandatoryColumnMask() | mask.groupMask;
            if (zeroTail > 0 && zeroTail != cuboid) {
                result.add(zeroTail);
            }
        }
        if ((cuboidWithoutMandatory & mask.uniqueMask) > 0)
            break;
    }
}
 

@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null || rcd.length == 0) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getColumn() == null || rd.getColumn().length() == 0) {
            context.addResult(ResultLevel.ERROR, "Rowkey column empty");
        }

    }

}
 

public static CubeDesc generateKylinCubeDesc(String tableName, int storageType,
        List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc,
        AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) {
    CubeDesc desc = new CubeDesc();
    desc.setName(tableName.replace('.', '_'));
    desc.setModelName(tableName.replace('.', '_'));
    desc.setDescription("");
    desc.setLastModified(0L);
    desc.setDimensions(dimensionDescList);
    desc.setMeasures(measureDescList);
    desc.setRowkey(rowKeyDesc);
    desc.setHbaseMapping(hBaseMapping);
    desc.setNotifyList(Lists.<String> newArrayList());
    desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString()));
    desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L, 7776000000L, 31104000000L });
    desc.setEngineType(IEngineAware.ID_MR_V2);
    desc.setStorageType(storageType);
    desc.setAggregationGroups(Lists.newArrayList(aggGroup));
    desc.getOverrideKylinProps().putAll(overrideProperties);
    desc.updateRandomUuid();
    return desc;
}
 

public Collection<Long> findSmallerSibling(long cuboid) {
    if (!Cuboid.isValid(cubeDef, cuboid)) {
        return Collections.emptyList();
    }

    RowKeyDesc rowkey = cubeDef.getRowkey();

    // do combination in all related groups
    long groupAllBitMask = 0;
    for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
        if ((mask.groupMask & cuboid) > 0) {
            groupAllBitMask |= mask.groupMask;
        }
    }

    long groupBitValue = cuboid & groupAllBitMask;
    long leftBitValue = cuboid & ~groupAllBitMask;
    long[] groupOneBits = bits(groupAllBitMask);

    Collection<Long> siblings = new HashSet<Long>();
    combination(cuboid, siblings, groupOneBits, 0, leftBitValue, Long.bitCount(groupBitValue));
    return siblings;
}
 

private static boolean checkHierarchy(RowKeyDesc rowkey, long cuboidID) {
    List<HierarchyMask> hierarchyMaskList = rowkey.getHierarchyMasks();
    // if no hierarchy defined in metadata
    if (hierarchyMaskList == null || hierarchyMaskList.size() == 0) {
        return true;
    }

    hier: for (HierarchyMask hierarchyMasks : hierarchyMaskList) {
        long result = cuboidID & hierarchyMasks.fullMask;
        if (result > 0) {
            // if match one of the hierarchy constrains, return true;
            for (long mask : hierarchyMasks.allMasks) {
                if (result == mask) {
                    continue hier;
                }
            }
            return false;
        }
    }
    return true;
}
 

@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null || rcd.length == 0) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getColumn() == null || rd.getColumn().length() == 0) {
            context.addResult(ResultLevel.ERROR, "Rowkey column empty");
        }

    }

}
 

public static CubeDesc generateKylinCubeDesc(String tableName, int storageType,
        List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc,
        AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) {
    CubeDesc desc = new CubeDesc();
    desc.setName(tableName.replace('.', '_'));
    desc.setModelName(tableName.replace('.', '_'));
    desc.setDescription("");
    desc.setLastModified(0L);
    desc.setDimensions(dimensionDescList);
    desc.setMeasures(measureDescList);
    desc.setRowkey(rowKeyDesc);
    desc.setHbaseMapping(hBaseMapping);
    desc.setNotifyList(Lists.<String> newArrayList());
    desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString()));
    desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L });
    desc.setEngineType(IEngineAware.ID_MR_V2);
    desc.setStorageType(storageType);
    desc.setAggregationGroups(Lists.newArrayList(aggGroup));
    desc.getOverrideKylinProps().putAll(overrideProperties);
    desc.updateRandomUuid();
    return desc;
}
 

private void init(Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics) {
    this.dimensions = new TblColRef[selectedDimensions.size()];
    this.metrics = new FunctionDesc[selectedMetrics.size()];
    this.measures = new MeasureDesc[selectedMetrics.size()];
    this.dimDataTypes = new DataType[dimensions.length];
    this.metricsDataTypes = new DataType[metrics.length];
    // sort dimensions according to the rowKey definition
    dimColIdxMap = Maps.newHashMap();
    RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
    int colIdx = 0;
    for (RowKeyColDesc rowKeyColDesc : rowKeyDesc.getRowKeyColumns()) {
        TblColRef dimension = rowKeyColDesc.getColRef();
        if (selectedDimensions.contains(dimension)) {
            dimensions[colIdx] = dimension;
            dimDataTypes[colIdx] = dimension.getType();
            dimColIdxMap.put(dimension, colIdx);
            colIdx++;
        }
    }

    nDimensions = colIdx;

    colIdx = 0;
    // metrics
    metricsColIdxMap = Maps.newHashMap();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc func = measure.getFunction();
        if (selectedMetrics.contains(func)) {
            metrics[colIdx] = func;
            measures[colIdx] = measure;
            metricsColIdxMap.put(func.getParameter().getColRef(), colIdx);
            metricsDataTypes[colIdx] = func.getReturnDataType();
            colIdx++;
        }
    }

    nMetrics = colIdx;
}
 

private static boolean checkMandatoryColumns(RowKeyDesc rowkey, long cuboidID) {
    long mandatoryColumnMask = rowkey.getMandatoryColumnMask();

    // note the all-zero cuboid (except for mandatory) is not valid
    if (cuboidID <= mandatoryColumnMask)
        return false;

    return (cuboidID & mandatoryColumnMask) == mandatoryColumnMask;
}
 

private static boolean checkBaseCuboid(RowKeyDesc rowkey, long cuboidID) {
    long baseCuboidId = rowkey.getFullMask();
    if (cuboidID > baseCuboidId) {
        throw new IllegalArgumentException("Cubiod " + cuboidID + " is out of scope 0-" + baseCuboidId);
    }
    return baseCuboidId == cuboidID;
}
 

private static boolean checkAggregationGroup(RowKeyDesc rowkey, long cuboidID) {
    long cuboidWithoutMandatory = cuboidID & ~rowkey.getMandatoryColumnMask();
    long leftover;
    for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
        if ((cuboidWithoutMandatory & mask.uniqueMask) != 0) {
            leftover = cuboidWithoutMandatory & ~mask.groupMask;
            return leftover == 0 || leftover == mask.leftoverMask;
        }
    }

    leftover = cuboidWithoutMandatory & rowkey.getTailMask();
    return leftover == 0 || leftover == rowkey.getTailMask();
}
 

private void init(Set<TblColRef> selectedDimensions, Set<FunctionDesc> selectedMetrics) {
    this.dimensions = new TblColRef[selectedDimensions.size()];
    this.metrics = new FunctionDesc[selectedMetrics.size()];
    this.measures = new MeasureDesc[selectedMetrics.size()];
    this.dimDataTypes = new DataType[dimensions.length];
    this.metricsDataTypes = new DataType[metrics.length];
    // sort dimensions according to the rowKey definition
    dimColIdxMap = Maps.newHashMap();
    RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
    int colIdx = 0;
    for (RowKeyColDesc rowKeyColDesc : rowKeyDesc.getRowKeyColumns()) {
        TblColRef dimension = rowKeyColDesc.getColRef();
        if (selectedDimensions.contains(dimension)) {
            dimensions[colIdx] = dimension;
            dimDataTypes[colIdx] = dimension.getType();
            dimColIdxMap.put(dimension, colIdx);
            colIdx++;
        }
    }

    nDimensions = colIdx;

    colIdx = 0;
    // metrics
    metricsColIdxMap = Maps.newHashMap();
    for (MeasureDesc measure : cubeDesc.getMeasures()) {
        FunctionDesc func = measure.getFunction();
        if (selectedMetrics.contains(func)) {
            metrics[colIdx] = func;
            measures[colIdx] = measure;
            metricsColIdxMap.put(func.getParameter().getColRef(), colIdx);
            metricsDataTypes[colIdx] = func.getReturnDataType();
            colIdx++;
        }
    }

    nMetrics = colIdx;
}
 

@Override
protected void setup(Context context) throws IOException {
    super.publishConfiguration(context.getConfiguration());

    Configuration conf = context.getConfiguration();

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
    cubeName = conf.get(BatchConstants.CFG_CUBE_NAME);
    cube = CubeManager.getInstance(config).getCube(cubeName);
    cubeDesc = cube.getDescriptor();
    intermediateTableDesc = new CubeJoinedFlatTableDesc(cubeDesc, null);

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
    List<TblColRef> columns = baseCuboid.getColumns();

    ArrayList<Integer> factDictCols = new ArrayList<Integer>();
    RowKeyDesc rowkey = cubeDesc.getRowkey();
    DictionaryManager dictMgr = DictionaryManager.getInstance(config);
    for (int i = 0; i < columns.size(); i++) {
        TblColRef col = columns.get(i);
        if (rowkey.isUseDictionary(col) == false)
            continue;

        String scanTable = (String) dictMgr.decideSourceData(cubeDesc.getModel(), cubeDesc.getRowkey().getDictionary(col), col, null)[0];
        if (cubeDesc.getModel().isFactTable(scanTable)) {
            factDictCols.add(i);
        }
    }
    this.factDictCols = new int[factDictCols.size()];
    for (int i = 0; i < factDictCols.size(); i++)
        this.factDictCols[i] = factDictCols.get(i);

    schema = HCatInputFormat.getTableSchema(context.getConfiguration());
}
 

public static long estimateCuboidSpace(long cuboidID, long[] cardinality, CubeDesc cubeDesc) {

        RowKeyColInfo rowKeyColInfo = extractRowKeyInfo(cubeDesc);
        RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();

        long rowCount = 1;
        int[] rowKeySpaces = estimateRowKeyColSpace(rowKeyDesc, cardinality);
        int dimensionSpace = 0;
        int measureSpace = getMeasureSpace(cubeDesc);

        for (List<Integer> hlist : rowKeyColInfo.hierachyColBitIndice) {
            // for hierachy columns, the cardinality of the most detailed column
            // nominates.
            int i;
            for (i = 0; i < hlist.size() && rowKeyColExists(hlist.get(i), cuboidID); ++i) {
                dimensionSpace += rowKeySpaces[hlist.get(i)];
            }

            if (i != 0)
                rowCount *= cardinality[hlist.get(i - 1)];
        }

        for (int index : rowKeyColInfo.nonHierachyColBitIndice) {
            if (rowKeyColExists(index, cuboidID)) {
                rowCount *= cardinality[index];
                dimensionSpace += rowKeySpaces[index];
            }
        }
        return rowCount * (dimensionSpace + measureSpace);
    }
 

@Override
public void validate(CubeDesc cube, ValidateContext context) {
    RowKeyDesc row = cube.getRowkey();
    if (row == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey does not exist");
        return;
    }

    RowKeyColDesc[] rcd = row.getRowKeyColumns();
    if (rcd == null) {
        context.addResult(ResultLevel.ERROR, "Rowkey columns do not exist");
        return;
    }
    if(rcd.length == 0){
        context.addResult(ResultLevel.ERROR, "Rowkey columns is empty");
        return;       	
    }

    for (int i = 0; i < rcd.length; i++) {
        RowKeyColDesc rd = rcd[i];
        if (rd.getLength() != 0 && (!StringUtils.isEmpty(rd.getDictionary())&&!rd.getDictionary().equals("false"))) {
            context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' attribute");
        }
        if (rd.getLength() == 0 && (StringUtils.isEmpty(rd.getDictionary())||rd.getDictionary().equals("false"))) {
            context.addResult(ResultLevel.ERROR, "Rowkey column " + rd.getColumn() + " must not have both 'length' and 'dictionary' empty");
        }
    }

}
 

private Collection<Long> generateChildren(long cuboid) {
    Collection<Long> result = new HashSet<Long>();

    // generate zero tail cuboid -- the one with all 1 in the first
    // aggregation group and all 0 for the rest bits
    generateZeroTailBase(cuboid, result);

    RowKeyDesc rowkey = cubeDef.getRowkey();
    long cuboidWithoutMandatory = cuboid & ~rowkey.getMandatoryColumnMask();
    for (AggrGroupMask mask : rowkey.getAggrGroupMasks()) {
        if (belongTo(cuboidWithoutMandatory, mask) == false)
            continue;

        long[] groupOneBitMasks = mask.groupOneBitMasks;
        for (int i = 0; i < groupOneBitMasks.length; i++) {
            long oneBit = groupOneBitMasks[i];
            if ((cuboid & oneBit) == 0)
                continue;

            long child = cuboid ^ oneBit;
            if (Cuboid.isValid(cubeDef, child)) {
                result.add(child);
            }
        }

        if ((cuboidWithoutMandatory & mask.uniqueMask) > 0)
            break;
    }

    return result;
}
 

private static int[] estimateRowKeyColSpace(RowKeyDesc rowKeyDesc, long[] cardinality) {
    RowKeyColDesc[] rowKeyColDescs = rowKeyDesc.getRowKeyColumns();
    int[] ret = new int[rowKeyColDescs.length];
    for (int i = 0; i < rowKeyColDescs.length; ++i) {
        RowKeyColDesc rowKeyColDesc = rowKeyColDescs[rowKeyColDescs.length - 1 - i];
        if (rowKeyColDesc.getDictionary() == null) {
            if (rowKeyColDesc.getLength() == 0)
                throw new IllegalStateException("The non-dictionary col " + rowKeyColDesc.getColumn() + " has length of 0");
            ret[i] = rowKeyColDesc.getLength();
        } else {
            ret[i] = estimateDictionaryColSpace(cardinality[i]);
        }
    }
    return ret;
}
 

public static int mathCalcCuboidCount(CubeDesc cube) {
    int result = 1; // 1 for base cuboid

    RowKeyDesc rowkey = cube.getRowkey();
    AggrGroupMask[] aggrGroupMasks = rowkey.getAggrGroupMasks();
    for (int i = 0; i < aggrGroupMasks.length; i++) {
        boolean hasTail = i < aggrGroupMasks.length - 1 || rowkey.getTailMask() > 0;
        result += mathCalcCuboidCount_aggrGroup(rowkey, aggrGroupMasks[i], hasTail);
    }

    return result;
}
 

private static RowKeyColInfo extractRowKeyInfo(CubeDesc cubeDesc) {
    RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
    RowKeyColInfo info = new RowKeyColInfo();
    info.hierachyColBitIndice = new ArrayList<List<Integer>>();
    info.nonHierachyColBitIndice = new ArrayList<Integer>();
    HashSet<Integer> heirachyIndexSet = new HashSet<Integer>();

    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        if (dim.getHierarchy() != null) {
            LinkedList<Integer> hlist = new LinkedList<Integer>();
            for (HierarchyDesc hierarchyDesc : dim.getHierarchy()) {
                int index = rowKeyDesc.getColumnBitIndex(hierarchyDesc.getColumnRef());
                hlist.add(index);
                heirachyIndexSet.add(index);
            }
            info.hierachyColBitIndice.add(hlist);
        }
    }

    for (int i = 0; i < rowKeyDesc.getRowKeyColumns().length; ++i) {
        if (!heirachyIndexSet.contains(i)) {
            info.nonHierachyColBitIndice.add(i);
        }
    }

    return info;
}
 

public static CubeDesc generateKylinCubeDescForMetricsQueryCube(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryCube();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());
    dimensions.remove(QueryCubePropertyEnum.PROJECT.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQueryCube();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryCube();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        if (!measure.equals(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString())) {
            measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
        }
    }

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.SEGMENT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_SOURCE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_TARGET.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.FILTER_MASK.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_MATCH.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_SUCCESS.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[] mandatory_dims = new String[] { QueryCubePropertyEnum.CUBE.toString() };
    mandatory_dims = refineColumnWithTable(tableName, mandatory_dims);

    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    String[][] joint_dims = new String[1][];
    joint_dims[0] = new String[] { QueryCubePropertyEnum.CUBOID_SOURCE.toString(),
            QueryCubePropertyEnum.CUBOID_TARGET.toString() };
    for (int i = 0; i < joint_dims.length; i++) {
        joint_dims[i] = refineColumnWithTable(tableName, joint_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = mandatory_dims;
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = joint_dims;

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 

public void setRowkey(RowKeyDesc rowkey) {
    this.rowkey = rowkey;
}
 

public RowKeyDesc getRowkey() {
    return rowkey;
}
 

public static CubeDesc generateKylinCubeDescForMetricsJobException(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJobException());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsJobException();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsJobException();
    measures.remove(JobPropertyEnum.ID_CODE.toString());
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(1);

    measureDescList.add(getMeasureCount());

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.EXCEPTION.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 

public static CubeDesc generateKylinCubeDescForMetricsJob(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJob());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsJob();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsJob();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize((measures.size() - 4) * 3 + 1 + 1 + 4);

    Set<String> stepDuration = Sets.newHashSet();
    stepDuration.add(JobPropertyEnum.STEP_DURATION_DISTINCT_COLUMNS.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_DICTIONARY.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_INMEM_CUBING.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_HFILE_CONVERT.toString());

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsJob();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
        if (!stepDuration.contains(measure)) {
            measureDescList.add(getMeasureMin(measure, measureTypeMap.get(measure)));
        }
    }
    measureDescList.add(getMeasurePercentile(JobPropertyEnum.BUILD_DURATION.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 

public static CubeDesc generateKylinCubeDescForMetricsQueryRPC(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryRPC();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQueryRPC();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryRPC();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
    }
    measureDescList.add(getMeasurePercentile(QueryRPCPropertyEnum.CALL_TIME.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.REALIZATION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.RPC_SERVER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.EXCEPTION.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 

public static CubeDesc generateKylinCubeDescForMetricsQueryCube(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryCube();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());
    dimensions.remove(QueryCubePropertyEnum.PROJECT.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQueryCube();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryCube();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        if (!measure.equals(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString())) {
            measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
        }
    }

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.SEGMENT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_SOURCE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_TARGET.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.FILTER_MASK.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_MATCH.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_SUCCESS.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[] mandatory_dims = new String[] { QueryCubePropertyEnum.CUBE.toString() };
    mandatory_dims = refineColumnWithTable(tableName, mandatory_dims);

    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    String[][] joint_dims = new String[1][];
    joint_dims[0] = new String[] { QueryCubePropertyEnum.CUBOID_SOURCE.toString(),
            QueryCubePropertyEnum.CUBOID_TARGET.toString() };
    for (int i = 0; i < joint_dims.length; i++) {
        joint_dims[i] = refineColumnWithTable(tableName, joint_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = mandatory_dims;
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = joint_dims;

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 

public static CubeDesc generateKylinCubeDescForMetricsQuery(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQuery());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQuery();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQuery();
    measures.remove(QueryPropertyEnum.ID_CODE.toString());
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQuery();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    measureDescList.add(getMeasureMin(QueryPropertyEnum.TIME_COST.toString(),
            measureTypeMap.get(QueryPropertyEnum.TIME_COST.toString())));
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
    }
    measureDescList.add(getMeasureHLL(QueryPropertyEnum.ID_CODE.toString()));
    measureDescList.add(getMeasurePercentile(QueryPropertyEnum.TIME_COST.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.USER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION_TYPE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.EXCEPTION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.TYPE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[2][];
    hierarchy_dims[0] = getTimeHierarchy();
    hierarchy_dims[1] = new String[2];
    hierarchy_dims[1][0] = QueryPropertyEnum.REALIZATION_TYPE.toString();
    hierarchy_dims[1][1] = QueryPropertyEnum.REALIZATION.toString();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}