Java Code Examples for org.apache.kylin.cube.model.HBaseMappingDesc

The following examples show how to use org.apache.kylin.cube.model.HBaseMappingDesc. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 6 votes vote down vote up
public static CubeDesc generateKylinCubeDesc(String tableName, int storageType,
        List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc,
        AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) {
    CubeDesc desc = new CubeDesc();
    desc.setName(tableName.replace('.', '_'));
    desc.setModelName(tableName.replace('.', '_'));
    desc.setDescription("");
    desc.setLastModified(0L);
    desc.setDimensions(dimensionDescList);
    desc.setMeasures(measureDescList);
    desc.setRowkey(rowKeyDesc);
    desc.setHbaseMapping(hBaseMapping);
    desc.setNotifyList(Lists.<String> newArrayList());
    desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString()));
    desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L });
    desc.setEngineType(IEngineAware.ID_MR_V2);
    desc.setStorageType(storageType);
    desc.setAggregationGroups(Lists.newArrayList(aggGroup));
    desc.getOverrideKylinProps().putAll(overrideProperties);
    desc.updateRandomUuid();
    return desc;
}
 
Example 2
Source Project: kylin-on-parquet-v2   Source File: CubeHBaseRPC.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * prune untouched hbase columns
 */
protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) {
    List<Pair<byte[], byte[]>> result = Lists.newArrayList();

    int colBlkIndex = 1;
    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        byte[] byteFamily = Bytes.toBytes(familyDesc.getName());
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier());
                result.add(Pair.newPair(byteFamily, byteQualifier));
            }
            colBlkIndex++;
        }
    }

    return result;
}
 
Example 3
Source Project: kylin-on-parquet-v2   Source File: CubeHBaseRPC.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * for each selected hbase column, it might contain values of multiple GT columns.
 * The mapping should be passed down to storage
 */
protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) {

    List<List<Integer>> ret = Lists.newArrayList();

    int colBlkIndex = 1;
    int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount();

    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                int[] metricIndexes = hbaseColDesc.getMeasureIndex();
                Integer[] gtIndexes = new Integer[metricIndexes.length];
                for (int i = 0; i < gtIndexes.length; i++) {
                    gtIndexes[i] = metricIndexes[i] + metricOffset;
                }
                ret.add(Arrays.asList(gtIndexes));
            }
            colBlkIndex++;
        }
    }

    Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1);
    return ret;
}
 
Example 4
Source Project: kylin   Source File: CubeDescCreator.java    License: Apache License 2.0 6 votes vote down vote up
public static CubeDesc generateKylinCubeDesc(String tableName, int storageType,
        List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc,
        AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) {
    CubeDesc desc = new CubeDesc();
    desc.setName(tableName.replace('.', '_'));
    desc.setModelName(tableName.replace('.', '_'));
    desc.setDescription("");
    desc.setLastModified(0L);
    desc.setDimensions(dimensionDescList);
    desc.setMeasures(measureDescList);
    desc.setRowkey(rowKeyDesc);
    desc.setHbaseMapping(hBaseMapping);
    desc.setNotifyList(Lists.<String> newArrayList());
    desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString()));
    desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L, 7776000000L, 31104000000L });
    desc.setEngineType(IEngineAware.ID_MR_V2);
    desc.setStorageType(storageType);
    desc.setAggregationGroups(Lists.newArrayList(aggGroup));
    desc.getOverrideKylinProps().putAll(overrideProperties);
    desc.updateRandomUuid();
    return desc;
}
 
Example 5
Source Project: kylin   Source File: CubeHBaseRPC.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * prune untouched hbase columns
 */
protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) {
    List<Pair<byte[], byte[]>> result = Lists.newArrayList();

    int colBlkIndex = 1;
    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        byte[] byteFamily = Bytes.toBytes(familyDesc.getName());
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier());
                result.add(Pair.newPair(byteFamily, byteQualifier));
            }
            colBlkIndex++;
        }
    }

    return result;
}
 
Example 6
Source Project: kylin   Source File: CubeHBaseRPC.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * for each selected hbase column, it might contain values of multiple GT columns.
 * The mapping should be passed down to storage
 */
protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) {

    List<List<Integer>> ret = Lists.newArrayList();

    int colBlkIndex = 1;
    int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount();

    HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping();
    for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) {
        for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) {
            if (selectedColBlocks.get(colBlkIndex)) {
                int[] metricIndexes = hbaseColDesc.getMeasureIndex();
                Integer[] gtIndexes = new Integer[metricIndexes.length];
                for (int i = 0; i < gtIndexes.length; i++) {
                    gtIndexes[i] = metricIndexes[i] + metricOffset;
                }
                ret.add(Arrays.asList(gtIndexes));
            }
            colBlkIndex++;
        }
    }

    Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1);
    return ret;
}
 
Example 7
Source Project: Kylin   Source File: CubeStorageEngine.java    License: Apache License 2.0 5 votes vote down vote up
private List<RowValueDecoder> translateAggregation(HBaseMappingDesc hbaseMapping, Collection<FunctionDesc> metrics, //
        StorageContext context) {
    Map<HBaseColumnDesc, RowValueDecoder> codecMap = Maps.newHashMap();
    for (FunctionDesc aggrFunc : metrics) {
        Collection<HBaseColumnDesc> hbCols = hbaseMapping.findHBaseColumnByFunction(aggrFunc);
        if (hbCols.isEmpty()) {
            throw new IllegalStateException("can't find HBaseColumnDesc for function " + aggrFunc.getFullExpression());
        }
        HBaseColumnDesc bestHBCol = null;
        int bestIndex = -1;
        for (HBaseColumnDesc hbCol : hbCols) {
            bestHBCol = hbCol;
            bestIndex = hbCol.findMeasureIndex(aggrFunc);
            MeasureDesc measure = hbCol.getMeasures()[bestIndex];
            // criteria for holistic measure: Exact Aggregation && Exact Cuboid
            if (measure.isHolisticCountDistinct() && context.isExactAggregation()) {
                logger.info("Holistic count distinct chosen for " + aggrFunc);
                break;
            }
        }

        RowValueDecoder codec = codecMap.get(bestHBCol);
        if (codec == null) {
            codec = new RowValueDecoder(bestHBCol);
            codecMap.put(bestHBCol, codec);
        }
        codec.setIndex(bestIndex);
    }
    return new ArrayList<RowValueDecoder>(codecMap.values());
}
 
Example 8
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsQuery(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQuery());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQuery();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQuery();
    measures.remove(QueryPropertyEnum.ID_CODE.toString());
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQuery();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    measureDescList.add(getMeasureMin(QueryPropertyEnum.TIME_COST.toString(),
            measureTypeMap.get(QueryPropertyEnum.TIME_COST.toString())));
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
    }
    measureDescList.add(getMeasureHLL(QueryPropertyEnum.ID_CODE.toString()));
    measureDescList.add(getMeasurePercentile(QueryPropertyEnum.TIME_COST.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.USER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION_TYPE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.EXCEPTION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.TYPE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[2][];
    hierarchy_dims[0] = getTimeHierarchy();
    hierarchy_dims[1] = new String[2];
    hierarchy_dims[1][0] = QueryPropertyEnum.REALIZATION_TYPE.toString();
    hierarchy_dims[1][1] = QueryPropertyEnum.REALIZATION.toString();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 9
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsQueryCube(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryCube();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());
    dimensions.remove(QueryCubePropertyEnum.PROJECT.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQueryCube();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryCube();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        if (!measure.equals(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString())) {
            measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
        }
    }

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.SEGMENT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_SOURCE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_TARGET.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.FILTER_MASK.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_MATCH.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_SUCCESS.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[] mandatory_dims = new String[] { QueryCubePropertyEnum.CUBE.toString() };
    mandatory_dims = refineColumnWithTable(tableName, mandatory_dims);

    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    String[][] joint_dims = new String[1][];
    joint_dims[0] = new String[] { QueryCubePropertyEnum.CUBOID_SOURCE.toString(),
            QueryCubePropertyEnum.CUBOID_TARGET.toString() };
    for (int i = 0; i < joint_dims.length; i++) {
        joint_dims[i] = refineColumnWithTable(tableName, joint_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = mandatory_dims;
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = joint_dims;

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 10
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsQueryRPC(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryRPC();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQueryRPC();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryRPC();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
    }
    measureDescList.add(getMeasurePercentile(QueryRPCPropertyEnum.CALL_TIME.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.REALIZATION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.RPC_SERVER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.EXCEPTION.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 11
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsJob(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJob());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsJob();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsJob();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize((measures.size() - 4) * 3 + 1 + 1 + 4);

    Set<String> stepDuration = Sets.newHashSet();
    stepDuration.add(JobPropertyEnum.STEP_DURATION_DISTINCT_COLUMNS.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_DICTIONARY.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_INMEM_CUBING.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_HFILE_CONVERT.toString());

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsJob();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
        if (!stepDuration.contains(measure)) {
            measureDescList.add(getMeasureMin(measure, measureTypeMap.get(measure)));
        }
    }
    measureDescList.add(getMeasurePercentile(JobPropertyEnum.BUILD_DURATION.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 12
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsJobException(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJobException());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsJobException();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsJobException();
    measures.remove(JobPropertyEnum.ID_CODE.toString());
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(1);

    measureDescList.add(getMeasureCount());

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.EXCEPTION.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 13
Source Project: kylin   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsQuery(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQuery());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQuery();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQuery();
    measures.remove(QueryPropertyEnum.ID_CODE.toString());
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQuery();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    measureDescList.add(getMeasureMin(QueryPropertyEnum.TIME_COST.toString(),
            measureTypeMap.get(QueryPropertyEnum.TIME_COST.toString())));
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
    }
    measureDescList.add(getMeasureHLL(QueryPropertyEnum.ID_CODE.toString()));
    measureDescList.add(getMeasurePercentile(QueryPropertyEnum.TIME_COST.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.USER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION_TYPE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.EXCEPTION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.TYPE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[2][];
    hierarchy_dims[0] = getTimeHierarchy();
    hierarchy_dims[1] = new String[2];
    hierarchy_dims[1][0] = QueryPropertyEnum.REALIZATION_TYPE.toString();
    hierarchy_dims[1][1] = QueryPropertyEnum.REALIZATION.toString();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 14
Source Project: kylin   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsQueryCube(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryCube();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());
    dimensions.remove(QueryCubePropertyEnum.PROJECT.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQueryCube();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryCube();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        if (!measure.equals(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString())) {
            measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
        }
    }

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.SEGMENT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_SOURCE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_TARGET.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.FILTER_MASK.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_MATCH.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_SUCCESS.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[] mandatory_dims = new String[] { QueryCubePropertyEnum.CUBE.toString() };
    mandatory_dims = refineColumnWithTable(tableName, mandatory_dims);

    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    String[][] joint_dims = new String[1][];
    joint_dims[0] = new String[] { QueryCubePropertyEnum.CUBOID_SOURCE.toString(),
            QueryCubePropertyEnum.CUBOID_TARGET.toString() };
    for (int i = 0; i < joint_dims.length; i++) {
        joint_dims[i] = refineColumnWithTable(tableName, joint_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = mandatory_dims;
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = joint_dims;

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 15
Source Project: kylin   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsQueryRPC(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryRPC();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQueryRPC();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryRPC();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
    }
    measureDescList.add(getMeasurePercentile(QueryRPCPropertyEnum.CALL_TIME.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.REALIZATION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.RPC_SERVER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.EXCEPTION.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 16
Source Project: kylin   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsJob(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJob());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsJob();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsJob();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize((measures.size() - 4) * 3 + 1 + 1 + 4);

    Set<String> stepDuration = Sets.newHashSet();
    stepDuration.add(JobPropertyEnum.STEP_DURATION_DISTINCT_COLUMNS.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_DICTIONARY.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_INMEM_CUBING.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_HFILE_CONVERT.toString());

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsJob();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
        if (!stepDuration.contains(measure)) {
            measureDescList.add(getMeasureMin(measure, measureTypeMap.get(measure)));
        }
    }
    measureDescList.add(getMeasurePercentile(JobPropertyEnum.BUILD_DURATION.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 17
Source Project: kylin   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsJobException(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJobException());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsJobException();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsJobException();
    measures.remove(JobPropertyEnum.ID_CODE.toString());
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(1);

    measureDescList.add(getMeasureCount());

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.EXCEPTION.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 18
Source Project: Kylin   Source File: CubeDesc.java    License: Apache License 2.0 4 votes vote down vote up
public HBaseMappingDesc getHBaseMapping() {
    return hbaseMapping;
}
 
Example 19
Source Project: Kylin   Source File: CubeDesc.java    License: Apache License 2.0 4 votes vote down vote up
public void setHBaseMapping(HBaseMappingDesc hbaseMapping) {
    this.hbaseMapping = hbaseMapping;
}