org.apache.kylin.cube.model.HBaseMappingDesc Java Examples
The following examples show how to use
org.apache.kylin.cube.model.HBaseMappingDesc.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CubeHBaseRPC.java From kylin with Apache License 2.0 | 6 votes |
/** * for each selected hbase column, it might contain values of multiple GT columns. * The mapping should be passed down to storage */ protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) { List<List<Integer>> ret = Lists.newArrayList(); int colBlkIndex = 1; int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount(); HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { int[] metricIndexes = hbaseColDesc.getMeasureIndex(); Integer[] gtIndexes = new Integer[metricIndexes.length]; for (int i = 0; i < gtIndexes.length; i++) { gtIndexes[i] = metricIndexes[i] + metricOffset; } ret.add(Arrays.asList(gtIndexes)); } colBlkIndex++; } } Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1); return ret; }
Example #2
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
public static CubeDesc generateKylinCubeDesc(String tableName, int storageType, List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc, AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) { CubeDesc desc = new CubeDesc(); desc.setName(tableName.replace('.', '_')); desc.setModelName(tableName.replace('.', '_')); desc.setDescription(""); desc.setLastModified(0L); desc.setDimensions(dimensionDescList); desc.setMeasures(measureDescList); desc.setRowkey(rowKeyDesc); desc.setHbaseMapping(hBaseMapping); desc.setNotifyList(Lists.<String> newArrayList()); desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString())); desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L }); desc.setEngineType(IEngineAware.ID_MR_V2); desc.setStorageType(storageType); desc.setAggregationGroups(Lists.newArrayList(aggGroup)); desc.getOverrideKylinProps().putAll(overrideProperties); desc.updateRandomUuid(); return desc; }
Example #3
Source File: CubeHBaseRPC.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
/** * prune untouched hbase columns */ protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) { List<Pair<byte[], byte[]>> result = Lists.newArrayList(); int colBlkIndex = 1; HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { byte[] byteFamily = Bytes.toBytes(familyDesc.getName()); for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier()); result.add(Pair.newPair(byteFamily, byteQualifier)); } colBlkIndex++; } } return result; }
Example #4
Source File: CubeHBaseRPC.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
/** * for each selected hbase column, it might contain values of multiple GT columns. * The mapping should be passed down to storage */ protected List<List<Integer>> getHBaseColumnsGTMapping(ImmutableBitSet selectedColBlocks) { List<List<Integer>> ret = Lists.newArrayList(); int colBlkIndex = 1; int metricOffset = fullGTInfo.getPrimaryKey().trueBitCount(); HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { int[] metricIndexes = hbaseColDesc.getMeasureIndex(); Integer[] gtIndexes = new Integer[metricIndexes.length]; for (int i = 0; i < gtIndexes.length; i++) { gtIndexes[i] = metricIndexes[i] + metricOffset; } ret.add(Arrays.asList(gtIndexes)); } colBlkIndex++; } } Preconditions.checkState(selectedColBlocks.trueBitCount() == ret.size() + 1); return ret; }
Example #5
Source File: CubeHBaseRPC.java From kylin with Apache License 2.0 | 6 votes |
/** * prune untouched hbase columns */ protected List<Pair<byte[], byte[]>> makeHBaseColumns(ImmutableBitSet selectedColBlocks) { List<Pair<byte[], byte[]>> result = Lists.newArrayList(); int colBlkIndex = 1; HBaseMappingDesc hbaseMapping = cubeSeg.getCubeDesc().getHbaseMapping(); for (HBaseColumnFamilyDesc familyDesc : hbaseMapping.getColumnFamily()) { byte[] byteFamily = Bytes.toBytes(familyDesc.getName()); for (HBaseColumnDesc hbaseColDesc : familyDesc.getColumns()) { if (selectedColBlocks.get(colBlkIndex)) { byte[] byteQualifier = Bytes.toBytes(hbaseColDesc.getQualifier()); result.add(Pair.newPair(byteFamily, byteQualifier)); } colBlkIndex++; } } return result; }
Example #6
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 6 votes |
public static CubeDesc generateKylinCubeDesc(String tableName, int storageType, List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc, AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) { CubeDesc desc = new CubeDesc(); desc.setName(tableName.replace('.', '_')); desc.setModelName(tableName.replace('.', '_')); desc.setDescription(""); desc.setLastModified(0L); desc.setDimensions(dimensionDescList); desc.setMeasures(measureDescList); desc.setRowkey(rowKeyDesc); desc.setHbaseMapping(hBaseMapping); desc.setNotifyList(Lists.<String> newArrayList()); desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString())); desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L, 7776000000L, 31104000000L }); desc.setEngineType(IEngineAware.ID_MR_V2); desc.setStorageType(storageType); desc.setAggregationGroups(Lists.newArrayList(aggGroup)); desc.getOverrideKylinProps().putAll(overrideProperties); desc.updateRandomUuid(); return desc; }
Example #7
Source File: CubeStorageEngine.java From Kylin with Apache License 2.0 | 5 votes |
private List<RowValueDecoder> translateAggregation(HBaseMappingDesc hbaseMapping, Collection<FunctionDesc> metrics, // StorageContext context) { Map<HBaseColumnDesc, RowValueDecoder> codecMap = Maps.newHashMap(); for (FunctionDesc aggrFunc : metrics) { Collection<HBaseColumnDesc> hbCols = hbaseMapping.findHBaseColumnByFunction(aggrFunc); if (hbCols.isEmpty()) { throw new IllegalStateException("can't find HBaseColumnDesc for function " + aggrFunc.getFullExpression()); } HBaseColumnDesc bestHBCol = null; int bestIndex = -1; for (HBaseColumnDesc hbCol : hbCols) { bestHBCol = hbCol; bestIndex = hbCol.findMeasureIndex(aggrFunc); MeasureDesc measure = hbCol.getMeasures()[bestIndex]; // criteria for holistic measure: Exact Aggregation && Exact Cuboid if (measure.isHolisticCountDistinct() && context.isExactAggregation()) { logger.info("Holistic count distinct chosen for " + aggrFunc); break; } } RowValueDecoder codec = codecMap.get(bestHBCol); if (codec == null) { codec = new RowValueDecoder(bestHBCol); codecMap.put(bestHBCol, codec); } codec.setIndex(bestIndex); } return new ArrayList<RowValueDecoder>(codecMap.values()); }
Example #8
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQueryCube(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryCube(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); dimensions.remove(QueryCubePropertyEnum.PROJECT.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQueryCube(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryCube(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); if (!measure.equals(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString())) { measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } } //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.SEGMENT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_SOURCE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_TARGET.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.FILTER_MASK.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_MATCH.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_SUCCESS.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[] mandatory_dims = new String[] { QueryCubePropertyEnum.CUBE.toString() }; mandatory_dims = refineColumnWithTable(tableName, mandatory_dims); String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } String[][] joint_dims = new String[1][]; joint_dims[0] = new String[] { QueryCubePropertyEnum.CUBOID_SOURCE.toString(), QueryCubePropertyEnum.CUBOID_TARGET.toString() }; for (int i = 0; i < joint_dims.length; i++) { joint_dims[i] = refineColumnWithTable(tableName, joint_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = mandatory_dims; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = joint_dims; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example #9
Source File: CubeDesc.java From Kylin with Apache License 2.0 | 4 votes |
public void setHBaseMapping(HBaseMappingDesc hbaseMapping) { this.hbaseMapping = hbaseMapping; }
Example #10
Source File: CubeDesc.java From Kylin with Apache License 2.0 | 4 votes |
public HBaseMappingDesc getHBaseMapping() { return hbaseMapping; }
Example #11
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsJobException(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJobException()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsJobException(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsJobException(); measures.remove(JobPropertyEnum.ID_CODE.toString()); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(1); measureDescList.add(getMeasureCount()); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example #12
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsJob(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJob()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsJob(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsJob(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize((measures.size() - 4) * 3 + 1 + 1 + 4); Set<String> stepDuration = Sets.newHashSet(); stepDuration.add(JobPropertyEnum.STEP_DURATION_DISTINCT_COLUMNS.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_DICTIONARY.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_INMEM_CUBING.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_HFILE_CONVERT.toString()); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsJob(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); if (!stepDuration.contains(measure)) { measureDescList.add(getMeasureMin(measure, measureTypeMap.get(measure))); } } measureDescList.add(getMeasurePercentile(JobPropertyEnum.BUILD_DURATION.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example #13
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQueryRPC(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryRPC(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQueryRPC(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryRPC(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } measureDescList.add(getMeasurePercentile(QueryRPCPropertyEnum.CALL_TIME.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.REALIZATION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.RPC_SERVER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example #14
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQuery(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQuery()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQuery(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQuery(); measures.remove(QueryPropertyEnum.ID_CODE.toString()); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQuery(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); measureDescList.add(getMeasureMin(QueryPropertyEnum.TIME_COST.toString(), measureTypeMap.get(QueryPropertyEnum.TIME_COST.toString()))); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } measureDescList.add(getMeasureHLL(QueryPropertyEnum.ID_CODE.toString())); measureDescList.add(getMeasurePercentile(QueryPropertyEnum.TIME_COST.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION_TYPE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.TYPE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[2][]; hierarchy_dims[0] = getTimeHierarchy(); hierarchy_dims[1] = new String[2]; hierarchy_dims[1][0] = QueryPropertyEnum.REALIZATION_TYPE.toString(); hierarchy_dims[1][1] = QueryPropertyEnum.REALIZATION.toString(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example #15
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQuery(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQuery()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQuery(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQuery(); measures.remove(QueryPropertyEnum.ID_CODE.toString()); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQuery(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); measureDescList.add(getMeasureMin(QueryPropertyEnum.TIME_COST.toString(), measureTypeMap.get(QueryPropertyEnum.TIME_COST.toString()))); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } measureDescList.add(getMeasureHLL(QueryPropertyEnum.ID_CODE.toString())); measureDescList.add(getMeasurePercentile(QueryPropertyEnum.TIME_COST.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION_TYPE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.TYPE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[2][]; hierarchy_dims[0] = getTimeHierarchy(); hierarchy_dims[1] = new String[2]; hierarchy_dims[1][0] = QueryPropertyEnum.REALIZATION_TYPE.toString(); hierarchy_dims[1][1] = QueryPropertyEnum.REALIZATION.toString(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example #16
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsJobException(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJobException()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsJobException(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsJobException(); measures.remove(JobPropertyEnum.ID_CODE.toString()); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(1); measureDescList.add(getMeasureCount()); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example #17
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsJob(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJob()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsJob(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsJob(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize((measures.size() - 4) * 3 + 1 + 1 + 4); Set<String> stepDuration = Sets.newHashSet(); stepDuration.add(JobPropertyEnum.STEP_DURATION_DISTINCT_COLUMNS.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_DICTIONARY.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_INMEM_CUBING.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_HFILE_CONVERT.toString()); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsJob(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); if (!stepDuration.contains(measure)) { measureDescList.add(getMeasureMin(measure, measureTypeMap.get(measure))); } } measureDescList.add(getMeasurePercentile(JobPropertyEnum.BUILD_DURATION.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example #18
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQueryRPC(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryRPC(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQueryRPC(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryRPC(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } measureDescList.add(getMeasurePercentile(QueryRPCPropertyEnum.CALL_TIME.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.REALIZATION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.RPC_SERVER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example #19
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQueryCube(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryCube(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); dimensions.remove(QueryCubePropertyEnum.PROJECT.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQueryCube(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryCube(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); if (!measure.equals(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString())) { measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } } //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.SEGMENT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_SOURCE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_TARGET.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.FILTER_MASK.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_MATCH.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_SUCCESS.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[] mandatory_dims = new String[] { QueryCubePropertyEnum.CUBE.toString() }; mandatory_dims = refineColumnWithTable(tableName, mandatory_dims); String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } String[][] joint_dims = new String[1][]; joint_dims[0] = new String[] { QueryCubePropertyEnum.CUBOID_SOURCE.toString(), QueryCubePropertyEnum.CUBOID_TARGET.toString() }; for (int i = 0; i < joint_dims.length; i++) { joint_dims[i] = refineColumnWithTable(tableName, joint_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = mandatory_dims; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = joint_dims; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }