Java Code Examples for org.apache.kylin.cube.model.HBaseMappingDesc#setColumnFamily()
The following examples show how to use
org.apache.kylin.cube.model.HBaseMappingDesc#setColumnFamily() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQuery(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQuery()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQuery(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQuery(); measures.remove(QueryPropertyEnum.ID_CODE.toString()); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQuery(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); measureDescList.add(getMeasureMin(QueryPropertyEnum.TIME_COST.toString(), measureTypeMap.get(QueryPropertyEnum.TIME_COST.toString()))); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } measureDescList.add(getMeasureHLL(QueryPropertyEnum.ID_CODE.toString())); measureDescList.add(getMeasurePercentile(QueryPropertyEnum.TIME_COST.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION_TYPE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.TYPE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[2][]; hierarchy_dims[0] = getTimeHierarchy(); hierarchy_dims[1] = new String[2]; hierarchy_dims[1][0] = QueryPropertyEnum.REALIZATION_TYPE.toString(); hierarchy_dims[1][1] = QueryPropertyEnum.REALIZATION.toString(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example 2
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQueryCube(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryCube(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); dimensions.remove(QueryCubePropertyEnum.PROJECT.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQueryCube(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryCube(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); if (!measure.equals(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString())) { measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } } //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.SEGMENT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_SOURCE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_TARGET.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.FILTER_MASK.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_MATCH.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_SUCCESS.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[] mandatory_dims = new String[] { QueryCubePropertyEnum.CUBE.toString() }; mandatory_dims = refineColumnWithTable(tableName, mandatory_dims); String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } String[][] joint_dims = new String[1][]; joint_dims[0] = new String[] { QueryCubePropertyEnum.CUBOID_SOURCE.toString(), QueryCubePropertyEnum.CUBOID_TARGET.toString() }; for (int i = 0; i < joint_dims.length; i++) { joint_dims[i] = refineColumnWithTable(tableName, joint_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = mandatory_dims; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = joint_dims; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example 3
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQueryRPC(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryRPC(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQueryRPC(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryRPC(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } measureDescList.add(getMeasurePercentile(QueryRPCPropertyEnum.CALL_TIME.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.REALIZATION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.RPC_SERVER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example 4
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsJob(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJob()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsJob(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsJob(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize((measures.size() - 4) * 3 + 1 + 1 + 4); Set<String> stepDuration = Sets.newHashSet(); stepDuration.add(JobPropertyEnum.STEP_DURATION_DISTINCT_COLUMNS.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_DICTIONARY.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_INMEM_CUBING.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_HFILE_CONVERT.toString()); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsJob(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); if (!stepDuration.contains(measure)) { measureDescList.add(getMeasureMin(measure, measureTypeMap.get(measure))); } } measureDescList.add(getMeasurePercentile(JobPropertyEnum.BUILD_DURATION.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example 5
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsJobException(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJobException()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsJobException(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsJobException(); measures.remove(JobPropertyEnum.ID_CODE.toString()); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(1); measureDescList.add(getMeasureCount()); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example 6
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQuery(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQuery()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQuery(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQuery(); measures.remove(QueryPropertyEnum.ID_CODE.toString()); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQuery(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); measureDescList.add(getMeasureMin(QueryPropertyEnum.TIME_COST.toString(), measureTypeMap.get(QueryPropertyEnum.TIME_COST.toString()))); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } measureDescList.add(getMeasureHLL(QueryPropertyEnum.ID_CODE.toString())); measureDescList.add(getMeasurePercentile(QueryPropertyEnum.TIME_COST.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION_TYPE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.TYPE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[2][]; hierarchy_dims[0] = getTimeHierarchy(); hierarchy_dims[1] = new String[2]; hierarchy_dims[1][0] = QueryPropertyEnum.REALIZATION_TYPE.toString(); hierarchy_dims[1][1] = QueryPropertyEnum.REALIZATION.toString(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example 7
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQueryCube(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryCube(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); dimensions.remove(QueryCubePropertyEnum.PROJECT.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQueryCube(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryCube(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); if (!measure.equals(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString())) { measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } } //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.SEGMENT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_SOURCE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_TARGET.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.FILTER_MASK.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_MATCH.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_SUCCESS.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[] mandatory_dims = new String[] { QueryCubePropertyEnum.CUBE.toString() }; mandatory_dims = refineColumnWithTable(tableName, mandatory_dims); String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } String[][] joint_dims = new String[1][]; joint_dims[0] = new String[] { QueryCubePropertyEnum.CUBOID_SOURCE.toString(), QueryCubePropertyEnum.CUBOID_TARGET.toString() }; for (int i = 0; i < joint_dims.length; i++) { joint_dims[i] = refineColumnWithTable(tableName, joint_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = mandatory_dims; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = joint_dims; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example 8
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQueryRPC(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryRPC(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQueryRPC(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryRPC(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } measureDescList.add(getMeasurePercentile(QueryRPCPropertyEnum.CALL_TIME.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.REALIZATION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.RPC_SERVER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example 9
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsJob(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJob()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsJob(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsJob(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize((measures.size() - 4) * 3 + 1 + 1 + 4); Set<String> stepDuration = Sets.newHashSet(); stepDuration.add(JobPropertyEnum.STEP_DURATION_DISTINCT_COLUMNS.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_DICTIONARY.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_INMEM_CUBING.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_HFILE_CONVERT.toString()); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsJob(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); if (!stepDuration.contains(measure)) { measureDescList.add(getMeasureMin(measure, measureTypeMap.get(measure))); } } measureDescList.add(getMeasurePercentile(JobPropertyEnum.BUILD_DURATION.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example 10
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsJobException(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJobException()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsJobException(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsJobException(); measures.remove(JobPropertyEnum.ID_CODE.toString()); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(1); measureDescList.add(getMeasureCount()); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }