org.apache.kylin.cube.model.DimensionDesc Java Examples
The following examples show how to use
org.apache.kylin.cube.model.DimensionDesc.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MetadataUpgradeTest.java From Kylin with Apache License 2.0 | 6 votes |
private void checkCubeDesc(String descName) { CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv()); CubeDesc cubedesc1 = cubeDescMgr.getCubeDesc(descName); Assert.assertNotNull(cubedesc1); DataModelDesc model = cubedesc1.getModel(); Assert.assertNotNull(model); Assert.assertTrue(model.getLookups().length > 0); List<DimensionDesc> dims = cubedesc1.getDimensions(); Assert.assertTrue(dims.size() > 0); for (DimensionDesc dim : dims) { Assert.assertTrue(dim.getColumn().length > 0); } Assert.assertTrue(cubedesc1.getMeasures().size() > 0); CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); List<CubeInstance> cubes = cubeMgr.getCubesByDesc(descName); Assert.assertTrue(cubes.size() > 0); }
Example #2
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 6 votes |
public static CubeDesc generateKylinCubeDesc(String tableName, int storageType, List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc, AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) { CubeDesc desc = new CubeDesc(); desc.setName(tableName.replace('.', '_')); desc.setModelName(tableName.replace('.', '_')); desc.setDescription(""); desc.setLastModified(0L); desc.setDimensions(dimensionDescList); desc.setMeasures(measureDescList); desc.setRowkey(rowKeyDesc); desc.setHbaseMapping(hBaseMapping); desc.setNotifyList(Lists.<String> newArrayList()); desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString())); desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L, 7776000000L, 31104000000L }); desc.setEngineType(IEngineAware.ID_MR_V2); desc.setStorageType(storageType); desc.setAggregationGroups(Lists.newArrayList(aggGroup)); desc.getOverrideKylinProps().putAll(overrideProperties); desc.updateRandomUuid(); return desc; }
Example #3
Source File: HBaseLookupMRSteps.java From kylin with Apache License 2.0 | 6 votes |
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) { CubeDesc cubeDesc = cube.getDescriptor(); Set<String> allLookupTables = Sets.newHashSet(); for (DimensionDesc dim : cubeDesc.getDimensions()) { TableRef table = dim.getTableRef(); if (cubeDesc.getModel().isLookupTable(table)) { allLookupTables.add(table.getTableIdentity()); } } List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) { if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType()) && allLookupTables.contains(snapshotTableDesc.getTableName())) { addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc); } } }
Example #4
Source File: SparkBuildDictionary.java From kylin with Apache License 2.0 | 6 votes |
private void checkSnapshot(CubeManager cubeManager, CubeSegment cubeSegment) { List<DimensionDesc> dimensionDescs = cubeSegment.getCubeDesc().getDimensions(); for (DimensionDesc dimensionDesc : dimensionDescs) { TableRef lookup = dimensionDesc.getTableRef(); String tableIdentity = lookup.getTableIdentity(); if (cubeSegment.getModel().isLookupTable(tableIdentity) && !cubeSegment.getCubeDesc().isExtSnapshotTable(tableIdentity)) { logger.info("Checking snapshot of {}", lookup); try { JoinDesc join = cubeSegment.getModel().getJoinsTree().getJoinByPKSide(lookup); ILookupTable table = cubeManager.getLookupTable(cubeSegment, join); if (table != null) { IOUtils.closeStream(table); } } catch (Throwable th) { throw new RuntimeException(String.format(Locale.ROOT, "Checking snapshot of %s failed.", lookup), th); } } } }
Example #5
Source File: SparkBuildDictionary.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private void checkSnapshot(CubeManager cubeManager, CubeSegment cubeSegment) { List<DimensionDesc> dimensionDescs = cubeSegment.getCubeDesc().getDimensions(); for (DimensionDesc dimensionDesc : dimensionDescs) { TableRef lookup = dimensionDesc.getTableRef(); String tableIdentity = lookup.getTableIdentity(); if (cubeSegment.getModel().isLookupTable(tableIdentity) && !cubeSegment.getCubeDesc().isExtSnapshotTable(tableIdentity)) { logger.info("Checking snapshot of {}", lookup); try { JoinDesc join = cubeSegment.getModel().getJoinsTree().getJoinByPKSide(lookup); ILookupTable table = cubeManager.getLookupTable(cubeSegment, join); if (table != null) { IOUtils.closeStream(table); } } catch (Throwable th) { throw new RuntimeException(String.format(Locale.ROOT, "Checking snapshot of %s failed.", lookup), th); } } } }
Example #6
Source File: CubeManager.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
/** * check if {toCheck} has snapshot of {lookupTableName} * @param lookupTableName look like {SCHMEA}.{TABLE} */ private boolean checkMeetSnapshotTable(CubeInstance toCheck, String lookupTableName) { boolean checkRes = false; String lookupTbl = lookupTableName; String[] strArr = lookupTableName.split("\\."); if (strArr.length > 1) { lookupTbl = strArr[strArr.length - 1]; } for (DimensionDesc dimensionDesc : toCheck.getDescriptor().getDimensions()) { if (dimensionDesc.getTableRef().getTableName().equalsIgnoreCase(lookupTbl)) { checkRes = true; break; } } return checkRes; }
Example #7
Source File: LookupTableEnumerator.java From Kylin with Apache License 2.0 | 6 votes |
public LookupTableEnumerator(OLAPContext olapContext) { //TODO: assuming LookupTableEnumerator is handled by a cube CubeInstance cube = (CubeInstance) olapContext.realization; String lookupTableName = olapContext.firstTableScan.getTableName(); DimensionDesc dim = cube.getDescriptor().findDimensionByTable(lookupTableName); if (dim == null) throw new IllegalStateException("No dimension with derived columns found for lookup table " + lookupTableName + ", cube desc " + cube.getDescriptor()); CubeManager cubeMgr = CubeManager.getInstance(cube.getConfig()); LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim); this.allRows = table.getAllRows(); OLAPTable olapTable = (OLAPTable) olapContext.firstTableScan.getOlapTable(); this.colDescs = olapTable.getExposedColumns(); this.current = new Object[colDescs.size()]; reset(); }
Example #8
Source File: HBaseLookupMRSteps.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) { CubeDesc cubeDesc = cube.getDescriptor(); Set<String> allLookupTables = Sets.newHashSet(); for (DimensionDesc dim : cubeDesc.getDimensions()) { TableRef table = dim.getTableRef(); if (cubeDesc.getModel().isLookupTable(table)) { allLookupTables.add(table.getTableIdentity()); } } List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) { if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType()) && allLookupTables.contains(snapshotTableDesc.getTableName())) { addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc); } } }
Example #9
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
public static CubeDesc generateKylinCubeDesc(String tableName, int storageType, List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc, AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) { CubeDesc desc = new CubeDesc(); desc.setName(tableName.replace('.', '_')); desc.setModelName(tableName.replace('.', '_')); desc.setDescription(""); desc.setLastModified(0L); desc.setDimensions(dimensionDescList); desc.setMeasures(measureDescList); desc.setRowkey(rowKeyDesc); desc.setHbaseMapping(hBaseMapping); desc.setNotifyList(Lists.<String> newArrayList()); desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString())); desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L }); desc.setEngineType(IEngineAware.ID_MR_V2); desc.setStorageType(storageType); desc.setAggregationGroups(Lists.newArrayList(aggGroup)); desc.getOverrideKylinProps().putAll(overrideProperties); desc.updateRandomUuid(); return desc; }
Example #10
Source File: DictionaryGeneratorCLI.java From Kylin with Apache License 2.0 | 6 votes |
private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException { CubeManager cubeMgr = CubeManager.getInstance(config); for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) { // dictionary for (TblColRef col : dim.getColumnRefs()) { if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) { logger.info("Building dictionary for " + col); cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath); } } // build snapshot if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) { // CubeSegment seg = cube.getTheOnlySegment(); logger.info("Building snapshot of " + dim.getTable()); cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable()); logger.info("Checking snapshot of " + dim.getTable()); cubeMgr.getLookupTable(cubeSeg, dim); // load the table for // sanity check } } }
Example #11
Source File: CubeManager.java From Kylin with Apache License 2.0 | 6 votes |
public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) { String tableName = dim.getTable(); String[] pkCols = dim.getJoin().getPrimaryKey(); String snapshotResPath = cubeSegment.getSnapshotResPath(tableName); if (snapshotResPath == null) throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment); try { SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath); TableDesc tableDesc = getMetadataManager().getTableDesc(tableName); return new LookupStringTable(tableDesc, pkCols, snapshot); } catch (IOException e) { throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e); } }
Example #12
Source File: CubeDescUpgrader.java From Kylin with Apache License 2.0 | 5 votes |
private DimensionDesc newDimensionDesc(org.apache.kylin.cube.model.v1.DimensionDesc dim, int dimId, String name) { DimensionDesc newDim = new DimensionDesc(); newDim.setId(dimId); newDim.setName(name); newDim.setTable(getMetadataManager().appendDBName(dim.getTable())); return newDim; }
Example #13
Source File: CubeCapabilityChecker.java From Kylin with Apache License 2.0 | 5 votes |
private static boolean isMatchedWithJoins(Collection<JoinDesc> joins, CubeInstance cube) { CubeDesc cubeDesc = cube.getDescriptor(); List<JoinDesc> cubeJoins = new ArrayList<JoinDesc>(cubeDesc.getDimensions().size()); for (DimensionDesc d : cubeDesc.getDimensions()) { if (d.getJoin() != null) { cubeJoins.add(d.getJoin()); } } for (JoinDesc j : joins) { // optiq engine can't decide which one is fk or pk String pTable = j.getPrimaryKeyColumns()[0].getTable(); String factTable = cubeDesc.getFactTable(); if (factTable.equals(pTable)) { j.swapPKFK(); } // check primary key, all PK column should refer to same tale, the Fact Table of cube. // Using first column's table name to check. String fTable = j.getForeignKeyColumns()[0].getTable(); if (!factTable.equals(fTable)) { logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + cube.getName()); return false; } // The hashcode() function of JoinDesc has been overwritten, // which takes into consideration: pk,fk,jointype if (!cubeJoins.contains(j)) { logger.info("Query joins don't macth on cube " + cube.getName()); return false; } } return true; }
Example #14
Source File: CubeDescUpgrader.java From Kylin with Apache License 2.0 | 5 votes |
private DataModelDesc extractDataModel(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) { DataModelDesc dm = new DataModelDesc(); dm.setUuid(UUID.randomUUID().toString()); String factTable = oldModel.getFactTable(); dm.setName(oldModel.getName()); dm.setFactTable(getMetadataManager().appendDBName(factTable)); newModel.setModelName(dm.getName()); List<org.apache.kylin.cube.model.v1.DimensionDesc> oldDimensions = oldModel.getDimensions(); List<LookupDesc> lookups = Lists.newArrayList(); for (org.apache.kylin.cube.model.v1.DimensionDesc dim : oldDimensions) { JoinDesc join = dim.getJoin(); if (join != null && !StringUtils.isEmpty(join.getType()) && join.getForeignKey() != null && join.getForeignKey().length > 0) { LookupDesc lookup = new LookupDesc(); lookup.setJoin(join); String table = dim.getTable(); lookup.setTable(getMetadataManager().appendDBName(table)); lookups.add(lookup); } } dm.setLookups(lookups.toArray(new LookupDesc[lookups.size()])); dm.setFilterCondition(oldModel.getFilterCondition()); updatePartitionDesc(oldModel, dm); if (oldModel.getCapacity() == CubeDesc.CubeCapacity.SMALL) { dm.setCapacity(DataModelDesc.RealizationCapacity.SMALL); } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.MEDIUM) { dm.setCapacity(DataModelDesc.RealizationCapacity.MEDIUM); } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.LARGE) { dm.setCapacity(DataModelDesc.RealizationCapacity.LARGE); } return dm; }
Example #15
Source File: FactTableGenerator.java From Kylin with Apache License 2.0 | 5 votes |
private List<DimensionDesc> getSortedDimentsionDescs() { List<DimensionDesc> dimensions = desc.getDimensions(); Collections.sort(dimensions, new Comparator<DimensionDesc>() { @Override public int compare(DimensionDesc o1, DimensionDesc o2) { JoinDesc j1 = o2.getJoin(); JoinDesc j2 = o1.getJoin(); return Integer.valueOf(j1 != null ? j1.getPrimaryKey().length : 0).compareTo(j2 != null ? j2.getPrimaryKey().length : 0); } }); return dimensions; }
Example #16
Source File: NewBaseCuboidMapper.java From Kylin with Apache License 2.0 | 5 votes |
private LinkedList<Pair<Integer, Integer>> findColumnRowKeyRelationships(DimensionDesc dim) { LinkedList<Pair<Integer, Integer>> dimTblColAsRowKey = new LinkedList<Pair<Integer, Integer>>(); for (TblColRef colRef : dim.getColumnRefs()) { int dimTableIndex = colRef.getColumn().getZeroBasedIndex(); int rowKeyIndex = cubeDesc.getRowkey().getRowKeyIndexByColumnName(colRef.getName()); dimTblColAsRowKey.add(new Pair<Integer, Integer>(dimTableIndex, rowKeyIndex)); } return dimTblColAsRowKey; }
Example #17
Source File: CubeSizeEstimationCLI.java From Kylin with Apache License 2.0 | 5 votes |
private static RowKeyColInfo extractRowKeyInfo(CubeDesc cubeDesc) { RowKeyDesc rowKeyDesc = cubeDesc.getRowkey(); RowKeyColInfo info = new RowKeyColInfo(); info.hierachyColBitIndice = new ArrayList<List<Integer>>(); info.nonHierachyColBitIndice = new ArrayList<Integer>(); HashSet<Integer> heirachyIndexSet = new HashSet<Integer>(); for (DimensionDesc dim : cubeDesc.getDimensions()) { if (dim.getHierarchy() != null) { LinkedList<Integer> hlist = new LinkedList<Integer>(); for (HierarchyDesc hierarchyDesc : dim.getHierarchy()) { int index = rowKeyDesc.getColumnBitIndex(hierarchyDesc.getColumnRef()); hlist.add(index); heirachyIndexSet.add(index); } info.hierachyColBitIndice.add(hlist); } } for (int i = 0; i < rowKeyDesc.getRowKeyColumns().length; ++i) { if (!heirachyIndexSet.contains(i)) { info.nonHierachyColBitIndice.add(i); } } return info; }
Example #18
Source File: NewBaseCuboidMapper.java From Kylin with Apache License 2.0 | 5 votes |
private void prepareJoins() throws IOException { this.lookupTables = new HashMap<String, LookupBytesTable>(); this.tableJoins = new LinkedList<TableJoin>(); this.factTblColAsRowKey = new LinkedList<Pair<Integer, Integer>>(); for (DimensionDesc dim : cubeDesc.getDimensions()) { JoinDesc join = dim.getJoin(); if (join != null) { String joinType = join.getType().toUpperCase(); String lookupTableName = dim.getTable(); // load lookup tables if (!lookupTables.containsKey(lookupTableName)) { HiveTable htable = new HiveTable(metadataManager, lookupTableName); LookupBytesTable btable = new LookupBytesTable(metadataManager.getTableDesc(lookupTableName), join.getPrimaryKey(), htable); lookupTables.put(lookupTableName, btable); } // create join infos LinkedList<Integer> fkIndice = new LinkedList<Integer>(); for (TblColRef colRef : join.getForeignKeyColumns()) { fkIndice.add(colRef.getColumn().getZeroBasedIndex()); } this.tableJoins.add(new TableJoin(joinType, fkIndice, lookupTableName, this.findColumnRowKeyRelationships(dim))); } else { this.factTblColAsRowKey.addAll(this.findColumnRowKeyRelationships(dim)); } } // put composite keys joins ahead of single key joins Collections.sort(tableJoins, new Comparator<TableJoin>() { @Override public int compare(TableJoin o1, TableJoin o2) { return Integer.valueOf(o2.fkIndice.size()).compareTo(Integer.valueOf(o1.fkIndice.size())); } }); }
Example #19
Source File: SparkBuildDictionary.java From kylin with Apache License 2.0 | 5 votes |
@Override public Boolean call(DimensionDesc dimensionDesc) throws Exception { if (initialized == false) { synchronized (SparkBuildDictionary.class) { if (initialized == false) { init(); } } } return !cubeSegment.getCubeDesc().isExtSnapshotTable(dimensionDesc.getTableRef().getTableIdentity()); }
Example #20
Source File: StreamingCubeRule.java From kylin with Apache License 2.0 | 5 votes |
@Override public void validate(CubeDesc cube, ValidateContext context) { DataModelDesc model = cube.getModel(); if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING && !model.getRootFactTable().getTableDesc().isStreamingTable()) { return; } if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) { context.addResult(ResultLevel.ERROR, "Must define a partition column."); return; } final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef(); boolean found = false; for (DimensionDesc dimensionDesc : cube.getDimensions()) { for (TblColRef dimCol : dimensionDesc.getColumnRefs()) { if (dimCol.equals(partitionCol)) { found = true; break; } } } if (found == false) { context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list."); return; } }
Example #21
Source File: CubeDescCreator.java From kylin with Apache License 2.0 | 5 votes |
public static DimensionDesc getDimensionDesc(String tableName, String dimension) { DimensionDesc dimensionDesc = new DimensionDesc(); dimensionDesc.setName(dimension); dimensionDesc.setTable(tableName.substring(tableName.lastIndexOf(".") + 1)); dimensionDesc.setColumn(dimension); return dimensionDesc; }
Example #22
Source File: CubeControllerTest.java From kylin with Apache License 2.0 | 5 votes |
@Test public void testGetSql() { GeneralResponse response = cubeController.getSql("test_kylin_cube_with_slr_ready"); String sql = response.getProperty("sql"); CubeDesc cubeDesc = cubeDescController.getDesc("test_kylin_cube_with_slr_ready"); for (DimensionDesc dimensionDesc : cubeDesc.getDimensions()) { if (dimensionDesc.getDerived() != null) { for (String derivedDimension : dimensionDesc.getDerived()) { Assert.assertTrue(sql.contains(derivedDimension)); } } } }
Example #23
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
public static DimensionDesc getDimensionDesc(String tableName, String dimension) { DimensionDesc dimensionDesc = new DimensionDesc(); dimensionDesc.setName(dimension); dimensionDesc.setTable(tableName.substring(tableName.lastIndexOf(".") + 1)); dimensionDesc.setColumn(dimension); return dimensionDesc; }
Example #24
Source File: CubeControllerTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Test public void testGetSql() { GeneralResponse response = cubeController.getSql("test_kylin_cube_with_slr_ready"); String sql = response.getProperty("sql"); CubeDesc cubeDesc = cubeDescController.getDesc("test_kylin_cube_with_slr_ready"); for (DimensionDesc dimensionDesc : cubeDesc.getDimensions()) { if (dimensionDesc.getDerived() != null) { for (String derivedDimension : dimensionDesc.getDerived()) { Assert.assertTrue(sql.contains(derivedDimension)); } } } }
Example #25
Source File: StreamingCubeRule.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Override public void validate(CubeDesc cube, ValidateContext context) { DataModelDesc model = cube.getModel(); if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING && !model.getRootFactTable().getTableDesc().isStreamingTable()) { return; } if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) { context.addResult(ResultLevel.ERROR, "Must define a partition column."); return; } final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef(); boolean found = false; for (DimensionDesc dimensionDesc : cube.getDimensions()) { for (TblColRef dimCol : dimensionDesc.getColumnRefs()) { if (dimCol.equals(partitionCol)) { found = true; break; } } } if (found == false) { context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list."); return; } }
Example #26
Source File: SparkBuildDictionary.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Override public Boolean call(DimensionDesc dimensionDesc) throws Exception { if (initialized == false) { synchronized (SparkBuildDictionary.class) { if (initialized == false) { init(); } } } return !cubeSegment.getCubeDesc().isExtSnapshotTable(dimensionDesc.getTableRef().getTableIdentity()); }
Example #27
Source File: CubeDescTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
@Test public void testCiCube() { CubeDescManager mgr = CubeDescManager.getInstance(getTestConfig()); CubeDesc lc = mgr.getCubeDesc("ci_left_join_cube"); CubeDesc ic = mgr.getCubeDesc("ci_inner_join_cube"); assertNotNull(lc); assertNotNull(ic); // assert the two CI cubes are identical apart from the left/inner difference assertEquals(lc.getDimensions().size(), ic.getDimensions().size()); for (int i = 0, n = lc.getDimensions().size(); i < n; i++) { DimensionDesc ld = lc.getDimensions().get(i); DimensionDesc id = ic.getDimensions().get(i); assertEquals(ld.getTable(), id.getTable()); assertEquals(ld.getColumn(), id.getColumn()); assertArrayEquals(ld.getDerived(), id.getDerived()); } // To enable spark in IT, the inner cube removed the percentile measure, so ignore that particular measure List<MeasureDesc> lcMeasures = dropPercentile(lc.getMeasures()); List<MeasureDesc> icMeasures = ic.getMeasures(); assertEquals(lcMeasures.size(), icMeasures.size()); for (int i = 0, n = lcMeasures.size(); i < n; i++) { MeasureDesc lm = lcMeasures.get(i); MeasureDesc im = icMeasures.get(i); assertEquals(lm.getName(), im.getName()); assertEquals(lm.getFunction().getFullExpression(), im.getFunction().getFullExpression()); assertEquals(lm.getFunction().getReturnType(), im.getFunction().getReturnType()); } assertEquals(lc.getAggregationGroups().size(), ic.getAggregationGroups().size()); for (int i = 0, n = lc.getAggregationGroups().size(); i < n; i++) { AggregationGroup lag = lc.getAggregationGroups().get(i); AggregationGroup iag = ic.getAggregationGroups().get(i); assertArrayEquals(lag.getIncludes(), iag.getIncludes()); assertArrayEquals(lag.getSelectRule().mandatoryDims, iag.getSelectRule().mandatoryDims); assertArrayEquals(lag.getSelectRule().hierarchyDims, iag.getSelectRule().hierarchyDims); assertArrayEquals(lag.getSelectRule().jointDims, iag.getSelectRule().jointDims); } assertEquals(lc.listAllColumnDescs().size(), ic.listAllColumnDescs().size()); assertEquals(lc.listAllColumns().size(), ic.listAllColumns().size()); // test KYLIN-2440 assertTrue(lc.listAllColumns().contains(lc.getModel().findColumn("SELLER_ACCOUNT.ACCOUNT_ID"))); assertTrue(ic.listAllColumns().contains(ic.getModel().findColumn("SELLER_ACCOUNT.ACCOUNT_ID"))); }
Example #28
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQueryCube(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryCube(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); dimensions.remove(QueryCubePropertyEnum.PROJECT.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQueryCube(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryCube(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); if (!measure.equals(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString())) { measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } } //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.SEGMENT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_SOURCE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_TARGET.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.FILTER_MASK.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_MATCH.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_SUCCESS.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[] mandatory_dims = new String[] { QueryCubePropertyEnum.CUBE.toString() }; mandatory_dims = refineColumnWithTable(tableName, mandatory_dims); String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } String[][] joint_dims = new String[1][]; joint_dims[0] = new String[] { QueryCubePropertyEnum.CUBOID_SOURCE.toString(), QueryCubePropertyEnum.CUBOID_TARGET.toString() }; for (int i = 0; i < joint_dims.length; i++) { joint_dims[i] = refineColumnWithTable(tableName, joint_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = mandatory_dims; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = joint_dims; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example #29
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsQueryRPC(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryRPC(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsQueryRPC(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryRPC(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); } measureDescList.add(getMeasurePercentile(QueryRPCPropertyEnum.CALL_TIME.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.REALIZATION.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.RPC_SERVER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.EXCEPTION.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }
Example #30
Source File: CubeDescCreator.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public static CubeDesc generateKylinCubeDescForMetricsJob(KylinConfig config, MetricsSinkDesc sinkDesc) { String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJob()); //Set for dimensions List<String> dimensions = ModelCreator.getDimensionsForMetricsJob(); dimensions.remove(TimePropertyEnum.DAY_TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString()); dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString()); List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size()); for (String dimensionName : dimensions) { dimensionDescList.add(getDimensionDesc(tableName, dimensionName)); } //Set for measures List<String> measures = ModelCreator.getMeasuresForMetricsJob(); List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize((measures.size() - 4) * 3 + 1 + 1 + 4); Set<String> stepDuration = Sets.newHashSet(); stepDuration.add(JobPropertyEnum.STEP_DURATION_DISTINCT_COLUMNS.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_DICTIONARY.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_INMEM_CUBING.toString()); stepDuration.add(JobPropertyEnum.STEP_DURATION_HFILE_CONVERT.toString()); List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsJob(); Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size()); for (Pair<String, String> entry : measureTypeList) { measureTypeMap.put(entry.getFirst(), entry.getSecond()); } measureDescList.add(getMeasureCount()); for (String measure : measures) { measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure))); measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure))); if (!stepDuration.contains(measure)) { measureDescList.add(getMeasureMin(measure, measureTypeMap.get(measure))); } } measureDescList.add(getMeasurePercentile(JobPropertyEnum.BUILD_DURATION.toString())); //Set for row key RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()]; int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs); rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1); idx++; rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1); idx++; RowKeyDesc rowKeyDesc = new RowKeyDesc(); rowKeyDesc.setRowkeyColumns(rowKeyColDescs); //Set for aggregation group String[][] hierarchy_dims = new String[1][]; hierarchy_dims[0] = getTimeHierarchy(); for (int i = 0; i < hierarchy_dims.length; i++) { hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]); } SelectRule selectRule = new SelectRule(); selectRule.mandatoryDims = new String[0]; selectRule.hierarchyDims = hierarchy_dims; selectRule.jointDims = new String[0][0]; AggregationGroup aggGroup = new AggregationGroup(); aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions)); aggGroup.setSelectRule(selectRule); //Set for hbase mapping HBaseMappingDesc hBaseMapping = new HBaseMappingDesc(); hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList)); return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList, rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties()); }