org.apache.kylin.cube.model.DimensionDesc Java Examples

private void checkCubeDesc(String descName) {
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeDesc cubedesc1 = cubeDescMgr.getCubeDesc(descName);
    Assert.assertNotNull(cubedesc1);
    DataModelDesc model = cubedesc1.getModel();
    Assert.assertNotNull(model);
    Assert.assertTrue(model.getLookups().length > 0);

    List<DimensionDesc> dims = cubedesc1.getDimensions();

    Assert.assertTrue(dims.size() > 0);

    for (DimensionDesc dim : dims) {
        Assert.assertTrue(dim.getColumn().length > 0);
    }

    Assert.assertTrue(cubedesc1.getMeasures().size() > 0);

    CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    List<CubeInstance> cubes = cubeMgr.getCubesByDesc(descName);

    Assert.assertTrue(cubes.size() > 0);
}
 

public static CubeDesc generateKylinCubeDesc(String tableName, int storageType,
        List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc,
        AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) {
    CubeDesc desc = new CubeDesc();
    desc.setName(tableName.replace('.', '_'));
    desc.setModelName(tableName.replace('.', '_'));
    desc.setDescription("");
    desc.setLastModified(0L);
    desc.setDimensions(dimensionDescList);
    desc.setMeasures(measureDescList);
    desc.setRowkey(rowKeyDesc);
    desc.setHbaseMapping(hBaseMapping);
    desc.setNotifyList(Lists.<String> newArrayList());
    desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString()));
    desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L, 7776000000L, 31104000000L });
    desc.setEngineType(IEngineAware.ID_MR_V2);
    desc.setStorageType(storageType);
    desc.setAggregationGroups(Lists.newArrayList(aggGroup));
    desc.getOverrideKylinProps().putAll(overrideProperties);
    desc.updateRandomUuid();
    return desc;
}
 

public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) {
    CubeDesc cubeDesc = cube.getDescriptor();
    Set<String> allLookupTables = Sets.newHashSet();
    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        TableRef table = dim.getTableRef();
        if (cubeDesc.getModel().isLookupTable(table)) {
            allLookupTables.add(table.getTableIdentity());
        }
    }
    List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList();
    for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) {
        if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType())
                && allLookupTables.contains(snapshotTableDesc.getTableName())) {
            addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc);
        }
    }
}
 

private void checkSnapshot(CubeManager cubeManager, CubeSegment cubeSegment) {
    List<DimensionDesc> dimensionDescs = cubeSegment.getCubeDesc().getDimensions();
    for (DimensionDesc dimensionDesc : dimensionDescs) {
        TableRef lookup = dimensionDesc.getTableRef();
        String tableIdentity = lookup.getTableIdentity();
        if (cubeSegment.getModel().isLookupTable(tableIdentity) && !cubeSegment.getCubeDesc().isExtSnapshotTable(tableIdentity)) {
            logger.info("Checking snapshot of {}", lookup);
            try {
                JoinDesc join = cubeSegment.getModel().getJoinsTree().getJoinByPKSide(lookup);
                ILookupTable table = cubeManager.getLookupTable(cubeSegment, join);
                if (table != null) {
                    IOUtils.closeStream(table);
                }
            } catch (Throwable th) {
                throw new RuntimeException(String.format(Locale.ROOT, "Checking snapshot of %s failed.", lookup), th);
            }
        }
    }
}
 

private void checkSnapshot(CubeManager cubeManager, CubeSegment cubeSegment) {
    List<DimensionDesc> dimensionDescs = cubeSegment.getCubeDesc().getDimensions();
    for (DimensionDesc dimensionDesc : dimensionDescs) {
        TableRef lookup = dimensionDesc.getTableRef();
        String tableIdentity = lookup.getTableIdentity();
        if (cubeSegment.getModel().isLookupTable(tableIdentity) && !cubeSegment.getCubeDesc().isExtSnapshotTable(tableIdentity)) {
            logger.info("Checking snapshot of {}", lookup);
            try {
                JoinDesc join = cubeSegment.getModel().getJoinsTree().getJoinByPKSide(lookup);
                ILookupTable table = cubeManager.getLookupTable(cubeSegment, join);
                if (table != null) {
                    IOUtils.closeStream(table);
                }
            } catch (Throwable th) {
                throw new RuntimeException(String.format(Locale.ROOT, "Checking snapshot of %s failed.", lookup), th);
            }
        }
    }
}
 

/**
 * check if {toCheck} has snapshot of {lookupTableName}
 * @param lookupTableName look like {SCHMEA}.{TABLE}
 */
private boolean checkMeetSnapshotTable(CubeInstance toCheck, String lookupTableName) {
    boolean checkRes = false;
    String lookupTbl = lookupTableName;
    String[] strArr = lookupTableName.split("\\.");
    if (strArr.length > 1) {
        lookupTbl = strArr[strArr.length - 1];
    }
    for (DimensionDesc dimensionDesc : toCheck.getDescriptor().getDimensions()) {
        if (dimensionDesc.getTableRef().getTableName().equalsIgnoreCase(lookupTbl)) {
            checkRes = true;
            break;
        }
    }
    return checkRes;
}
 

public LookupTableEnumerator(OLAPContext olapContext) {

        //TODO: assuming LookupTableEnumerator is handled by a cube
        CubeInstance cube = (CubeInstance) olapContext.realization;

        String lookupTableName = olapContext.firstTableScan.getTableName();
        DimensionDesc dim = cube.getDescriptor().findDimensionByTable(lookupTableName);
        if (dim == null)
            throw new IllegalStateException("No dimension with derived columns found for lookup table " + lookupTableName + ", cube desc " + cube.getDescriptor());

        CubeManager cubeMgr = CubeManager.getInstance(cube.getConfig());
        LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim);
        this.allRows = table.getAllRows();

        OLAPTable olapTable = (OLAPTable) olapContext.firstTableScan.getOlapTable();
        this.colDescs = olapTable.getExposedColumns();
        this.current = new Object[colDescs.size()];

        reset();
    }
 

public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) {
    CubeDesc cubeDesc = cube.getDescriptor();
    Set<String> allLookupTables = Sets.newHashSet();
    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        TableRef table = dim.getTableRef();
        if (cubeDesc.getModel().isLookupTable(table)) {
            allLookupTables.add(table.getTableIdentity());
        }
    }
    List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList();
    for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) {
        if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType())
                && allLookupTables.contains(snapshotTableDesc.getTableName())) {
            addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc);
        }
    }
}
 

public static CubeDesc generateKylinCubeDesc(String tableName, int storageType,
        List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc,
        AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) {
    CubeDesc desc = new CubeDesc();
    desc.setName(tableName.replace('.', '_'));
    desc.setModelName(tableName.replace('.', '_'));
    desc.setDescription("");
    desc.setLastModified(0L);
    desc.setDimensions(dimensionDescList);
    desc.setMeasures(measureDescList);
    desc.setRowkey(rowKeyDesc);
    desc.setHbaseMapping(hBaseMapping);
    desc.setNotifyList(Lists.<String> newArrayList());
    desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString()));
    desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L });
    desc.setEngineType(IEngineAware.ID_MR_V2);
    desc.setStorageType(storageType);
    desc.setAggregationGroups(Lists.newArrayList(aggGroup));
    desc.getOverrideKylinProps().putAll(overrideProperties);
    desc.updateRandomUuid();
    return desc;
}
 

private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) {
        // dictionary
        for (TblColRef col : dim.getColumnRefs()) {
            if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) {
                logger.info("Building dictionary for " + col);
                cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath);
            }
        }

        // build snapshot
        if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) {
            // CubeSegment seg = cube.getTheOnlySegment();
            logger.info("Building snapshot of " + dim.getTable());
            cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable());
            logger.info("Checking snapshot of " + dim.getTable());
            cubeMgr.getLookupTable(cubeSeg, dim); // load the table for
                                                  // sanity check
        }
    }
}
 

public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {

        String tableName = dim.getTable();
        String[] pkCols = dim.getJoin().getPrimaryKey();
        String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
        if (snapshotResPath == null)
            throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);

        try {
            SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
            TableDesc tableDesc = getMetadataManager().getTableDesc(tableName);
            return new LookupStringTable(tableDesc, pkCols, snapshot);
        } catch (IOException e) {
            throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e);
        }
    }
 

private DimensionDesc newDimensionDesc(org.apache.kylin.cube.model.v1.DimensionDesc dim, int dimId, String name) {
    DimensionDesc newDim = new DimensionDesc();

    newDim.setId(dimId);
    newDim.setName(name);
    newDim.setTable(getMetadataManager().appendDBName(dim.getTable()));

    return newDim;
}
 

private static boolean isMatchedWithJoins(Collection<JoinDesc> joins, CubeInstance cube) {
    CubeDesc cubeDesc = cube.getDescriptor();

    List<JoinDesc> cubeJoins = new ArrayList<JoinDesc>(cubeDesc.getDimensions().size());
    for (DimensionDesc d : cubeDesc.getDimensions()) {
        if (d.getJoin() != null) {
            cubeJoins.add(d.getJoin());
        }
    }
    for (JoinDesc j : joins) {
        // optiq engine can't decide which one is fk or pk
        String pTable = j.getPrimaryKeyColumns()[0].getTable();
        String factTable = cubeDesc.getFactTable();
        if (factTable.equals(pTable)) {
            j.swapPKFK();
        }

        // check primary key, all PK column should refer to same tale, the Fact Table of cube.
        // Using first column's table name to check.
        String fTable = j.getForeignKeyColumns()[0].getTable();
        if (!factTable.equals(fTable)) {
            logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + cube.getName());
            return false;
        }

        // The hashcode() function of JoinDesc has been overwritten,
        // which takes into consideration: pk,fk,jointype
        if (!cubeJoins.contains(j)) {
            logger.info("Query joins don't macth on cube " + cube.getName());
            return false;
        }
    }
    return true;
}
 

private DataModelDesc extractDataModel(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {

        DataModelDesc dm = new DataModelDesc();
        dm.setUuid(UUID.randomUUID().toString());
        String factTable = oldModel.getFactTable();
        dm.setName(oldModel.getName());
        dm.setFactTable(getMetadataManager().appendDBName(factTable));

        newModel.setModelName(dm.getName());

        List<org.apache.kylin.cube.model.v1.DimensionDesc> oldDimensions = oldModel.getDimensions();

        List<LookupDesc> lookups = Lists.newArrayList();
        for (org.apache.kylin.cube.model.v1.DimensionDesc dim : oldDimensions) {
            JoinDesc join = dim.getJoin();
            if (join != null && !StringUtils.isEmpty(join.getType()) && join.getForeignKey() != null && join.getForeignKey().length > 0) {
                LookupDesc lookup = new LookupDesc();
                lookup.setJoin(join);
                String table = dim.getTable();
                lookup.setTable(getMetadataManager().appendDBName(table));

                lookups.add(lookup);
            }
        }

        dm.setLookups(lookups.toArray(new LookupDesc[lookups.size()]));
        dm.setFilterCondition(oldModel.getFilterCondition());
        updatePartitionDesc(oldModel, dm);


        if (oldModel.getCapacity() == CubeDesc.CubeCapacity.SMALL) {
            dm.setCapacity(DataModelDesc.RealizationCapacity.SMALL);
        } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.MEDIUM) {
            dm.setCapacity(DataModelDesc.RealizationCapacity.MEDIUM);
        } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.LARGE) {
            dm.setCapacity(DataModelDesc.RealizationCapacity.LARGE);
        }

        return dm;
    }
 

private List<DimensionDesc> getSortedDimentsionDescs() {
    List<DimensionDesc> dimensions = desc.getDimensions();
    Collections.sort(dimensions, new Comparator<DimensionDesc>() {
        @Override
        public int compare(DimensionDesc o1, DimensionDesc o2) {
            JoinDesc j1 = o2.getJoin();
            JoinDesc j2 = o1.getJoin();
            return Integer.valueOf(j1 != null ? j1.getPrimaryKey().length : 0).compareTo(j2 != null ? j2.getPrimaryKey().length : 0);
        }
    });
    return dimensions;
}
 

private LinkedList<Pair<Integer, Integer>> findColumnRowKeyRelationships(DimensionDesc dim) {
    LinkedList<Pair<Integer, Integer>> dimTblColAsRowKey = new LinkedList<Pair<Integer, Integer>>();
    for (TblColRef colRef : dim.getColumnRefs()) {
        int dimTableIndex = colRef.getColumn().getZeroBasedIndex();
        int rowKeyIndex = cubeDesc.getRowkey().getRowKeyIndexByColumnName(colRef.getName());
        dimTblColAsRowKey.add(new Pair<Integer, Integer>(dimTableIndex, rowKeyIndex));
    }
    return dimTblColAsRowKey;
}
 

private static RowKeyColInfo extractRowKeyInfo(CubeDesc cubeDesc) {
    RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
    RowKeyColInfo info = new RowKeyColInfo();
    info.hierachyColBitIndice = new ArrayList<List<Integer>>();
    info.nonHierachyColBitIndice = new ArrayList<Integer>();
    HashSet<Integer> heirachyIndexSet = new HashSet<Integer>();

    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        if (dim.getHierarchy() != null) {
            LinkedList<Integer> hlist = new LinkedList<Integer>();
            for (HierarchyDesc hierarchyDesc : dim.getHierarchy()) {
                int index = rowKeyDesc.getColumnBitIndex(hierarchyDesc.getColumnRef());
                hlist.add(index);
                heirachyIndexSet.add(index);
            }
            info.hierachyColBitIndice.add(hlist);
        }
    }

    for (int i = 0; i < rowKeyDesc.getRowKeyColumns().length; ++i) {
        if (!heirachyIndexSet.contains(i)) {
            info.nonHierachyColBitIndice.add(i);
        }
    }

    return info;
}
 

private void prepareJoins() throws IOException {
    this.lookupTables = new HashMap<String, LookupBytesTable>();
    this.tableJoins = new LinkedList<TableJoin>();
    this.factTblColAsRowKey = new LinkedList<Pair<Integer, Integer>>();

    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        JoinDesc join = dim.getJoin();
        if (join != null) {
            String joinType = join.getType().toUpperCase();
            String lookupTableName = dim.getTable();

            // load lookup tables
            if (!lookupTables.containsKey(lookupTableName)) {
                HiveTable htable = new HiveTable(metadataManager, lookupTableName);
                LookupBytesTable btable = new LookupBytesTable(metadataManager.getTableDesc(lookupTableName), join.getPrimaryKey(), htable);
                lookupTables.put(lookupTableName, btable);
            }

            // create join infos
            LinkedList<Integer> fkIndice = new LinkedList<Integer>();
            for (TblColRef colRef : join.getForeignKeyColumns()) {
                fkIndice.add(colRef.getColumn().getZeroBasedIndex());
            }
            this.tableJoins.add(new TableJoin(joinType, fkIndice, lookupTableName, this.findColumnRowKeyRelationships(dim)));

        } else {

            this.factTblColAsRowKey.addAll(this.findColumnRowKeyRelationships(dim));
        }
    }

    // put composite keys joins ahead of single key joins
    Collections.sort(tableJoins, new Comparator<TableJoin>() {
        @Override
        public int compare(TableJoin o1, TableJoin o2) {
            return Integer.valueOf(o2.fkIndice.size()).compareTo(Integer.valueOf(o1.fkIndice.size()));
        }
    });
}
 

@Override
public Boolean call(DimensionDesc dimensionDesc) throws Exception {
    if (initialized == false) {
        synchronized (SparkBuildDictionary.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    return !cubeSegment.getCubeDesc().isExtSnapshotTable(dimensionDesc.getTableRef().getTableIdentity());
}
 

@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
 

public static DimensionDesc getDimensionDesc(String tableName, String dimension) {
    DimensionDesc dimensionDesc = new DimensionDesc();
    dimensionDesc.setName(dimension);
    dimensionDesc.setTable(tableName.substring(tableName.lastIndexOf(".") + 1));
    dimensionDesc.setColumn(dimension);
    return dimensionDesc;
}
 

@Test
public void testGetSql() {
    GeneralResponse response = cubeController.getSql("test_kylin_cube_with_slr_ready");
    String sql = response.getProperty("sql");
    CubeDesc cubeDesc = cubeDescController.getDesc("test_kylin_cube_with_slr_ready");

    for (DimensionDesc dimensionDesc : cubeDesc.getDimensions()) {
        if (dimensionDesc.getDerived() != null) {
            for (String derivedDimension : dimensionDesc.getDerived()) {
                Assert.assertTrue(sql.contains(derivedDimension));
            }
        }
    }
}
 

public static DimensionDesc getDimensionDesc(String tableName, String dimension) {
    DimensionDesc dimensionDesc = new DimensionDesc();
    dimensionDesc.setName(dimension);
    dimensionDesc.setTable(tableName.substring(tableName.lastIndexOf(".") + 1));
    dimensionDesc.setColumn(dimension);
    return dimensionDesc;
}
 

@Test
public void testGetSql() {
    GeneralResponse response = cubeController.getSql("test_kylin_cube_with_slr_ready");
    String sql = response.getProperty("sql");
    CubeDesc cubeDesc = cubeDescController.getDesc("test_kylin_cube_with_slr_ready");

    for (DimensionDesc dimensionDesc : cubeDesc.getDimensions()) {
        if (dimensionDesc.getDerived() != null) {
            for (String derivedDimension : dimensionDesc.getDerived()) {
                Assert.assertTrue(sql.contains(derivedDimension));
            }
        }
    }
}
 

@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
 

@Override
public Boolean call(DimensionDesc dimensionDesc) throws Exception {
    if (initialized == false) {
        synchronized (SparkBuildDictionary.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    return !cubeSegment.getCubeDesc().isExtSnapshotTable(dimensionDesc.getTableRef().getTableIdentity());
}
 

@Test
public void testCiCube() {
    CubeDescManager mgr = CubeDescManager.getInstance(getTestConfig());
    CubeDesc lc = mgr.getCubeDesc("ci_left_join_cube");
    CubeDesc ic = mgr.getCubeDesc("ci_inner_join_cube");
    assertNotNull(lc);
    assertNotNull(ic);

    // assert the two CI cubes are identical apart from the left/inner difference
    assertEquals(lc.getDimensions().size(), ic.getDimensions().size());
    for (int i = 0, n = lc.getDimensions().size(); i < n; i++) {
        DimensionDesc ld = lc.getDimensions().get(i);
        DimensionDesc id = ic.getDimensions().get(i);
        assertEquals(ld.getTable(), id.getTable());
        assertEquals(ld.getColumn(), id.getColumn());
        assertArrayEquals(ld.getDerived(), id.getDerived());
    }

    // To enable spark in IT, the inner cube removed the percentile measure, so ignore that particular measure
    List<MeasureDesc> lcMeasures = dropPercentile(lc.getMeasures());
    List<MeasureDesc> icMeasures = ic.getMeasures();

    assertEquals(lcMeasures.size(), icMeasures.size());
    for (int i = 0, n = lcMeasures.size(); i < n; i++) {
        MeasureDesc lm = lcMeasures.get(i);
        MeasureDesc im = icMeasures.get(i);
        assertEquals(lm.getName(), im.getName());
        assertEquals(lm.getFunction().getFullExpression(), im.getFunction().getFullExpression());
        assertEquals(lm.getFunction().getReturnType(), im.getFunction().getReturnType());
    }

    assertEquals(lc.getAggregationGroups().size(), ic.getAggregationGroups().size());
    for (int i = 0, n = lc.getAggregationGroups().size(); i < n; i++) {
        AggregationGroup lag = lc.getAggregationGroups().get(i);
        AggregationGroup iag = ic.getAggregationGroups().get(i);
        assertArrayEquals(lag.getIncludes(), iag.getIncludes());
        assertArrayEquals(lag.getSelectRule().mandatoryDims, iag.getSelectRule().mandatoryDims);
        assertArrayEquals(lag.getSelectRule().hierarchyDims, iag.getSelectRule().hierarchyDims);
        assertArrayEquals(lag.getSelectRule().jointDims, iag.getSelectRule().jointDims);
    }

    assertEquals(lc.listAllColumnDescs().size(), ic.listAllColumnDescs().size());
    assertEquals(lc.listAllColumns().size(), ic.listAllColumns().size());

    // test KYLIN-2440
    assertTrue(lc.listAllColumns().contains(lc.getModel().findColumn("SELLER_ACCOUNT.ACCOUNT_ID")));
    assertTrue(ic.listAllColumns().contains(ic.getModel().findColumn("SELLER_ACCOUNT.ACCOUNT_ID")));
}
 

public static CubeDesc generateKylinCubeDescForMetricsQueryCube(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryCube();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());
    dimensions.remove(QueryCubePropertyEnum.PROJECT.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQueryCube();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryCube();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        if (!measure.equals(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString())) {
            measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
        }
    }

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.SEGMENT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_SOURCE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_TARGET.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.FILTER_MASK.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_MATCH.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_SUCCESS.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[] mandatory_dims = new String[] { QueryCubePropertyEnum.CUBE.toString() };
    mandatory_dims = refineColumnWithTable(tableName, mandatory_dims);

    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    String[][] joint_dims = new String[1][];
    joint_dims[0] = new String[] { QueryCubePropertyEnum.CUBOID_SOURCE.toString(),
            QueryCubePropertyEnum.CUBOID_TARGET.toString() };
    for (int i = 0; i < joint_dims.length; i++) {
        joint_dims[i] = refineColumnWithTable(tableName, joint_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = mandatory_dims;
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = joint_dims;

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 

public static CubeDesc generateKylinCubeDescForMetricsQueryRPC(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryRPC();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQueryRPC();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryRPC();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
    }
    measureDescList.add(getMeasurePercentile(QueryRPCPropertyEnum.CALL_TIME.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.REALIZATION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.RPC_SERVER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.EXCEPTION.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 

public static CubeDesc generateKylinCubeDescForMetricsJob(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJob());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsJob();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsJob();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize((measures.size() - 4) * 3 + 1 + 1 + 4);

    Set<String> stepDuration = Sets.newHashSet();
    stepDuration.add(JobPropertyEnum.STEP_DURATION_DISTINCT_COLUMNS.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_DICTIONARY.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_INMEM_CUBING.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_HFILE_CONVERT.toString());

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsJob();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
        if (!stepDuration.contains(measure)) {
            measureDescList.add(getMeasureMin(measure, measureTypeMap.get(measure)));
        }
    }
    measureDescList.add(getMeasurePercentile(JobPropertyEnum.BUILD_DURATION.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}