Java Code Examples for org.apache.kylin.cube.model.DimensionDesc

The following examples show how to use org.apache.kylin.cube.model.DimensionDesc. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 6 votes vote down vote up
public static CubeDesc generateKylinCubeDesc(String tableName, int storageType,
        List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc,
        AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) {
    CubeDesc desc = new CubeDesc();
    desc.setName(tableName.replace('.', '_'));
    desc.setModelName(tableName.replace('.', '_'));
    desc.setDescription("");
    desc.setLastModified(0L);
    desc.setDimensions(dimensionDescList);
    desc.setMeasures(measureDescList);
    desc.setRowkey(rowKeyDesc);
    desc.setHbaseMapping(hBaseMapping);
    desc.setNotifyList(Lists.<String> newArrayList());
    desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString()));
    desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L });
    desc.setEngineType(IEngineAware.ID_MR_V2);
    desc.setStorageType(storageType);
    desc.setAggregationGroups(Lists.newArrayList(aggGroup));
    desc.getOverrideKylinProps().putAll(overrideProperties);
    desc.updateRandomUuid();
    return desc;
}
 
Example 2
Source Project: kylin-on-parquet-v2   Source File: HBaseLookupMRSteps.java    License: Apache License 2.0 6 votes vote down vote up
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) {
    CubeDesc cubeDesc = cube.getDescriptor();
    Set<String> allLookupTables = Sets.newHashSet();
    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        TableRef table = dim.getTableRef();
        if (cubeDesc.getModel().isLookupTable(table)) {
            allLookupTables.add(table.getTableIdentity());
        }
    }
    List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList();
    for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) {
        if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType())
                && allLookupTables.contains(snapshotTableDesc.getTableName())) {
            addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc);
        }
    }
}
 
Example 3
Source Project: kylin-on-parquet-v2   Source File: CubeManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * check if {toCheck} has snapshot of {lookupTableName}
 * @param lookupTableName look like {SCHMEA}.{TABLE}
 */
private boolean checkMeetSnapshotTable(CubeInstance toCheck, String lookupTableName) {
    boolean checkRes = false;
    String lookupTbl = lookupTableName;
    String[] strArr = lookupTableName.split("\\.");
    if (strArr.length > 1) {
        lookupTbl = strArr[strArr.length - 1];
    }
    for (DimensionDesc dimensionDesc : toCheck.getDescriptor().getDimensions()) {
        if (dimensionDesc.getTableRef().getTableName().equalsIgnoreCase(lookupTbl)) {
            checkRes = true;
            break;
        }
    }
    return checkRes;
}
 
Example 4
Source Project: kylin-on-parquet-v2   Source File: SparkBuildDictionary.java    License: Apache License 2.0 6 votes vote down vote up
private void checkSnapshot(CubeManager cubeManager, CubeSegment cubeSegment) {
    List<DimensionDesc> dimensionDescs = cubeSegment.getCubeDesc().getDimensions();
    for (DimensionDesc dimensionDesc : dimensionDescs) {
        TableRef lookup = dimensionDesc.getTableRef();
        String tableIdentity = lookup.getTableIdentity();
        if (cubeSegment.getModel().isLookupTable(tableIdentity) && !cubeSegment.getCubeDesc().isExtSnapshotTable(tableIdentity)) {
            logger.info("Checking snapshot of {}", lookup);
            try {
                JoinDesc join = cubeSegment.getModel().getJoinsTree().getJoinByPKSide(lookup);
                ILookupTable table = cubeManager.getLookupTable(cubeSegment, join);
                if (table != null) {
                    IOUtils.closeStream(table);
                }
            } catch (Throwable th) {
                throw new RuntimeException(String.format(Locale.ROOT, "Checking snapshot of %s failed.", lookup), th);
            }
        }
    }
}
 
Example 5
Source Project: kylin   Source File: CubeDescCreator.java    License: Apache License 2.0 6 votes vote down vote up
public static CubeDesc generateKylinCubeDesc(String tableName, int storageType,
        List<DimensionDesc> dimensionDescList, List<MeasureDesc> measureDescList, RowKeyDesc rowKeyDesc,
        AggregationGroup aggGroup, HBaseMappingDesc hBaseMapping, Map<String, String> overrideProperties) {
    CubeDesc desc = new CubeDesc();
    desc.setName(tableName.replace('.', '_'));
    desc.setModelName(tableName.replace('.', '_'));
    desc.setDescription("");
    desc.setLastModified(0L);
    desc.setDimensions(dimensionDescList);
    desc.setMeasures(measureDescList);
    desc.setRowkey(rowKeyDesc);
    desc.setHbaseMapping(hBaseMapping);
    desc.setNotifyList(Lists.<String> newArrayList());
    desc.setStatusNeedNotify(Lists.newArrayList(JobStatusEnum.ERROR.toString()));
    desc.setAutoMergeTimeRanges(new long[] { 86400000L, 604800000L, 2419200000L, 7776000000L, 31104000000L });
    desc.setEngineType(IEngineAware.ID_MR_V2);
    desc.setStorageType(storageType);
    desc.setAggregationGroups(Lists.newArrayList(aggGroup));
    desc.getOverrideKylinProps().putAll(overrideProperties);
    desc.updateRandomUuid();
    return desc;
}
 
Example 6
Source Project: kylin   Source File: HBaseLookupMRSteps.java    License: Apache License 2.0 6 votes vote down vote up
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) {
    CubeDesc cubeDesc = cube.getDescriptor();
    Set<String> allLookupTables = Sets.newHashSet();
    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        TableRef table = dim.getTableRef();
        if (cubeDesc.getModel().isLookupTable(table)) {
            allLookupTables.add(table.getTableIdentity());
        }
    }
    List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList();
    for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) {
        if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType())
                && allLookupTables.contains(snapshotTableDesc.getTableName())) {
            addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc);
        }
    }
}
 
Example 7
Source Project: kylin   Source File: SparkBuildDictionary.java    License: Apache License 2.0 6 votes vote down vote up
private void checkSnapshot(CubeManager cubeManager, CubeSegment cubeSegment) {
    List<DimensionDesc> dimensionDescs = cubeSegment.getCubeDesc().getDimensions();
    for (DimensionDesc dimensionDesc : dimensionDescs) {
        TableRef lookup = dimensionDesc.getTableRef();
        String tableIdentity = lookup.getTableIdentity();
        if (cubeSegment.getModel().isLookupTable(tableIdentity) && !cubeSegment.getCubeDesc().isExtSnapshotTable(tableIdentity)) {
            logger.info("Checking snapshot of {}", lookup);
            try {
                JoinDesc join = cubeSegment.getModel().getJoinsTree().getJoinByPKSide(lookup);
                ILookupTable table = cubeManager.getLookupTable(cubeSegment, join);
                if (table != null) {
                    IOUtils.closeStream(table);
                }
            } catch (Throwable th) {
                throw new RuntimeException(String.format(Locale.ROOT, "Checking snapshot of %s failed.", lookup), th);
            }
        }
    }
}
 
Example 8
Source Project: Kylin   Source File: LookupTableEnumerator.java    License: Apache License 2.0 6 votes vote down vote up
public LookupTableEnumerator(OLAPContext olapContext) {

        //TODO: assuming LookupTableEnumerator is handled by a cube
        CubeInstance cube = (CubeInstance) olapContext.realization;

        String lookupTableName = olapContext.firstTableScan.getTableName();
        DimensionDesc dim = cube.getDescriptor().findDimensionByTable(lookupTableName);
        if (dim == null)
            throw new IllegalStateException("No dimension with derived columns found for lookup table " + lookupTableName + ", cube desc " + cube.getDescriptor());

        CubeManager cubeMgr = CubeManager.getInstance(cube.getConfig());
        LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim);
        this.allRows = table.getAllRows();

        OLAPTable olapTable = (OLAPTable) olapContext.firstTableScan.getOlapTable();
        this.colDescs = olapTable.getExposedColumns();
        this.current = new Object[colDescs.size()];

        reset();
    }
 
Example 9
Source Project: Kylin   Source File: DictionaryGeneratorCLI.java    License: Apache License 2.0 6 votes vote down vote up
private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) {
        // dictionary
        for (TblColRef col : dim.getColumnRefs()) {
            if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) {
                logger.info("Building dictionary for " + col);
                cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath);
            }
        }

        // build snapshot
        if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) {
            // CubeSegment seg = cube.getTheOnlySegment();
            logger.info("Building snapshot of " + dim.getTable());
            cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable());
            logger.info("Checking snapshot of " + dim.getTable());
            cubeMgr.getLookupTable(cubeSeg, dim); // load the table for
                                                  // sanity check
        }
    }
}
 
Example 10
Source Project: Kylin   Source File: CubeManager.java    License: Apache License 2.0 6 votes vote down vote up
public LookupStringTable getLookupTable(CubeSegment cubeSegment, DimensionDesc dim) {

        String tableName = dim.getTable();
        String[] pkCols = dim.getJoin().getPrimaryKey();
        String snapshotResPath = cubeSegment.getSnapshotResPath(tableName);
        if (snapshotResPath == null)
            throw new IllegalStateException("No snaphot for table '" + tableName + "' found on cube segment" + cubeSegment.getCubeInstance().getName() + "/" + cubeSegment);

        try {
            SnapshotTable snapshot = getSnapshotManager().getSnapshotTable(snapshotResPath);
            TableDesc tableDesc = getMetadataManager().getTableDesc(tableName);
            return new LookupStringTable(tableDesc, pkCols, snapshot);
        } catch (IOException e) {
            throw new IllegalStateException("Failed to load lookup table " + tableName + " from snapshot " + snapshotResPath, e);
        }
    }
 
Example 11
Source Project: Kylin   Source File: MetadataUpgradeTest.java    License: Apache License 2.0 6 votes vote down vote up
private void checkCubeDesc(String descName) {
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeDesc cubedesc1 = cubeDescMgr.getCubeDesc(descName);
    Assert.assertNotNull(cubedesc1);
    DataModelDesc model = cubedesc1.getModel();
    Assert.assertNotNull(model);
    Assert.assertTrue(model.getLookups().length > 0);

    List<DimensionDesc> dims = cubedesc1.getDimensions();

    Assert.assertTrue(dims.size() > 0);

    for (DimensionDesc dim : dims) {
        Assert.assertTrue(dim.getColumn().length > 0);
    }

    Assert.assertTrue(cubedesc1.getMeasures().size() > 0);

    CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    List<CubeInstance> cubes = cubeMgr.getCubesByDesc(descName);

    Assert.assertTrue(cubes.size() > 0);
}
 
Example 12
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 5 votes vote down vote up
public static DimensionDesc getDimensionDesc(String tableName, String dimension) {
    DimensionDesc dimensionDesc = new DimensionDesc();
    dimensionDesc.setName(dimension);
    dimensionDesc.setTable(tableName.substring(tableName.lastIndexOf(".") + 1));
    dimensionDesc.setColumn(dimension);
    return dimensionDesc;
}
 
Example 13
Source Project: kylin-on-parquet-v2   Source File: CubeControllerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetSql() {
    GeneralResponse response = cubeController.getSql("test_kylin_cube_with_slr_ready");
    String sql = response.getProperty("sql");
    CubeDesc cubeDesc = cubeDescController.getDesc("test_kylin_cube_with_slr_ready");

    for (DimensionDesc dimensionDesc : cubeDesc.getDimensions()) {
        if (dimensionDesc.getDerived() != null) {
            for (String derivedDimension : dimensionDesc.getDerived()) {
                Assert.assertTrue(sql.contains(derivedDimension));
            }
        }
    }
}
 
Example 14
Source Project: kylin-on-parquet-v2   Source File: StreamingCubeRule.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
 
Example 15
Source Project: kylin-on-parquet-v2   Source File: SparkBuildDictionary.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Boolean call(DimensionDesc dimensionDesc) throws Exception {
    if (initialized == false) {
        synchronized (SparkBuildDictionary.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    return !cubeSegment.getCubeDesc().isExtSnapshotTable(dimensionDesc.getTableRef().getTableIdentity());
}
 
Example 16
Source Project: kylin   Source File: CubeDescCreator.java    License: Apache License 2.0 5 votes vote down vote up
public static DimensionDesc getDimensionDesc(String tableName, String dimension) {
    DimensionDesc dimensionDesc = new DimensionDesc();
    dimensionDesc.setName(dimension);
    dimensionDesc.setTable(tableName.substring(tableName.lastIndexOf(".") + 1));
    dimensionDesc.setColumn(dimension);
    return dimensionDesc;
}
 
Example 17
Source Project: kylin   Source File: CubeControllerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetSql() {
    GeneralResponse response = cubeController.getSql("test_kylin_cube_with_slr_ready");
    String sql = response.getProperty("sql");
    CubeDesc cubeDesc = cubeDescController.getDesc("test_kylin_cube_with_slr_ready");

    for (DimensionDesc dimensionDesc : cubeDesc.getDimensions()) {
        if (dimensionDesc.getDerived() != null) {
            for (String derivedDimension : dimensionDesc.getDerived()) {
                Assert.assertTrue(sql.contains(derivedDimension));
            }
        }
    }
}
 
Example 18
Source Project: kylin   Source File: StreamingCubeRule.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
 
Example 19
Source Project: kylin   Source File: SparkBuildDictionary.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Boolean call(DimensionDesc dimensionDesc) throws Exception {
    if (initialized == false) {
        synchronized (SparkBuildDictionary.class) {
            if (initialized == false) {
                init();
            }
        }
    }

    return !cubeSegment.getCubeDesc().isExtSnapshotTable(dimensionDesc.getTableRef().getTableIdentity());
}
 
Example 20
Source Project: Kylin   Source File: NewBaseCuboidMapper.java    License: Apache License 2.0 5 votes vote down vote up
private void prepareJoins() throws IOException {
    this.lookupTables = new HashMap<String, LookupBytesTable>();
    this.tableJoins = new LinkedList<TableJoin>();
    this.factTblColAsRowKey = new LinkedList<Pair<Integer, Integer>>();

    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        JoinDesc join = dim.getJoin();
        if (join != null) {
            String joinType = join.getType().toUpperCase();
            String lookupTableName = dim.getTable();

            // load lookup tables
            if (!lookupTables.containsKey(lookupTableName)) {
                HiveTable htable = new HiveTable(metadataManager, lookupTableName);
                LookupBytesTable btable = new LookupBytesTable(metadataManager.getTableDesc(lookupTableName), join.getPrimaryKey(), htable);
                lookupTables.put(lookupTableName, btable);
            }

            // create join infos
            LinkedList<Integer> fkIndice = new LinkedList<Integer>();
            for (TblColRef colRef : join.getForeignKeyColumns()) {
                fkIndice.add(colRef.getColumn().getZeroBasedIndex());
            }
            this.tableJoins.add(new TableJoin(joinType, fkIndice, lookupTableName, this.findColumnRowKeyRelationships(dim)));

        } else {

            this.factTblColAsRowKey.addAll(this.findColumnRowKeyRelationships(dim));
        }
    }

    // put composite keys joins ahead of single key joins
    Collections.sort(tableJoins, new Comparator<TableJoin>() {
        @Override
        public int compare(TableJoin o1, TableJoin o2) {
            return Integer.valueOf(o2.fkIndice.size()).compareTo(Integer.valueOf(o1.fkIndice.size()));
        }
    });
}
 
Example 21
Source Project: Kylin   Source File: NewBaseCuboidMapper.java    License: Apache License 2.0 5 votes vote down vote up
private LinkedList<Pair<Integer, Integer>> findColumnRowKeyRelationships(DimensionDesc dim) {
    LinkedList<Pair<Integer, Integer>> dimTblColAsRowKey = new LinkedList<Pair<Integer, Integer>>();
    for (TblColRef colRef : dim.getColumnRefs()) {
        int dimTableIndex = colRef.getColumn().getZeroBasedIndex();
        int rowKeyIndex = cubeDesc.getRowkey().getRowKeyIndexByColumnName(colRef.getName());
        dimTblColAsRowKey.add(new Pair<Integer, Integer>(dimTableIndex, rowKeyIndex));
    }
    return dimTblColAsRowKey;
}
 
Example 22
Source Project: Kylin   Source File: FactTableGenerator.java    License: Apache License 2.0 5 votes vote down vote up
private List<DimensionDesc> getSortedDimentsionDescs() {
    List<DimensionDesc> dimensions = desc.getDimensions();
    Collections.sort(dimensions, new Comparator<DimensionDesc>() {
        @Override
        public int compare(DimensionDesc o1, DimensionDesc o2) {
            JoinDesc j1 = o2.getJoin();
            JoinDesc j2 = o1.getJoin();
            return Integer.valueOf(j1 != null ? j1.getPrimaryKey().length : 0).compareTo(j2 != null ? j2.getPrimaryKey().length : 0);
        }
    });
    return dimensions;
}
 
Example 23
Source Project: Kylin   Source File: CubeCapabilityChecker.java    License: Apache License 2.0 5 votes vote down vote up
private static boolean isMatchedWithJoins(Collection<JoinDesc> joins, CubeInstance cube) {
    CubeDesc cubeDesc = cube.getDescriptor();

    List<JoinDesc> cubeJoins = new ArrayList<JoinDesc>(cubeDesc.getDimensions().size());
    for (DimensionDesc d : cubeDesc.getDimensions()) {
        if (d.getJoin() != null) {
            cubeJoins.add(d.getJoin());
        }
    }
    for (JoinDesc j : joins) {
        // optiq engine can't decide which one is fk or pk
        String pTable = j.getPrimaryKeyColumns()[0].getTable();
        String factTable = cubeDesc.getFactTable();
        if (factTable.equals(pTable)) {
            j.swapPKFK();
        }

        // check primary key, all PK column should refer to same tale, the Fact Table of cube.
        // Using first column's table name to check.
        String fTable = j.getForeignKeyColumns()[0].getTable();
        if (!factTable.equals(fTable)) {
            logger.info("Fact Table" + factTable + " not matched in join: " + j + " on cube " + cube.getName());
            return false;
        }

        // The hashcode() function of JoinDesc has been overwritten,
        // which takes into consideration: pk,fk,jointype
        if (!cubeJoins.contains(j)) {
            logger.info("Query joins don't macth on cube " + cube.getName());
            return false;
        }
    }
    return true;
}
 
Example 24
Source Project: Kylin   Source File: CubeDescUpgrader.java    License: Apache License 2.0 5 votes vote down vote up
private DimensionDesc newDimensionDesc(org.apache.kylin.cube.model.v1.DimensionDesc dim, int dimId, String name) {
    DimensionDesc newDim = new DimensionDesc();

    newDim.setId(dimId);
    newDim.setName(name);
    newDim.setTable(getMetadataManager().appendDBName(dim.getTable()));

    return newDim;
}
 
Example 25
Source Project: Kylin   Source File: CubeDescUpgrader.java    License: Apache License 2.0 5 votes vote down vote up
private DataModelDesc extractDataModel(CubeDesc oldModel, org.apache.kylin.cube.model.CubeDesc newModel) {

        DataModelDesc dm = new DataModelDesc();
        dm.setUuid(UUID.randomUUID().toString());
        String factTable = oldModel.getFactTable();
        dm.setName(oldModel.getName());
        dm.setFactTable(getMetadataManager().appendDBName(factTable));

        newModel.setModelName(dm.getName());

        List<org.apache.kylin.cube.model.v1.DimensionDesc> oldDimensions = oldModel.getDimensions();

        List<LookupDesc> lookups = Lists.newArrayList();
        for (org.apache.kylin.cube.model.v1.DimensionDesc dim : oldDimensions) {
            JoinDesc join = dim.getJoin();
            if (join != null && !StringUtils.isEmpty(join.getType()) && join.getForeignKey() != null && join.getForeignKey().length > 0) {
                LookupDesc lookup = new LookupDesc();
                lookup.setJoin(join);
                String table = dim.getTable();
                lookup.setTable(getMetadataManager().appendDBName(table));

                lookups.add(lookup);
            }
        }

        dm.setLookups(lookups.toArray(new LookupDesc[lookups.size()]));
        dm.setFilterCondition(oldModel.getFilterCondition());
        updatePartitionDesc(oldModel, dm);


        if (oldModel.getCapacity() == CubeDesc.CubeCapacity.SMALL) {
            dm.setCapacity(DataModelDesc.RealizationCapacity.SMALL);
        } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.MEDIUM) {
            dm.setCapacity(DataModelDesc.RealizationCapacity.MEDIUM);
        } else if (oldModel.getCapacity() == CubeDesc.CubeCapacity.LARGE) {
            dm.setCapacity(DataModelDesc.RealizationCapacity.LARGE);
        }

        return dm;
    }
 
Example 26
Source Project: Kylin   Source File: CubeSizeEstimationCLI.java    License: Apache License 2.0 5 votes vote down vote up
private static RowKeyColInfo extractRowKeyInfo(CubeDesc cubeDesc) {
    RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
    RowKeyColInfo info = new RowKeyColInfo();
    info.hierachyColBitIndice = new ArrayList<List<Integer>>();
    info.nonHierachyColBitIndice = new ArrayList<Integer>();
    HashSet<Integer> heirachyIndexSet = new HashSet<Integer>();

    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        if (dim.getHierarchy() != null) {
            LinkedList<Integer> hlist = new LinkedList<Integer>();
            for (HierarchyDesc hierarchyDesc : dim.getHierarchy()) {
                int index = rowKeyDesc.getColumnBitIndex(hierarchyDesc.getColumnRef());
                hlist.add(index);
                heirachyIndexSet.add(index);
            }
            info.hierachyColBitIndice.add(hlist);
        }
    }

    for (int i = 0; i < rowKeyDesc.getRowKeyColumns().length; ++i) {
        if (!heirachyIndexSet.contains(i)) {
            info.nonHierachyColBitIndice.add(i);
        }
    }

    return info;
}
 
Example 27
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsQuery(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQuery());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQuery();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQuery();
    measures.remove(QueryPropertyEnum.ID_CODE.toString());
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQuery();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    measureDescList.add(getMeasureMin(QueryPropertyEnum.TIME_COST.toString(),
            measureTypeMap.get(QueryPropertyEnum.TIME_COST.toString())));
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
    }
    measureDescList.add(getMeasureHLL(QueryPropertyEnum.ID_CODE.toString()));
    measureDescList.add(getMeasurePercentile(QueryPropertyEnum.TIME_COST.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.USER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.REALIZATION_TYPE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.EXCEPTION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryPropertyEnum.TYPE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[2][];
    hierarchy_dims[0] = getTimeHierarchy();
    hierarchy_dims[1] = new String[2];
    hierarchy_dims[1][0] = QueryPropertyEnum.REALIZATION_TYPE.toString();
    hierarchy_dims[1][1] = QueryPropertyEnum.REALIZATION.toString();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 28
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsQueryCube(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryCube());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryCube();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());
    dimensions.remove(QueryCubePropertyEnum.PROJECT.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQueryCube();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryCube();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        if (!measure.equals(QueryCubePropertyEnum.WEIGHT_PER_HIT.toString())) {
            measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
        }
    }

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.SEGMENT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_SOURCE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.CUBOID_TARGET.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.FILTER_MASK.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_MATCH.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryCubePropertyEnum.IF_SUCCESS.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[] mandatory_dims = new String[] { QueryCubePropertyEnum.CUBE.toString() };
    mandatory_dims = refineColumnWithTable(tableName, mandatory_dims);

    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    String[][] joint_dims = new String[1][];
    joint_dims[0] = new String[] { QueryCubePropertyEnum.CUBOID_SOURCE.toString(),
            QueryCubePropertyEnum.CUBOID_TARGET.toString() };
    for (int i = 0; i < joint_dims.length; i++) {
        joint_dims[i] = refineColumnWithTable(tableName, joint_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = mandatory_dims;
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = joint_dims;

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 29
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsQueryRPC(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectQueryRpcCall());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsQueryRPC();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsQueryRPC();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize(measures.size() * 2 + 1 + 1);

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsQueryRPC();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
    }
    measureDescList.add(getMeasurePercentile(QueryRPCPropertyEnum.CALL_TIME.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.REALIZATION.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.RPC_SERVER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, RecordEvent.RecordReserveKeyEnum.HOST.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, QueryRPCPropertyEnum.EXCEPTION.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}
 
Example 30
Source Project: kylin-on-parquet-v2   Source File: CubeDescCreator.java    License: Apache License 2.0 4 votes vote down vote up
public static CubeDesc generateKylinCubeDescForMetricsJob(KylinConfig config, MetricsSinkDesc sinkDesc) {
    String tableName = sinkDesc.getTableNameForMetrics(config.getKylinMetricsSubjectJob());

    //Set for dimensions
    List<String> dimensions = ModelCreator.getDimensionsForMetricsJob();
    dimensions.remove(TimePropertyEnum.DAY_TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.TIME.toString());
    dimensions.remove(RecordEvent.RecordReserveKeyEnum.HOST.toString());

    List<DimensionDesc> dimensionDescList = Lists.newArrayListWithExpectedSize(dimensions.size());
    for (String dimensionName : dimensions) {
        dimensionDescList.add(getDimensionDesc(tableName, dimensionName));
    }

    //Set for measures
    List<String> measures = ModelCreator.getMeasuresForMetricsJob();
    List<MeasureDesc> measureDescList = Lists.newArrayListWithExpectedSize((measures.size() - 4) * 3 + 1 + 1 + 4);

    Set<String> stepDuration = Sets.newHashSet();
    stepDuration.add(JobPropertyEnum.STEP_DURATION_DISTINCT_COLUMNS.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_DICTIONARY.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_INMEM_CUBING.toString());
    stepDuration.add(JobPropertyEnum.STEP_DURATION_HFILE_CONVERT.toString());

    List<Pair<String, String>> measureTypeList = HiveTableCreator.getHiveColumnsForMetricsJob();
    Map<String, String> measureTypeMap = Maps.newHashMapWithExpectedSize(measureTypeList.size());
    for (Pair<String, String> entry : measureTypeList) {
        measureTypeMap.put(entry.getFirst(), entry.getSecond());
    }
    measureDescList.add(getMeasureCount());
    for (String measure : measures) {
        measureDescList.add(getMeasureSum(measure, measureTypeMap.get(measure)));
        measureDescList.add(getMeasureMax(measure, measureTypeMap.get(measure)));
        if (!stepDuration.contains(measure)) {
            measureDescList.add(getMeasureMin(measure, measureTypeMap.get(measure)));
        }
    }
    measureDescList.add(getMeasurePercentile(JobPropertyEnum.BUILD_DURATION.toString()));

    //Set for row key
    RowKeyColDesc[] rowKeyColDescs = new RowKeyColDesc[dimensionDescList.size()];
    int idx = getTimeRowKeyColDesc(tableName, rowKeyColDescs);
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.USER.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.PROJECT.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.CUBE.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.ALGORITHM.toString(), idx + 1);
    idx++;
    rowKeyColDescs[idx] = getRowKeyColDesc(tableName, JobPropertyEnum.TYPE.toString(), idx + 1);
    idx++;

    RowKeyDesc rowKeyDesc = new RowKeyDesc();
    rowKeyDesc.setRowkeyColumns(rowKeyColDescs);

    //Set for aggregation group
    String[][] hierarchy_dims = new String[1][];
    hierarchy_dims[0] = getTimeHierarchy();
    for (int i = 0; i < hierarchy_dims.length; i++) {
        hierarchy_dims[i] = refineColumnWithTable(tableName, hierarchy_dims[i]);
    }

    SelectRule selectRule = new SelectRule();
    selectRule.mandatoryDims = new String[0];
    selectRule.hierarchyDims = hierarchy_dims;
    selectRule.jointDims = new String[0][0];

    AggregationGroup aggGroup = new AggregationGroup();
    aggGroup.setIncludes(refineColumnWithTable(tableName, dimensions));
    aggGroup.setSelectRule(selectRule);

    //Set for hbase mapping
    HBaseMappingDesc hBaseMapping = new HBaseMappingDesc();
    hBaseMapping.setColumnFamily(getHBaseColumnFamily(measureDescList));

    return generateKylinCubeDesc(tableName, sinkDesc.getStorageType(), dimensionDescList, measureDescList,
            rowKeyDesc, aggGroup, hBaseMapping, sinkDesc.getCubeDescOverrideProperties());
}