Java Code Examples for org.apache.kylin.metadata.model.DataModelDesc#findColumn()

The following examples show how to use org.apache.kylin.metadata.model.DataModelDesc#findColumn() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FunctionRule.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
/**
 * @param context
 * @param cube
 * @param value
 */
private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) {
    DataModelDesc model = cube.getModel();
    try {
        model.findColumn(value);
    } catch (IllegalArgumentException e) {
        context.addResult(ResultLevel.ERROR, e.getMessage());
    }
}
Example 2
Source File: DictionaryDesc.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
void init(CubeDesc cubeDesc) {
    DataModelDesc model = cubeDesc.getModel();

    column = column.toUpperCase(Locale.ROOT);
    colRef = model.findColumn(column);

    if (reuseColumn != null) {
        reuseColumn = reuseColumn.toUpperCase(Locale.ROOT);
        reuseColRef = model.findColumn(reuseColumn);
    }
}
Example 3
Source File: FunctionRule.java    From kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * @param context
 * @param cube
 * @param value
 */
private void validateColumnParameter(ValidateContext context, CubeDesc cube, String value) {
    DataModelDesc model = cube.getModel();
    try {
        model.findColumn(value);
    } catch (IllegalArgumentException e) {
        context.addResult(ResultLevel.ERROR, e.getMessage());
    }
}
Example 4
Source File: DictionaryDesc.java    From kylin with Apache License 2.0 5 votes vote down vote up 
void init(CubeDesc cubeDesc) {
    DataModelDesc model = cubeDesc.getModel();

    column = column.toUpperCase(Locale.ROOT);
    colRef = model.findColumn(column);

    if (reuseColumn != null) {
        reuseColumn = reuseColumn.toUpperCase(Locale.ROOT);
        reuseColRef = model.findColumn(reuseColumn);
    }
}
Example 5
Source File: DictionaryManagerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Test
public void testBuildSaveDictionary() throws IOException, InterruptedException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    DictionaryManager dictMgr = DictionaryManager.getInstance(config);
    DataModelManager metaMgr = DataModelManager.getInstance(config);
    DataModelDesc model = metaMgr.getDataModelDesc("test_kylin_inner_join_model_desc");
    TblColRef col = model.findColumn("lstg_format_name");

    // non-exist input returns null;
    DictionaryInfo nullInfo = dictMgr.buildDictionary(col, MockupReadableTable.newNonExistTable("/a/path"));
    assertEquals(null, nullInfo);

    DictionaryInfo info1 = dictMgr.buildDictionary(col,
            MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3"));
    assertEquals(3, info1.getDictionaryObject().getSize());

    long info1LastModified = info1.getLastModified();

    // same input returns same dict
    // sleep 1 second to avoid file resource store timestamp precision lost when update
    Thread.sleep(1000);
    DictionaryInfo info2 = dictMgr.buildDictionary(col,
            MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3"));
    assertTrue(info1 != info2);
    assertEquals(info1.getResourcePath(), info2.getResourcePath());

    // update last modified when reused dict
    long info2LastModified = info2.getLastModified();
    assertTrue(info2LastModified > info1LastModified);

    // same input values (different path) returns same dict
    // sleep 1 second to avoid file resource store timestamp precision lost when update
    Thread.sleep(1000);
    DictionaryInfo info3 = dictMgr.buildDictionary(col,
            MockupReadableTable.newSingleColumnTable("/a/different/path", "1", "2", "3"));
    assertTrue(info1 != info3);
    assertTrue(info2 != info3);
    assertEquals(info1.getResourcePath(), info3.getResourcePath());
    assertEquals(info2.getResourcePath(), info3.getResourcePath());

    // update last modified when reused dict
    long info3LastModified = info3.getLastModified();
    assertTrue(info3LastModified > info2LastModified);

    // save dictionary works in spite of non-exist table
    Dictionary<String> dict = DictionaryGenerator.buildDictionary(col.getType(),
            new IterableDictionaryValueEnumerator("1", "2", "3"));
    DictionaryInfo info4 = dictMgr.saveDictionary(col, MockupReadableTable.newNonExistTable("/a/path"), dict);
    assertEquals(info1.getResourcePath(), info4.getResourcePath());

    Dictionary<String> dict2 = DictionaryGenerator.buildDictionary(col.getType(),
            new IterableDictionaryValueEnumerator("1", "2", "3", "4"));
    DictionaryInfo info5 = dictMgr.saveDictionary(col, MockupReadableTable.newNonExistTable("/a/path"), dict2);
    assertNotEquals(info1.getResourcePath(), info5.getResourcePath());
}
Example 6
Source File: CubeDescTiretreeGlobalDomainDictUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
/**
 * get reuse global tiretree global dic path
 * @param tblColRef
 * @param cubeDesc
 * @return
 */
public static String globalReuseDictPath(KylinConfig config, TblColRef tblColRef, CubeDesc cubeDesc) {
    String globalResumeDictPath = null;
    List<GlobalDict> globalDicts = cubeDesc.listDomainDict();
    DataModelManager metadataManager = DataModelManager.getInstance(config);
    CubeManager cubeManager = CubeManager.getInstance(config);
    for (GlobalDict dict : globalDicts) {
        if (dict.getSrc().getIdentity().equalsIgnoreCase(tblColRef.getIdentity())) {
            String model = dict.getModel();
            String cube = dict.getCube();
            logger.info("cube:{} column:{} tiretree global domain dic reuse model:{} cube{} column:{} ",
                    cubeDesc.getName(), tblColRef.getName(), model, cube, dict.getDesc());

            DataModelDesc dataModel = metadataManager.getDataModelDesc(model);
            if (Objects.isNull(dataModel)) {
                logger.error("get cube:{} column:{} tiretree global domain dic reuse DataModelDesc error",
                        cubeDesc.getName(), tblColRef.getName());
                return null;
            }

            CubeInstance cubeInstance = cubeManager.getCube(cube);
            CubeSegment cubeSegment = cubeInstance.getLatestReadySegment();

            TblColRef colRef = dataModel.findColumn(dict.getDesc());
            if (Objects.isNull(colRef)) {
                logger.error("get cube:{} column:{} tiretree global domain dic TblColRef error");
                return null;
            }

            globalResumeDictPath = cubeSegment.getDictResPath(colRef);

            if (StringUtils.isBlank(globalResumeDictPath)) {
                logger.error("get cube:{} column:{} tiretree global domain dic resume dict path error");
            }
            logger.error("get cube:{} column:{} tiretree global domain dic resume dict path is {}",
                    globalResumeDictPath);
            break;
        }
    }
    return globalResumeDictPath;
}
Example 7
Source File: CubeDescTiretreeGlobalDomainDictUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
/**
 * add resuce global tiretree global dic for baseid job
 * @param cubeDesc
 * @param dumpList
 */
public static void cuboidJob(CubeDesc cubeDesc, Set<String> dumpList) {
    logger.info("cube {} start to add global domain dic", cubeDesc.getName());
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    DataModelManager metadataManager = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());

    cubeManager.getCube(cubeDesc.getName());
    List<GlobalDict> globalDicts = cubeDesc.listDomainDict();

    for (GlobalDict dict : globalDicts) {
        String cube = dict.getCube();
        String model = dict.getModel();
        logger.debug("cube {} column {} start to add global domain dic ,reuse {}.{}.{}", cubeDesc.getName(),
                dict.getSrc(), model, cube, dict.getDesc());
        CubeInstance instance = cubeManager.getCube(cube);
        logger.debug("cube {} column {} start to add global domain dic ,reuse cube{} dict", cubeDesc.getName(),
                dict.getSrc(), instance.getName());

        // cube, model_desc, cube_desc, table
        dumpList.add(instance.getResourcePath());
        dumpList.add(instance.getDescriptor().getModel().getResourcePath());
        dumpList.add(instance.getDescriptor().getResourcePath());
        dumpList.add(instance.getProjectInstance().getResourcePath());

        for (TableRef tableRef : instance.getDescriptor().getModel().getAllTables()) {
            TableDesc table = tableRef.getTableDesc();
            dumpList.add(table.getResourcePath());
            dumpList.addAll(SourceManager.getMRDependentResources(table));
        }

        DataModelDesc dataModelDesc = metadataManager.getDataModelDesc(model);
        logger.debug("cube {} column {} start to add global domain dic ,reuse model{} dict", cubeDesc.getName(),
                dict.getSrc(), dataModelDesc.getName());
        TblColRef tblColRef = dataModelDesc.findColumn(dict.getDesc());
        CubeSegment segment = instance.getLatestReadySegment();
        logger.debug(
                "cube {} column {} start to add global domain dic ,reuse mode:{} cube:{} segment:{} dict,tblColRef:{}",
                cubeDesc.getName(), dict.getSrc(), dataModelDesc.getName(), cube, segment.getName(),
                tblColRef.getIdentity());
        if (segment.getDictResPath(tblColRef) != null) {
            dumpList.addAll(ImmutableList.of(segment.getDictResPath(tblColRef)));
        }
    }
}
Example 8
Source File: DictionaryManagerTest.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Test
public void testBuildSaveDictionary() throws IOException, InterruptedException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    DictionaryManager dictMgr = DictionaryManager.getInstance(config);
    DataModelManager metaMgr = DataModelManager.getInstance(config);
    DataModelDesc model = metaMgr.getDataModelDesc("test_kylin_inner_join_model_desc");
    TblColRef col = model.findColumn("lstg_format_name");

    // non-exist input returns null;
    DictionaryInfo nullInfo = dictMgr.buildDictionary(col, MockupReadableTable.newNonExistTable("/a/path"));
    assertEquals(null, nullInfo);

    DictionaryInfo info1 = dictMgr.buildDictionary(col,
            MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3"));
    assertEquals(3, info1.getDictionaryObject().getSize());

    long info1LastModified = info1.getLastModified();

    // same input returns same dict
    // sleep 1 second to avoid file resource store timestamp precision lost when update
    Thread.sleep(1000);
    DictionaryInfo info2 = dictMgr.buildDictionary(col,
            MockupReadableTable.newSingleColumnTable("/a/path", "1", "2", "3"));
    assertTrue(info1 != info2);
    assertEquals(info1.getResourcePath(), info2.getResourcePath());

    // update last modified when reused dict
    long info2LastModified = info2.getLastModified();
    assertTrue(info2LastModified > info1LastModified);

    // same input values (different path) returns same dict
    // sleep 1 second to avoid file resource store timestamp precision lost when update
    Thread.sleep(1000);
    DictionaryInfo info3 = dictMgr.buildDictionary(col,
            MockupReadableTable.newSingleColumnTable("/a/different/path", "1", "2", "3"));
    assertTrue(info1 != info3);
    assertTrue(info2 != info3);
    assertEquals(info1.getResourcePath(), info3.getResourcePath());
    assertEquals(info2.getResourcePath(), info3.getResourcePath());

    // update last modified when reused dict
    long info3LastModified = info3.getLastModified();
    assertTrue(info3LastModified > info2LastModified);

    // save dictionary works in spite of non-exist table
    Dictionary<String> dict = DictionaryGenerator.buildDictionary(col.getType(),
            new IterableDictionaryValueEnumerator("1", "2", "3"));
    DictionaryInfo info4 = dictMgr.saveDictionary(col, MockupReadableTable.newNonExistTable("/a/path"), dict);
    assertEquals(info1.getResourcePath(), info4.getResourcePath());

    Dictionary<String> dict2 = DictionaryGenerator.buildDictionary(col.getType(),
            new IterableDictionaryValueEnumerator("1", "2", "3", "4"));
    DictionaryInfo info5 = dictMgr.saveDictionary(col, MockupReadableTable.newNonExistTable("/a/path"), dict2);
    assertNotEquals(info1.getResourcePath(), info5.getResourcePath());
}
Example 9
Source File: CubeDescTiretreeGlobalDomainDictUtil.java    From kylin with Apache License 2.0 4 votes vote down vote up 
/**
 * get reuse global tiretree global dic path
 * @param tblColRef
 * @param cubeDesc
 * @return
 */
public static String globalReuseDictPath(KylinConfig config, TblColRef tblColRef, CubeDesc cubeDesc) {
    String globalResumeDictPath = null;
    List<GlobalDict> globalDicts = cubeDesc.listDomainDict();
    DataModelManager metadataManager = DataModelManager.getInstance(config);
    CubeManager cubeManager = CubeManager.getInstance(config);
    for (GlobalDict dict : globalDicts) {
        if (dict.getSrc().getIdentity().equalsIgnoreCase(tblColRef.getIdentity())) {
            String model = dict.getModel();
            String cube = dict.getCube();
            logger.info("cube:{} column:{} tiretree global domain dic reuse model:{} cube{} column:{} ",
                    cubeDesc.getName(), tblColRef.getName(), model, cube, dict.getDesc());

            DataModelDesc dataModel = metadataManager.getDataModelDesc(model);
            if (Objects.isNull(dataModel)) {
                logger.error("get cube:{} column:{} tiretree global domain dic reuse DataModelDesc error",
                        cubeDesc.getName(), tblColRef.getName());
                return null;
            }

            CubeInstance cubeInstance = cubeManager.getCube(cube);
            CubeSegment cubeSegment = cubeInstance.getLatestReadySegment();

            TblColRef colRef = dataModel.findColumn(dict.getDesc());
            if (Objects.isNull(colRef)) {
                logger.error("get cube:{} column:{} tiretree global domain dic TblColRef error");
                return null;
            }

            globalResumeDictPath = cubeSegment.getDictResPath(colRef);

            if (StringUtils.isBlank(globalResumeDictPath)) {
                logger.error("get cube:{} column:{} tiretree global domain dic resume dict path error");
            }
            logger.error("get cube:{} column:{} tiretree global domain dic resume dict path is {}",
                    globalResumeDictPath);
            break;
        }
    }
    return globalResumeDictPath;
}
Example 10
Source File: CubeDescTiretreeGlobalDomainDictUtil.java    From kylin with Apache License 2.0 4 votes vote down vote up 
/**
 * add resuce global tiretree global dic for baseid job
 * @param cubeDesc
 * @param dumpList
 */
public static void cuboidJob(CubeDesc cubeDesc, Set<String> dumpList) {
    logger.info("cube {} start to add global domain dic", cubeDesc.getName());
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    DataModelManager metadataManager = DataModelManager.getInstance(KylinConfig.getInstanceFromEnv());

    cubeManager.getCube(cubeDesc.getName());
    List<GlobalDict> globalDicts = cubeDesc.listDomainDict();

    for (GlobalDict dict : globalDicts) {
        String cube = dict.getCube();
        String model = dict.getModel();
        logger.debug("cube {} column {} start to add global domain dic ,reuse {}.{}.{}", cubeDesc.getName(),
                dict.getSrc(), model, cube, dict.getDesc());
        CubeInstance instance = cubeManager.getCube(cube);
        logger.debug("cube {} column {} start to add global domain dic ,reuse cube{} dict", cubeDesc.getName(),
                dict.getSrc(), instance.getName());

        // cube, model_desc, cube_desc, table
        dumpList.add(instance.getResourcePath());
        dumpList.add(instance.getDescriptor().getModel().getResourcePath());
        dumpList.add(instance.getDescriptor().getResourcePath());
        dumpList.add(instance.getProjectInstance().getResourcePath());

        for (TableRef tableRef : instance.getDescriptor().getModel().getAllTables()) {
            TableDesc table = tableRef.getTableDesc();
            dumpList.add(table.getResourcePath());
            dumpList.addAll(SourceManager.getMRDependentResources(table));
        }

        DataModelDesc dataModelDesc = metadataManager.getDataModelDesc(model);
        logger.debug("cube {} column {} start to add global domain dic ,reuse model{} dict", cubeDesc.getName(),
                dict.getSrc(), dataModelDesc.getName());
        TblColRef tblColRef = dataModelDesc.findColumn(dict.getDesc());
        CubeSegment segment = instance.getLatestReadySegment();
        logger.debug(
                "cube {} column {} start to add global domain dic ,reuse mode:{} cube:{} segment:{} dict,tblColRef:{}",
                cubeDesc.getName(), dict.getSrc(), dataModelDesc.getName(), cube, segment.getName(),
                tblColRef.getIdentity());
        if (segment.getDictResPath(tblColRef) != null) {
            dumpList.addAll(ImmutableList.of(segment.getDictResPath(tblColRef)));
        }
    }
}