Java Code Examples for org.apache.kylin.cube.CubeManager#listAllCubes()

The following examples show how to use org.apache.kylin.cube.CubeManager#listAllCubes() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SampleCubeSetupTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCubes() throws Exception {
    DeployUtil.initCliWorkDir();
    DeployUtil.deployMetadata();
    DeployUtil.deployJobJars();
    deployJobConfToEtc();
    DeployUtil.prepareTestData("inner", "test_kylin_cube_with_slr_empty");

    // remove all other cubes to keep it clean
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    for (CubeInstance cubeInstance : cubeManager.listAllCubes()) {
        if (!cubeInstance.getName().equalsIgnoreCase("test_kylin_cube_without_slr_empty") && !cubeInstance.getName().equalsIgnoreCase("test_kylin_cube_with_slr_empty"))
            cubeManager.dropCube(cubeInstance.getName(), false);
    }

}
Example 2
Source File: SCCreatorTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Test
public void testExecute() throws Exception {
    String metadataPath = tempMetadataDir.getPath();

    String inputPath = "src/main/resources/SCSinkTools.json";

    SCCreator cli = new SCCreator();
    cli.execute("ADMIN", metadataPath, inputPath);
    Assert.assertTrue(tempMetadataDir.isDirectory());

    KylinConfig local = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
    local.setMetadataUrl(metadataPath);

    CubeManager cubeManager = CubeManager.getInstance(local);
    List<CubeInstance> cubeList = cubeManager.listAllCubes();
    System.out.println("System cubes: " + cubeList);
    assertEquals(cubeList.size(), 10);

    for (CubeInstance cube : cubeList) {
        Assert.assertTrue(cube.getStatus() != RealizationStatusEnum.DESCBROKEN);
    }
}
Example 3
Source File: UpdateHTableHostCLI.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static List<String> getHTableNames(KylinConfig config) {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    ArrayList<String> result = new ArrayList<>();
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
            String tableName = seg.getStorageLocationIdentifier();
            if (!StringUtils.isBlank(tableName)) {
                result.add(tableName);
                logger.info("added new table: {}", tableName);
            }
        }
    }

    return result;
}
Example 4
Source File: DeployCoprocessorCLI.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static List<String> getHTableNames(KylinConfig config) {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    ArrayList<String> result = new ArrayList<String>();
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        if (cube.getStorageType() == IStorageAware.ID_HBASE
                || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE
                || cube.getStorageType() == IStorageAware.ID_REALTIME_AND_HBASE) {
            for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
                String tableName = seg.getStorageLocationIdentifier();
                if (StringUtils.isBlank(tableName) == false) {
                    result.add(tableName);
                    System.out.println("added new table: " + tableName);
                }
            }
        }
    }

    return result;
}
Example 5
Source File: DeployCoprocessorCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static List<String> getHTableNames(KylinConfig config) {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    ArrayList<String> result = new ArrayList<String>();
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        if (cube.getStorageType() == IStorageAware.ID_HBASE
                || cube.getStorageType() == IStorageAware.ID_SHARDED_HBASE
                || cube.getStorageType() == IStorageAware.ID_REALTIME_AND_HBASE) {
            for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
                String tableName = seg.getStorageLocationIdentifier();
                if (StringUtils.isBlank(tableName) == false) {
                    result.add(tableName);
                    System.out.println("added new table: " + tableName);
                }
            }
        }
    }

    return result;
}
Example 6
Source File: UpdateHTableHostCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static List<String> getHTableNames(KylinConfig config) {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    ArrayList<String> result = new ArrayList<>();
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        for (CubeSegment seg : cube.getSegments(SegmentStatusEnum.READY)) {
            String tableName = seg.getStorageLocationIdentifier();
            if (!StringUtils.isBlank(tableName)) {
                result.add(tableName);
                logger.info("added new table: {}", tableName);
            }
        }
    }

    return result;
}
Example 7
Source File: SCCreatorTest.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Test
public void testExecute() throws Exception {
    String metadataPath = tempMetadataDir.getPath();

    String inputPath = "src/main/resources/SCSinkTools.json";

    SCCreator cli = new SCCreator();
    cli.execute("ADMIN", metadataPath, inputPath);
    Assert.assertTrue(tempMetadataDir.isDirectory());

    KylinConfig local = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
    local.setMetadataUrl(metadataPath);

    CubeManager cubeManager = CubeManager.getInstance(local);
    List<CubeInstance> cubeList = cubeManager.listAllCubes();
    System.out.println("System cubes: " + cubeList);
    assertEquals(cubeList.size(), 10);

    for (CubeInstance cube : cubeList) {
        Assert.assertTrue(cube.getStatus() != RealizationStatusEnum.DESCBROKEN);
    }
}
Example 8
Source File: CubeMigrationCheckCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public void checkAll() {
    List<String> segFullNameList = Lists.newArrayList();

    CubeManager cubeMgr = CubeManager.getInstance(dstCfg);
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        addHTableNamesForCube(cube, segFullNameList);
    }

    check(segFullNameList);
}
Example 9
Source File: KylinHealthCheckJob.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void checkCubeHealth() throws Exception {
    CubeManager cubeManager = CubeManager.getInstance(config);

    List<CubeInstance> cubes = cubeManager.listAllCubes();
    checkErrorMeta();

    // Check if the cubeid data exist for later cube merge
    checkSegmentHDFSPath(cubes);

    // Check if the hbase table exits or online
    checkHBaseTables(cubes);

    // Check if there are holes in cube
    // TODO: check if there are overlaps in segments of cube
    checkCubeHoles(cubes);

    // Check if there are too many segments
    checkTooManySegments(cubes);

    // Check if there are stale metadata
    checkStaleSegments(cubes);

    // Disable/Delete the out-of-date cube
    checkOutOfDateCube(cubes);

    // Check data expand rate
    checkDataExpansionRate(cubes);

    // Check auto merge param
    checkCubeDescParams(cubes);

    // ERROR history stopped build job
    checkStoppedJob();

    sendMail(reporter.getBufferedLog());
}
Example 10
Source File: CubeMigrationCheckCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public void checkAll() {
    List<String> segFullNameList = Lists.newArrayList();

    CubeManager cubeMgr = CubeManager.getInstance(dstCfg);
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        addHTableNamesForCube(cube, segFullNameList);
    }

    check(segFullNameList);
}
Example 11
Source File: CubeMetaIngester.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void injest(File metaRoot) throws IOException {
    KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath());
    TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig);
    DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig);
    HybridManager srcHybridManager = HybridManager.getInstance(srcConfig);
    CubeManager srcCubeManager = CubeManager.getInstance(srcConfig);
    CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig);

    checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager);
    new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources));

    // clear the cache
    Broadcaster.getInstance(kylinConfig).notifyClearAll();
    
    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        logger.info("add " + tableDesc + " to " + targetProjectName);
        projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName);
    }

    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        logger.info("add " + cube + " to " + targetProjectName);
        projectManager.addModelToProject(cube.getModel().getName(), targetProjectName);
        projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null);
    }

}
Example 12
Source File: CubeMigrationCheckCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public void checkAll() {
    List<String> segFullNameList = Lists.newArrayList();

    CubeManager cubeMgr = CubeManager.getInstance(dstCfg);
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        addHTableNamesForCube(cube, segFullNameList);
    }

    check(segFullNameList);
}
Example 13
Source File: KylinHealthCheckJob.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void checkCubeHealth() throws Exception {
    CubeManager cubeManager = CubeManager.getInstance(config);

    List<CubeInstance> cubes = cubeManager.listAllCubes();
    checkErrorMeta();

    // Check if the cubeid data exist for later cube merge
    checkSegmentHDFSPath(cubes);

    // Check if the hbase table exits or online
    checkHBaseTables(cubes);

    // Check if there are holes in cube
    // TODO: check if there are overlaps in segments of cube
    checkCubeHoles(cubes);

    // Check if there are too many segments
    checkTooManySegments(cubes);

    // Check if there are stale metadata
    checkStaleSegments(cubes);

    // Disable/Delete the out-of-date cube
    checkOutOfDateCube(cubes);

    // Check data expand rate
    checkDataExpansionRate(cubes);

    // Check auto merge param
    checkCubeDescParams(cubes);

    // ERROR history stopped build job
    checkStoppedJob();

    sendMail(reporter.getBufferedLog());
}
Example 14
Source File: CubeMigrationCheckCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public void checkAll() {
    List<String> segFullNameList = Lists.newArrayList();

    CubeManager cubeMgr = CubeManager.getInstance(dstCfg);
    for (CubeInstance cube : cubeMgr.listAllCubes()) {
        addHTableNamesForCube(cube, segFullNameList);
    }

    check(segFullNameList);
}
Example 15
Source File: CubeMetaIngester.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void injest(File metaRoot) throws IOException {
    KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath());
    TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig);
    DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig);
    HybridManager srcHybridManager = HybridManager.getInstance(srcConfig);
    CubeManager srcCubeManager = CubeManager.getInstance(srcConfig);
    CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig);

    checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager);
    new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources));

    // clear the cache
    Broadcaster.getInstance(kylinConfig).notifyClearAll();
    
    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        logger.info("add " + tableDesc + " to " + targetProjectName);
        projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName);
    }

    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        logger.info("add " + cube + " to " + targetProjectName);
        projectManager.addModelToProject(cube.getModel().getName(), targetProjectName);
        projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null);
    }

}
Example 16
Source File: CubeMetaIngester.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) {
    if (srcHybridManager.listHybridInstances().size() > 0) {
        throw new IllegalStateException("Does not support ingest hybrid yet");
    }

    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    ProjectInstance targetProject = projectManager.getProject(targetProjectName);
    if (targetProject == null) {
        throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
    }

    TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        TableDesc existing = metadataManager.getTableDesc(tableDesc.getIdentity(), targetProjectName);
        if (existing != null && !existing.equals(tableDesc)) {
            logger.info("Table {} already has a different version in target metadata store", tableDesc.getIdentity());
            logger.info("Existing version: {}", existing);
            logger.info("New version: {}", tableDesc);

            if (!forceIngest && !overwriteTables) {
                throw new IllegalStateException("table already exists with a different version: " + tableDesc.getIdentity() + ". Consider adding -overwriteTables option to force overwriting (with caution)");
            } else {
                logger.warn("Overwriting the old table desc: {}", tableDesc.getIdentity());
            }
        }
        requiredResources.add(tableDesc.getResourcePath());
    }

    DataModelManager modelManager = DataModelManager.getInstance(kylinConfig);
    for (DataModelDesc dataModelDesc : srcModelManager.listDataModels()) {
        checkExesting(modelManager.getDataModelDesc(dataModelDesc.getName()), "model", dataModelDesc.getName());
        requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName()));
    }

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) {
        checkExesting(cubeDescManager.getCubeDesc(cubeDesc.getName()), "cube desc", cubeDesc.getName());
        requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName()));
    }

    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        checkExesting(cubeManager.getCube(cube.getName()), "cube", cube.getName());
        requiredResources.add(CubeInstance.concatResourcePath(cube.getName()));
    }

}
Example 17
Source File: MetadataCleanupJob.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public void cleanup() throws Exception {
    CubeManager cubeManager = CubeManager.getInstance(config);

    List<String> toDeleteResource = Lists.newArrayList();

    // two level resources, snapshot tables and cube statistics
    for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) {
        NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot);

        if (snapshotTables != null) {
            for (String snapshotTable : snapshotTables) {
                NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
                if (snapshotNames != null)
                    for (String snapshot : snapshotNames) {
                        if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
                            toDeleteResource.add(snapshot);
                    }
            }
        }
    }

    // three level resources, only dictionaries
    NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);

    if (dictTables != null) {
        for (String table : dictTables) {
            NavigableSet<String> tableColNames = getStore().listResources(table);
            if (tableColNames != null)
                for (String tableCol : tableColNames) {
                    NavigableSet<String> dictionaries = getStore().listResources(tableCol);
                    if (dictionaries != null)
                        for (String dict : dictionaries)
                            if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
                                toDeleteResource.add(dict);
                }
        }
    }

    Set<String> activeResourceList = Sets.newHashSet();
    for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
        for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
            activeResourceList.addAll(segment.getSnapshotPaths());
            activeResourceList.addAll(segment.getDictionaryPaths());
            activeResourceList.add(segment.getStatisticsResourcePath());
        }
    }

    toDeleteResource.removeAll(activeResourceList);

    // delete old and completed jobs
    ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid());
        if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) {
            toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid());
            toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());

            for (ExecutablePO task : executable.getTasks()) {
                toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
            }
        }
    }

    if (toDeleteResource.size() > 0) {
        logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n");

        for (String s : toDeleteResource) {
            logger.info(s);
            if (delete == true) {
                getStore().deleteResource(s);
            }
        }
    } else {
        logger.info("No resource to be cleaned up from metadata store;");
    }

}
Example 18
Source File: MetadataCleanupJob.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public void cleanup() throws Exception {
    CubeManager cubeManager = CubeManager.getInstance(config);

    List<String> toDeleteResource = Lists.newArrayList();

    // two level resources, snapshot tables and cube statistics
    for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) {
        NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot);

        if (snapshotTables != null) {
            for (String snapshotTable : snapshotTables) {
                NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
                if (snapshotNames != null)
                    for (String snapshot : snapshotNames) {
                        if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
                            toDeleteResource.add(snapshot);
                    }
            }
        }
    }

    // three level resources, only dictionaries
    NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);

    if (dictTables != null) {
        for (String table : dictTables) {
            NavigableSet<String> tableColNames = getStore().listResources(table);
            if (tableColNames != null)
                for (String tableCol : tableColNames) {
                    NavigableSet<String> dictionaries = getStore().listResources(tableCol);
                    if (dictionaries != null)
                        for (String dict : dictionaries)
                            if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
                                toDeleteResource.add(dict);
                }
        }
    }

    Set<String> activeResourceList = Sets.newHashSet();
    for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
        for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
            activeResourceList.addAll(segment.getSnapshotPaths());
            activeResourceList.addAll(segment.getDictionaryPaths());
            activeResourceList.add(segment.getStatisticsResourcePath());
        }
    }

    toDeleteResource.removeAll(activeResourceList);

    // delete old and completed jobs
    ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid());
        if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) {
            toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid());
            toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());

            for (ExecutablePO task : executable.getTasks()) {
                toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
            }
        }
    }

    if (toDeleteResource.size() > 0) {
        logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n");

        for (String s : toDeleteResource) {
            logger.info(s);
            if (delete == true) {
                getStore().deleteResource(s);
            }
        }
    } else {
        logger.info("No resource to be cleaned up from metadata store;");
    }

}
Example 19
Source File: CubeMetaIngester.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) {
    if (srcHybridManager.listHybridInstances().size() > 0) {
        throw new IllegalStateException("Does not support ingest hybrid yet");
    }

    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    ProjectInstance targetProject = projectManager.getProject(targetProjectName);
    if (targetProject == null) {
        throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
    }

    TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        TableDesc existing = metadataManager.getTableDesc(tableDesc.getIdentity(), targetProjectName);
        if (existing != null && !existing.equals(tableDesc)) {
            logger.info("Table {} already has a different version in target metadata store", tableDesc.getIdentity());
            logger.info("Existing version: {}", existing);
            logger.info("New version: {}", tableDesc);

            if (!forceIngest && !overwriteTables) {
                throw new IllegalStateException("table already exists with a different version: " + tableDesc.getIdentity() + ". Consider adding -overwriteTables option to force overwriting (with caution)");
            } else {
                logger.warn("Overwriting the old table desc: {}", tableDesc.getIdentity());
            }
        }
        requiredResources.add(tableDesc.getResourcePath());
    }

    DataModelManager modelManager = DataModelManager.getInstance(kylinConfig);
    for (DataModelDesc dataModelDesc : srcModelManager.listDataModels()) {
        checkExesting(modelManager.getDataModelDesc(dataModelDesc.getName()), "model", dataModelDesc.getName());
        requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName()));
    }

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) {
        checkExesting(cubeDescManager.getCubeDesc(cubeDesc.getName()), "cube desc", cubeDesc.getName());
        requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName()));
    }

    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        checkExesting(cubeManager.getCube(cube.getName()), "cube", cube.getName());
        requiredResources.add(CubeInstance.concatResourcePath(cube.getName()));
    }

}