Java Code Examples for org.apache.kylin.cube.CubeDescManager#getInstance()

The following examples show how to use org.apache.kylin.cube.CubeDescManager#getInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: CubeSignatureRefresher.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public void update() {
    logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
    CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
    List<CubeDesc> cubeDescs;
    if (ArrayUtils.isEmpty(cubeNames)) {
        cubeDescs = cubeDescManager.listAllDesc();
    } else {
        String[] names = StringUtil.splitByComma(cubeNames[0]);
        if (ArrayUtils.isEmpty(names))
            return;
        cubeDescs = Lists.newArrayListWithCapacity(names.length);
        for (String name : names) {
            cubeDescs.add(cubeDescManager.getCubeDesc(name));
        }
    }
    for (CubeDesc cubeDesc : cubeDescs) {
        updateCubeDesc(cubeDesc);
    }

    verify();
}

Example 2

Source File: BuildCubeWithEngine.java From kylin with Apache License 2.0

6 votes

public void before() throws Exception {
    deployEnv();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        AbstractExecutable executable = jobService.getJob(jobId);
        if (executable instanceof CubingJob || executable instanceof CheckpointExecutable) {
            jobService.deleteJob(jobId);
        }
    }

    cubeDescManager = CubeDescManager.getInstance(kylinConfig);

    // update enginType
    updateCubeEngineType(Lists.newArrayList("ci_inner_join_cube", "ci_left_join_cube"));
}

Example 3

Source File: CubeMetaIngester.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private void injest(File metaRoot) throws IOException {
    KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath());
    TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig);
    DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig);
    HybridManager srcHybridManager = HybridManager.getInstance(srcConfig);
    CubeManager srcCubeManager = CubeManager.getInstance(srcConfig);
    CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig);

    checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager);
    new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources));

    // clear the cache
    Broadcaster.getInstance(kylinConfig).notifyClearAll();
    
    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        logger.info("add " + tableDesc + " to " + targetProjectName);
        projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName);
    }

    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        logger.info("add " + cube + " to " + targetProjectName);
        projectManager.addModelToProject(cube.getModel().getName(), targetProjectName);
        projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null);
    }

}

Example 4

Source File: ProjectManagerTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testExistingProject() throws Exception {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());

    int originalProjectCount = prjMgr.listAllProjects().size();
    int originalCubeCount = cubeMgr.listAllCubes().size();
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/new_cube_in_default.json");

    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null);
    assertTrue(createdCube.equals(cubeMgr.getCube("new_cube_in_default")));

    //System.out.println(JsonUtil.writeValueAsIndentString(createdCube));

    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);

    CubeInstance droppedCube = cubeMgr.dropCube("new_cube_in_default", true);

    assertTrue(createdCube.equals(droppedCube));
    assertNull(cubeMgr.getCube("new_cube_in_default"));
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
}

Example 5

Source File: ExtendCubeToHybridCLI.java From kylin with Apache License 2.0

5 votes

public ExtendCubeToHybridCLI() {
    this.kylinConfig = KylinConfig.getInstanceFromEnv();
    this.store = ResourceStore.getStore(kylinConfig);
    this.cubeManager = CubeManager.getInstance(kylinConfig);
    this.cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    this.metadataManager = DataModelManager.getInstance(kylinConfig);
}

Example 6

Source File: CacheControllerTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testClearCacheForCubeMigration() throws IOException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String CUBENAME = "test_kylin_cube_without_slr_desc";

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc(CUBENAME);
    DataModelDesc modelDesc = cubeDesc.getModel();
    Map<String, String> tableToProjects = new HashMap<>();
    for (TableRef tableRef : modelDesc.getAllTables()) {
        tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject());
    }

    String uuid = cubeDesc.getUuid();
    String signature = cubeDesc.getSignature();

    assertEquals(cubeDesc.getRetentionRange(), 0);

    //update cubeDesc
    cubeDesc.setRetentionRange(2018);
    cubeDesc.updateRandomUuid();

    //directly update metadata in store to simulate cube migration
    Serializer<CubeDesc> cubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
    getStore().checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, cubeDescSerializer);

    CubeMigrationRequest request = new CubeMigrationRequest();
    request.setCube(cubeDesc.getName());
    request.setModel(modelDesc.getName());
    request.setProject(modelDesc.getProject());
    request.setTableToProjects(tableToProjects);

    cacheController.clearCacheForCubeMigration(request);

    assertEquals(2018, cubeDescManager.getCubeDesc(CUBENAME).getRetentionRange());
    assertEquals(signature, cubeDescManager.getCubeDesc(CUBENAME).getSignature());
    assertNotEquals(uuid, cubeDescManager.getCubeDesc(CUBENAME).getUuid());
}

Example 7

Source File: ExtendCubeToHybridCLI.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public ExtendCubeToHybridCLI() {
    this.kylinConfig = KylinConfig.getInstanceFromEnv();
    this.store = ResourceStore.getStore(kylinConfig);
    this.cubeManager = CubeManager.getInstance(kylinConfig);
    this.cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    this.metadataManager = DataModelManager.getInstance(kylinConfig);
}

Example 8

Source File: CubeMetadataUpgrade.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public void verify() {
    logger.info("=================================================================");
    logger.info("The changes are applied, now it's time to verify the new metadata store by reloading all metadata:");
    logger.info("=================================================================");
    config.clearManagers();
    DataModelManager.getInstance(config);
    CubeDescManager.getInstance(config);
    CubeManager.getInstance(config);
    ProjectManager.getInstance(config);
    //cleanup();
}

Example 9

Source File: CacheServiceTest.java From Kylin with Apache License 2.0

4 votes

private static CubeDescManager getCubeDescManager(KylinConfig config) throws Exception {
    return CubeDescManager.getInstance(config);
}

Example 10

Source File: SegmentMemoryStoreTest.java From kylin with Apache License 2.0

4 votes

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getTestConfig());
}

Example 11

Source File: SegmentMemoryStoreTest.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getTestConfig());
}

Example 12

Source File: BasicService.java From kylin with Apache License 2.0

4 votes

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getConfig());
}

Example 13

Source File: BasicService.java From Kylin with Apache License 2.0

4 votes

public final CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getConfig());
}

Example 14

Source File: ProjectManagerTest.java From Kylin with Apache License 2.0

4 votes

@Test
public void testNewProject() throws Exception {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());
    
    int originalProjectCount = prjMgr.listAllProjects().size();
    int originalCubeCount = cubeMgr.listAllCubes().size();
    int originalCubeCountInDefault = prjMgr.listAllRealizations("default").size();
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/cube_in_alien_project.json");

    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("cube_in_alien_project", "alien", desc, null);
    assertTrue(createdCube == cubeMgr.getCube("cube_in_alien_project"));
    ProjectManager proMgr = ProjectManager.getInstance(getTestConfig());
    Set<IRealization> realizations = proMgr.listAllRealizations("alien");
    assertTrue(realizations.contains(createdCube));

    System.out.println(JsonUtil.writeValueAsIndentString(createdCube));

    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
    assertTrue(prjMgr.listAllRealizations("ALIEN").iterator().next().getName().equalsIgnoreCase("CUBE_IN_ALIEN_PROJECT"));
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);

    prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "default", null);
    assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 0);
    assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault + 1);
    assertTrue(ProjectManager.getInstance(getTestConfig()).listAllRealizations("default").contains(createdCube));

    prjMgr.moveRealizationToProject(RealizationType.CUBE, "cube_in_alien_project", "alien", null);
    assertTrue(prjMgr.listAllRealizations("ALIEN").size() == 1);
    assertTrue(prjMgr.listAllRealizations("default").size() == originalCubeCountInDefault);
    assertTrue(ProjectManager.getInstance(getTestConfig()).listAllRealizations("alien").contains(createdCube));

    CubeInstance droppedCube = cubeMgr.dropCube("cube_in_alien_project", true);

    assertTrue(createdCube == droppedCube);
    assertNull(cubeMgr.getCube("cube_in_alien_project"));
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount + 1);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);

    prjMgr.dropProject("alien");
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
}

Example 15

Source File: CuboidSchedulerTest.java From kylin with Apache License 2.0

4 votes

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getTestConfig());
}

Example 16

Source File: BasicService.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getConfig());
}

Example 17

Source File: CuboidTest.java From kylin with Apache License 2.0

4 votes

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getTestConfig());
}

Example 18

Source File: CuboidTest.java From Kylin with Apache License 2.0

4 votes

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getTestConfig());
}

Example 19

Source File: BasicService.java From kylin with Apache License 2.0

4 votes

public CubeDescManager getCubeDescManager() {
    return CubeDescManager.getInstance(getConfig());
}

Example 20

Source File: CubeMetaIngester.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

private void checkAndMark(TableMetadataManager srcMetadataManager, DataModelManager srcModelManager, HybridManager srcHybridManager, CubeManager srcCubeManager, CubeDescManager srcCubeDescManager) {
    if (srcHybridManager.listHybridInstances().size() > 0) {
        throw new IllegalStateException("Does not support ingest hybrid yet");
    }

    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    ProjectInstance targetProject = projectManager.getProject(targetProjectName);
    if (targetProject == null) {
        throw new IllegalStateException("Target project does not exist in target metadata: " + targetProjectName);
    }

    TableMetadataManager metadataManager = TableMetadataManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        TableDesc existing = metadataManager.getTableDesc(tableDesc.getIdentity(), targetProjectName);
        if (existing != null && !existing.equals(tableDesc)) {
            logger.info("Table {} already has a different version in target metadata store", tableDesc.getIdentity());
            logger.info("Existing version: {}", existing);
            logger.info("New version: {}", tableDesc);

            if (!forceIngest && !overwriteTables) {
                throw new IllegalStateException("table already exists with a different version: " + tableDesc.getIdentity() + ". Consider adding -overwriteTables option to force overwriting (with caution)");
            } else {
                logger.warn("Overwriting the old table desc: {}", tableDesc.getIdentity());
            }
        }
        requiredResources.add(tableDesc.getResourcePath());
    }

    DataModelManager modelManager = DataModelManager.getInstance(kylinConfig);
    for (DataModelDesc dataModelDesc : srcModelManager.listDataModels()) {
        checkExesting(modelManager.getDataModelDesc(dataModelDesc.getName()), "model", dataModelDesc.getName());
        requiredResources.add(DataModelDesc.concatResourcePath(dataModelDesc.getName()));
    }

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    for (CubeDesc cubeDesc : srcCubeDescManager.listAllDesc()) {
        checkExesting(cubeDescManager.getCubeDesc(cubeDesc.getName()), "cube desc", cubeDesc.getName());
        requiredResources.add(CubeDesc.concatResourcePath(cubeDesc.getName()));
    }

    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        checkExesting(cubeManager.getCube(cube.getName()), "cube", cube.getName());
        requiredResources.add(CubeInstance.concatResourcePath(cube.getName()));
    }

}