Java Code Examples for org.apache.kylin.metadata.project.ProjectManager

The following examples show how to use org.apache.kylin.metadata.project.ProjectManager. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: kylin   Source File: LookupSnapshotBuildJob.java    License: Apache License 2.0 6 votes vote down vote up
private static LookupSnapshotBuildJob initJob(CubeInstance cube, String tableName, String submitter,
        KylinConfig kylinConfig) {
    List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),
            cube.getName());
    if (projList == null || projList.size() == 0) {
        throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
    } else if (projList.size() >= 2) {
        String msg = "Find more than one project containing the cube " + cube.getName()
                + ". It does't meet the uniqueness requirement!!! ";
        throw new RuntimeException(msg);
    }

    LookupSnapshotBuildJob result = new LookupSnapshotBuildJob();
    SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    format.setTimeZone(TimeZone.getTimeZone(kylinConfig.getTimeZone()));
    result.setDeployEnvName(kylinConfig.getDeployEnv());
    result.setProjectName(projList.get(0).getName());
    CubingExecutableUtil.setCubeName(cube.getName(), result.getParams());
    result.setName(JOB_TYPE + " CUBE - " + cube.getName() + " - " + " TABLE - " + tableName + " - "
            + format.format(new Date(System.currentTimeMillis())));
    result.setSubmitter(submitter);
    result.setNotifyList(cube.getDescriptor().getNotifyList());
    return result;
}
 
Example 2
Source Project: kylin   Source File: CubeDesc.java    License: Apache License 2.0 6 votes vote down vote up
public String getProject() {
    DataModelDesc modelDesc = getModel();
    if (modelDesc == null) {
        // In case the model for cube not exists in metadata
        List<ProjectInstance> ownerPrj = ProjectManager.getInstance(config).findProjects(RealizationType.CUBE,
                name);

        if (ownerPrj.size() == 1) {
            return ownerPrj.get(0).getName();
        } else {
            throw new IllegalStateException("No project found for cube " + name);
        }
    } else {
        return getModel().getProject();
    }
}
 
Example 3
Source Project: kylin-on-parquet-v2   Source File: TableMetadataManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
        throws IOException {
    try (AutoLock lock = srcTableMapLock.lockForWrite()) {
        if (event == Event.DROP)
            srcTableMap.removeLocal(cacheKey);
        else
            srcTableCrud.reloadQuietly(cacheKey);
    }

    TableProject tableProject = TableDesc.parseResourcePath(cacheKey);
    String table = tableProject.getTable();
    String prj = tableProject.getProject();

    if (prj == null) {
        for (ProjectInstance p : ProjectManager.getInstance(config).findProjectsByTable(table)) {
            broadcaster.notifyProjectSchemaUpdate(p.getName());
        }
    } else {
        broadcaster.notifyProjectSchemaUpdate(prj);
    }
}
 
Example 4
Source Project: kylin-on-parquet-v2   Source File: TableMetadataManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * the project-specific table desc will be expand by computed columns from the projects' models
 * when the projects' model list changed, project-specific table should be reset and get expanded
 * again
 */
public void resetProjectSpecificTableDesc(String prj) throws IOException {
    // avoid cyclic locks
    ProjectInstance project = ProjectManager.getInstance(config).getProject(prj);
    
    try (AutoLock lock = srcTableMapLock.lockForWrite()) {
        for (String tableName : project.getTables()) {
            String tableIdentity = getTableIdentity(tableName);
            String key = mapKey(tableIdentity, prj);
            TableDesc originTableDesc = srcTableMap.get(key);
            if (originTableDesc == null) {
                continue;
            }

            if (originTableDesc.isBorrowedFromGlobal()) {
                srcTableMap.removeLocal(key);//delete it so that getProjectSpecificTableDesc will create again
            } else {
                srcTableCrud.reload(key);
            }
        }
    }
}
 
Example 5
private static LookupSnapshotBuildJob initJob(CubeInstance cube, String tableName, String submitter,
        KylinConfig kylinConfig) {
    List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),
            cube.getName());
    if (projList == null || projList.size() == 0) {
        throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
    } else if (projList.size() >= 2) {
        String msg = "Find more than one project containing the cube " + cube.getName()
                + ". It does't meet the uniqueness requirement!!! ";
        throw new RuntimeException(msg);
    }

    LookupSnapshotBuildJob result = new LookupSnapshotBuildJob();
    SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    format.setTimeZone(TimeZone.getTimeZone(kylinConfig.getTimeZone()));
    result.setDeployEnvName(kylinConfig.getDeployEnv());
    result.setProjectName(projList.get(0).getName());
    CubingExecutableUtil.setCubeName(cube.getName(), result.getParams());
    result.setName(JOB_TYPE + " CUBE - " + cube.getName() + " - " + " TABLE - " + tableName + " - "
            + format.format(new Date(System.currentTimeMillis())));
    result.setSubmitter(submitter);
    result.setNotifyList(cube.getDescriptor().getNotifyList());
    return result;
}
 
Example 6
Source Project: kylin-on-parquet-v2   Source File: ProjectControllerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAddUpdateProject() throws IOException {
    int originalProjectCount = projectController.getProjects(null, null).size();

    //test add project
    ProjectInstance project = new ProjectInstance();
    project.setName("new_project");
    ProjectInstance ret = projectController.saveProject(getProjectRequest(project, null));

    Assert.assertEquals(ret.getOwner(), "ADMIN");
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);

    //test update project description only
    ProjectInstance newProject2 = new ProjectInstance();
    newProject2.setName("new_project");
    newProject2.setDescription("hello world");
    projectController.updateProject(getProjectRequest(newProject2, "new_project"));

    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);
    Assert.assertNotEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null);
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project").getDescription(), "hello world");
}
 
Example 7
Source Project: kylin   Source File: TableMetadataManager.java    License: Apache License 2.0 6 votes vote down vote up
public Map<String, TableDesc> getAllTablesMap(String prj) {
    // avoid cyclic locks
    ProjectInstance project = (prj == null) ? null : ProjectManager.getInstance(config).getProject(prj);
    
    try (AutoLock lock = srcTableMapLock.lockForWrite()) {
        //TODO prj == null case is now only used by test case and CubeMetaIngester
        //should refactor these test case and tool ASAP and stop supporting null case
        if (prj == null) {
            Map<String, TableDesc> globalTables = new LinkedHashMap<>();

            for (TableDesc t : srcTableMap.values()) {
                globalTables.put(t.getIdentity(), t);
            }
            return globalTables;
        }

        Set<String> prjTableNames = project.getTables();

        Map<String, TableDesc> ret = new LinkedHashMap<>();
        for (String tableName : prjTableNames) {
            String tableIdentity = getTableIdentity(tableName);
            ret.put(tableIdentity, getProjectSpecificTableDesc(tableIdentity, prj, true));
        }
        return ret;
    }
}
 
Example 8
protected static void populateSSWithCSVData(KylinConfig kylinConfig, String project, SparkSession sparkSession) {

        ProjectInstance projectInstance = ProjectManager.getInstance(kylinConfig).getProject(project);
        Preconditions.checkArgument(projectInstance != null);
        for (String table : projectInstance.getTables()) {

            if ("DEFAULT.STREAMING_TABLE".equals(table)) {
                continue;
            }
            if (!new File(String.format(Locale.ROOT, CSV_TABLE_DIR, table)).exists()) {
                continue;
            }
            TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(table, project);
            ColumnDesc[] columns = tableDesc.getColumns();
            StructType schema = new StructType();
            for (ColumnDesc column : columns) {
                schema = schema.add(column.getName(), convertType(column.getType()), false);
            }
            Dataset<Row> ret = sparkSession.read().schema(schema).csv(String.format(Locale.ROOT, CSV_TABLE_DIR, table));
            ret.createOrReplaceTempView(tableDesc.getName());
        }

    }
 
Example 9
Source Project: kylin   Source File: TableACLService.java    License: Apache License 2.0 6 votes vote down vote up
private <T> List<T> filterByAcl(List<T> tables, String project, AclFilter filter) throws IOException {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());

    if (aclEvaluate.hasProjectAdminPermission(projectManager.getProject(project))) {
        return tables;
    }

    String username = SecurityContextHolder.getContext().getAuthentication().getName();
    Set<String> blockedTables = getBlockedTablesByUser(project, username, TYPE_USER);
    List<T> result = Lists.newArrayList();
    for (T table : tables) {
        if (filter.filter(table, blockedTables)) {
            result.add(table);
        }
    }
    return result;
}
 
Example 10
Source Project: kylin-on-parquet-v2   Source File: CubeManager.java    License: Apache License 2.0 6 votes vote down vote up
public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner)
        throws IOException {
    try (AutoLock lock = cubeMapLock.lockForWrite()) {
        logger.info("Creating cube '{}-->{}' from desc '{}'", projectName, cubeName, desc.getName());

        // save cube resource
        CubeInstance cube = CubeInstance.create(cubeName, desc);
        cube.setOwner(owner);
        updateCubeWithRetry(new CubeUpdate(cube), 0);

        ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName,
                owner);

        return cube;
    }
}
 
Example 11
Source Project: kylin   Source File: TableMetadataManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * the project-specific table desc will be expand by computed columns from the projects' models
 * when the projects' model list changed, project-specific table should be reset and get expanded
 * again
 */
public void resetProjectSpecificTableDesc(String prj) throws IOException {
    // avoid cyclic locks
    ProjectInstance project = ProjectManager.getInstance(config).getProject(prj);
    
    try (AutoLock lock = srcTableMapLock.lockForWrite()) {
        for (String tableName : project.getTables()) {
            String tableIdentity = getTableIdentity(tableName);
            String key = mapKey(tableIdentity, prj);
            TableDesc originTableDesc = srcTableMap.get(key);
            if (originTableDesc == null) {
                continue;
            }

            if (originTableDesc.isBorrowedFromGlobal()) {
                srcTableMap.removeLocal(key);//delete it so that getProjectSpecificTableDesc will create again
            } else {
                srcTableCrud.reload(key);
            }
        }
    }
}
 
Example 12
Source Project: kylin   Source File: OLAPSchema.java    License: Apache License 2.0 6 votes vote down vote up
private Map<String, Table> buildTableMap() {
    Map<String, Table> olapTables = new HashMap<String, Table>();

    Collection<TableDesc> projectTables = ProjectManager.getInstance(config).listExposedTables(projectName,
            exposeMore);

    for (TableDesc tableDesc : projectTables) {
        if (tableDesc.getDatabase().equals(schemaName)) {
            final String tableName = tableDesc.getName();//safe to use tableDesc.getName() here, it is in a DB context now
            final OLAPTable table = new OLAPTable(this, tableDesc, exposeMore);
            olapTables.put(tableName, table);
            //logger.debug("Project " + projectName + " exposes table " + tableName);
        }
    }

    return olapTables;
}
 
Example 13
Source Project: kylin   Source File: DataModelManager.java    License: Apache License 2.0 6 votes vote down vote up
protected void init(KylinConfig cfg) throws IOException {
    this.config = cfg;
    this.dataModelDescMap = new CaseInsensitiveStringCache<>(config, "data_model");
    this.crud = new CachedCrudAssist<DataModelDesc>(getStore(), ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT,
            getDataModelImplClass(), dataModelDescMap) {
        @Override
        protected DataModelDesc initEntityAfterReload(DataModelDesc model, String resourceName) {
            String prj = (null == model.getProjectName()
                    ? ProjectManager.getInstance(config).getProjectOfModel(model.getName()).getName()
                    : model.getProjectName());
            if (!model.isDraft()) {
                model.init(config, getAllTablesMap(prj));
            }
            return model;
        }
    };

    // touch lower level metadata before registering model listener
    TableMetadataManager.getInstance(config);
    crud.reloadAll();
    Broadcaster.getInstance(config).registerListener(new DataModelSyncListener(), "data_model");
}
 
Example 14
Source Project: kylin-on-parquet-v2   Source File: CubeManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateAndDrop() throws Exception {

    KylinConfig config = getTestConfig();
    CubeManager cubeMgr = CubeManager.getInstance(config);
    ProjectManager prjMgr = ProjectManager.getInstance(config);
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/a_whole_new_cube.json");

    CubeDescManager cubeDescMgr = getCubeDescManager();
    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("a_whole_new_cube", ProjectInstance.DEFAULT_PROJECT_NAME, desc,
            null);
    assertTrue(createdCube.equals(cubeMgr.getCube("a_whole_new_cube")));

    assertTrue(prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(createdCube));

    CubeInstance droppedCube = CubeManager.getInstance(getTestConfig()).dropCube("a_whole_new_cube", false);
    assertTrue(createdCube.equals(droppedCube));

    assertTrue(!prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(droppedCube));

    assertNull(CubeManager.getInstance(getTestConfig()).getCube("a_whole_new_cube"));
}
 
Example 15
Source Project: kylin-on-parquet-v2   Source File: TableACLService.java    License: Apache License 2.0 6 votes vote down vote up
private <T> List<T> filterByAcl(List<T> tables, String project, AclFilter filter) throws IOException {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());

    if (aclEvaluate.hasProjectAdminPermission(projectManager.getProject(project))) {
        return tables;
    }

    String username = SecurityContextHolder.getContext().getAuthentication().getName();
    Set<String> blockedTables = getBlockedTablesByUser(project, username, TYPE_USER);
    List<T> result = Lists.newArrayList();
    for (T table : tables) {
        if (filter.filter(table, blockedTables)) {
            result.add(table);
        }
    }
    return result;
}
 
Example 16
Source Project: kylin-on-parquet-v2   Source File: CubeService.java    License: Apache License 2.0 6 votes vote down vote up
public List<CubeInstance> listAllCubes(String projectName) {
    ProjectManager projectManager = getProjectManager();
    ProjectInstance project = projectManager.getProject(projectName);
    if (project == null) {
        return Collections.emptyList();
    }
    ArrayList<CubeInstance> result = new ArrayList<CubeInstance>();
    for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
        if (projectDataModel.getType() == RealizationType.CUBE) {
            CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
            if (cube != null)
                result.add(cube);
            else
                logger.error("Cube instance " + projectDataModel.getRealization() + " is failed to load");
        }
    }
    return result;
}
 
Example 17
Source Project: kylin   Source File: AccessService.java    License: Apache License 2.0 6 votes vote down vote up
private Map<String, Integer> getProjectPermission(String project) {
    Map<String, Integer> SidWithPermission = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);

    String uuid = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project).getUuid();
    AclEntity ae = getAclEntity(AclEntityType.PROJECT_INSTANCE, uuid);
    Acl acl = getAcl(ae);
    if (acl != null && acl.getEntries() != null) {
        List<AccessControlEntry> aces = acl.getEntries();
        for (AccessControlEntry ace : aces) {
            Sid sid = ace.getSid();
            if (sid instanceof PrincipalSid) {
                String principal = ((PrincipalSid) sid).getPrincipal();
                SidWithPermission.put(principal, ace.getPermission().getMask());
            }
            if (sid instanceof GrantedAuthoritySid) {
                String grantedAuthority = ((GrantedAuthoritySid) sid).getGrantedAuthority();
                SidWithPermission.put(grantedAuthority, ace.getPermission().getMask());
            }
        }
    }
    return SidWithPermission;
}
 
Example 18
Source Project: kylin-on-parquet-v2   Source File: AccessService.java    License: Apache License 2.0 6 votes vote down vote up
private Map<String, Integer> getProjectPermission(String project) {
    Map<String, Integer> SidWithPermission = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);

    String uuid = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project).getUuid();
    AclEntity ae = getAclEntity(AclEntityType.PROJECT_INSTANCE, uuid);
    Acl acl = getAcl(ae);
    if (acl != null && acl.getEntries() != null) {
        List<AccessControlEntry> aces = acl.getEntries();
        for (AccessControlEntry ace : aces) {
            Sid sid = ace.getSid();
            if (sid instanceof PrincipalSid) {
                String principal = ((PrincipalSid) sid).getPrincipal();
                SidWithPermission.put(principal, ace.getPermission().getMask());
            }
            if (sid instanceof GrantedAuthoritySid) {
                String grantedAuthority = ((GrantedAuthoritySid) sid).getGrantedAuthority();
                SidWithPermission.put(grantedAuthority, ace.getPermission().getMask());
            }
        }
    }
    return SidWithPermission;
}
 
Example 19
Source Project: kylin   Source File: TableMetadataManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
        throws IOException {
    try (AutoLock lock = srcTableMapLock.lockForWrite()) {
        if (event == Event.DROP)
            srcTableMap.removeLocal(cacheKey);
        else
            srcTableCrud.reloadQuietly(cacheKey);
    }

    TableProject tableProject = TableDesc.parseResourcePath(cacheKey);
    String table = tableProject.getTable();
    String prj = tableProject.getProject();

    if (prj == null) {
        for (ProjectInstance p : ProjectManager.getInstance(config).findProjectsByTable(table)) {
            broadcaster.notifyProjectSchemaUpdate(p.getName());
        }
    } else {
        broadcaster.notifyProjectSchemaUpdate(prj);
    }
}
 
Example 20
Source Project: Kylin   Source File: OLAPTable.java    License: Apache License 2.0 6 votes vote down vote up
private List<ColumnDesc> listSourceColumns() {
    ProjectManager mgr = ProjectManager.getInstance(olapSchema.getConfig());
    List<ColumnDesc> exposedColumns = Lists.newArrayList(mgr.listExposedColumns(olapSchema.getProjectName(), sourceTable.getIdentity()));

    List<MeasureDesc> countMeasures = mgr.listEffectiveRewriteMeasures(olapSchema.getProjectName(), sourceTable.getIdentity());
    HashSet<String> metFields = new HashSet<String>();
    for (MeasureDesc m : countMeasures) {
        FunctionDesc func = m.getFunction();
        String fieldName = func.getRewriteFieldName();
        if (!metFields.contains(fieldName)) {
            metFields.add(fieldName);
            ColumnDesc fakeCountCol = new ColumnDesc();
            fakeCountCol.setName(fieldName);
            fakeCountCol.setDatatype(func.getSQLType());
            fakeCountCol.setNullable(false);
            fakeCountCol.init(sourceTable);
            exposedColumns.add(fakeCountCol);
        }
    }

    return exposedColumns;
}
 
Example 21
Source Project: kylin   Source File: DataModelManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
    //clean up the current project's table desc
    TableMetadataManager.getInstance(config).resetProjectSpecificTableDesc(project);
    logger.info("Update models in project: " + project);
    try (AutoLock lock = modelMapLock.lockForWrite()) {
        for (String model : ProjectManager.getInstance(config).getProject(project).getModels()) {
            crud.reloadQuietly(model);
        }
    }
}
 
Example 22
Source Project: kylin   Source File: CubeHBaseEndpointRPC.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ExecutorService load(String projName) throws Exception {
    ExecutorService sharedPool = HBaseConnection.getCoprocessorPool();
    ProjectInstance projInst = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getProject(projName);
    return new SubThreadPoolExecutor(sharedPool, "PROJECT",
            projInst.getConfig().getHBaseMaxConnectionThreadsPerProject());
}
 
Example 23
Source Project: kylin   Source File: CubeManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
    ProjectManager projectManager = ProjectManager.getInstance(config);
    for (IRealization real : projectManager.listAllRealizations(project)) {
        if (real instanceof CubeInstance) {
            reloadCubeQuietly(real.getName());
        }
    }
    projectManager.reloadProjectL2Cache(project);
}
 
Example 24
Source Project: Kylin   Source File: ProjectControllerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAddUpdateProject() throws IOException {

    List<ProjectInstance> projects = projectController.getProjects(null, null);

    int originalProjectCount = projects.size();
    CreateProjectRequest request = new CreateProjectRequest();
    request.setName("new_project");
    ProjectInstance ret = projectController.saveProject(request);

    Assert.assertEquals(ret.getOwner(), "ADMIN");
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);

    UpdateProjectRequest updateR = new UpdateProjectRequest();
    updateR.setFormerProjectName("new_project");
    updateR.setNewProjectName("new_project_2");
    projectController.updateProject(updateR);

    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null);

    Assert.assertNotEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project_2"), null);

    // only update desc:
    updateR = new UpdateProjectRequest();
    updateR.setFormerProjectName("new_project_2");
    updateR.setNewProjectName("new_project_2");
    updateR.setNewDescription("hello world");
    projectController.updateProject(updateR);

    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null);
    Assert.assertNotEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project_2"), null);
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project_2").getDescription(), "hello world");
}
 
Example 25
Source Project: kylin-on-parquet-v2   Source File: HBaseUsageExtractor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void executeExtract(OptionsHelper optionsHelper, File exportDir) throws Exception {
    if (cachedHMasterUrl == null) {
        return;
    }
    kylinConfig = KylinConfig.getInstanceFromEnv();
    cubeManager = CubeManager.getInstance(kylinConfig);
    realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
    projectManager = ProjectManager.getInstance(kylinConfig);

    if (optionsHelper.hasOption(OPTION_PROJECT)) {
        String projectNames = optionsHelper.getOptionValue(OPTION_PROJECT);
        for (String projectName : StringUtil.splitByComma(projectNames)) {
            ProjectInstance projectInstance = projectManager.getProject(projectName);
            if (projectInstance == null) {
                throw new IllegalArgumentException("Project " + projectName + " does not exist");
            }
            List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
            for (RealizationEntry realizationEntry : realizationEntries) {
                retrieveResourcePath(getRealization(realizationEntry));
            }
        }
    } else if (optionsHelper.hasOption(OPTION_CUBE)) {
        String cubeNames = optionsHelper.getOptionValue(OPTION_CUBE);
        for (String cubeName : StringUtil.splitByComma(cubeNames)) {
            IRealization realization = cubeManager.getRealization(cubeName);
            if (realization != null) {
                retrieveResourcePath(realization);
            } else {
                throw new IllegalArgumentException("No cube found with name of " + cubeName);
            }
        }
    }

    extractCommonInfo(exportDir, kylinConfig);
    extractHTables(exportDir);
}
 
Example 26
Source Project: kylin-on-parquet-v2   Source File: KylinLogExtractor.java    License: Apache License 2.0 5 votes vote down vote up
private void beforeExtract() {
    // reload metadata before extract diagnosis info
    logger.info("Start to reload metadata from diagnosis.");
    
    config.clearManagers();

    CubeManager.getInstance(config);
    CubeDescManager.getInstance(config);
    DataModelManager.getInstance(config);
    ProjectManager.getInstance(config);
}
 
Example 27
Source Project: kylin-on-parquet-v2   Source File: JobInstanceExtractor.java    License: Apache License 2.0 5 votes vote down vote up
public JobInstanceExtractor() {
    config = KylinConfig.getInstanceFromEnv();
    executableManager = ExecutableManager.getInstance(config);
    projectManager = ProjectManager.getInstance(config);

    packageType = "jobInstances";

    options.addOption(OPTION_PROJECT);
    options.addOption(OPTION_CUBE);
    options.addOption(OPTION_PERIOD);
}
 
Example 28
Source Project: kylin-on-parquet-v2   Source File: CubeMetaIngesterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHappyIngest() {
    String srcPath = Thread.currentThread().getContextClassLoader().getResource("cloned_cube_and_model.zip").getPath();
    CubeMetaIngester.main(new String[] { "-project", "default", "-srcPath", srcPath });

    ProjectInstance project = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject("default");
    Assert.assertEquals(1, Collections.frequency(project.getTables(), "DEFAULT.TEST_KYLIN_FACT"));
    Assert.assertTrue(project.getModels().contains("cloned_model"));
    Assert.assertTrue(project.getRealizationEntries().contains(RealizationEntry.create(RealizationType.CUBE, "cloned_cube")));

    getTestConfig().clearManagers();
    CubeInstance instance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("cloned_cube");
    Assert.assertTrue(instance != null);
}
 
Example 29
Source Project: kylin-on-parquet-v2   Source File: CubeMetaIngesterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHappyIngest2() {
    String srcPath = Thread.currentThread().getContextClassLoader().getResource("benchmark_meta.zip").getPath();
    CubeMetaIngester.main(new String[] { "-project", "default", "-srcPath", srcPath, "-overwriteTables", "true" });

    ProjectInstance project = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject("default");
    Assert.assertEquals(1, Collections.frequency(project.getTables(), "SSB.CUSTOMER"));
    Assert.assertTrue(project.getModels().contains("benchmark_model"));
    Assert.assertTrue(project.getRealizationEntries().contains(RealizationEntry.create(RealizationType.CUBE, "benchmark_cube")));

    getTestConfig().clearManagers();
    CubeInstance instance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("benchmark_cube");
    Assert.assertTrue(instance != null);
}
 
Example 30
Source Project: kylin-on-parquet-v2   Source File: HybridCubeCLITest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void test3Delete() throws IOException {
    HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
    Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
    HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });
    Assert.assertNotNull(hybridManager.getHybridInstance("ssb_hybrid"));
    HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-action", "delete" });

    HybridInstance hybridInstance = hybridManager.getHybridInstance("ssb_hybrid");
    Assert.assertNull(hybridInstance);
    Assert.assertEquals(0, ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).findProjects(RealizationType.HYBRID, "ssb_hybrid").size());
}