org.apache.kylin.metadata.project.ProjectManager Java Examples

The following examples show how to use org.apache.kylin.metadata.project.ProjectManager. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: OLAPTable.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private List<ColumnDesc> listSourceColumns() {
    ProjectManager mgr = ProjectManager.getInstance(olapSchema.getConfig());
    List<ColumnDesc> exposedColumns = Lists.newArrayList(mgr.listExposedColumns(olapSchema.getProjectName(), sourceTable.getIdentity()));

    List<MeasureDesc> countMeasures = mgr.listEffectiveRewriteMeasures(olapSchema.getProjectName(), sourceTable.getIdentity());
    HashSet<String> metFields = new HashSet<String>();
    for (MeasureDesc m : countMeasures) {
        FunctionDesc func = m.getFunction();
        String fieldName = func.getRewriteFieldName();
        if (!metFields.contains(fieldName)) {
            metFields.add(fieldName);
            ColumnDesc fakeCountCol = new ColumnDesc();
            fakeCountCol.setName(fieldName);
            fakeCountCol.setDatatype(func.getSQLType());
            fakeCountCol.setNullable(false);
            fakeCountCol.init(sourceTable);
            exposedColumns.add(fakeCountCol);
        }
    }

    return exposedColumns;
}
 
Example #2
Source File: LookupSnapshotBuildJob.java    From kylin with Apache License 2.0 6 votes vote down vote up
private static LookupSnapshotBuildJob initJob(CubeInstance cube, String tableName, String submitter,
        KylinConfig kylinConfig) {
    List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),
            cube.getName());
    if (projList == null || projList.size() == 0) {
        throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
    } else if (projList.size() >= 2) {
        String msg = "Find more than one project containing the cube " + cube.getName()
                + ". It does't meet the uniqueness requirement!!! ";
        throw new RuntimeException(msg);
    }

    LookupSnapshotBuildJob result = new LookupSnapshotBuildJob();
    SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    format.setTimeZone(TimeZone.getTimeZone(kylinConfig.getTimeZone()));
    result.setDeployEnvName(kylinConfig.getDeployEnv());
    result.setProjectName(projList.get(0).getName());
    CubingExecutableUtil.setCubeName(cube.getName(), result.getParams());
    result.setName(JOB_TYPE + " CUBE - " + cube.getName() + " - " + " TABLE - " + tableName + " - "
            + format.format(new Date(System.currentTimeMillis())));
    result.setSubmitter(submitter);
    result.setNotifyList(cube.getDescriptor().getNotifyList());
    return result;
}
 
Example #3
Source File: CubeDesc.java    From kylin with Apache License 2.0 6 votes vote down vote up
public String getProject() {
    DataModelDesc modelDesc = getModel();
    if (modelDesc == null) {
        // In case the model for cube not exists in metadata
        List<ProjectInstance> ownerPrj = ProjectManager.getInstance(config).findProjects(RealizationType.CUBE,
                name);

        if (ownerPrj.size() == 1) {
            return ownerPrj.get(0).getName();
        } else {
            throw new IllegalStateException("No project found for cube " + name);
        }
    } else {
        return getModel().getProject();
    }
}
 
Example #4
Source File: TableMetadataManager.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
        throws IOException {
    try (AutoLock lock = srcTableMapLock.lockForWrite()) {
        if (event == Event.DROP)
            srcTableMap.removeLocal(cacheKey);
        else
            srcTableCrud.reloadQuietly(cacheKey);
    }

    TableProject tableProject = TableDesc.parseResourcePath(cacheKey);
    String table = tableProject.getTable();
    String prj = tableProject.getProject();

    if (prj == null) {
        for (ProjectInstance p : ProjectManager.getInstance(config).findProjectsByTable(table)) {
            broadcaster.notifyProjectSchemaUpdate(p.getName());
        }
    } else {
        broadcaster.notifyProjectSchemaUpdate(prj);
    }
}
 
Example #5
Source File: TableMetadataManager.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
/**
 * the project-specific table desc will be expand by computed columns from the projects' models
 * when the projects' model list changed, project-specific table should be reset and get expanded
 * again
 */
public void resetProjectSpecificTableDesc(String prj) throws IOException {
    // avoid cyclic locks
    ProjectInstance project = ProjectManager.getInstance(config).getProject(prj);
    
    try (AutoLock lock = srcTableMapLock.lockForWrite()) {
        for (String tableName : project.getTables()) {
            String tableIdentity = getTableIdentity(tableName);
            String key = mapKey(tableIdentity, prj);
            TableDesc originTableDesc = srcTableMap.get(key);
            if (originTableDesc == null) {
                continue;
            }

            if (originTableDesc.isBorrowedFromGlobal()) {
                srcTableMap.removeLocal(key);//delete it so that getProjectSpecificTableDesc will create again
            } else {
                srcTableCrud.reload(key);
            }
        }
    }
}
 
Example #6
Source File: LookupSnapshotBuildJob.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private static LookupSnapshotBuildJob initJob(CubeInstance cube, String tableName, String submitter,
        KylinConfig kylinConfig) {
    List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),
            cube.getName());
    if (projList == null || projList.size() == 0) {
        throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
    } else if (projList.size() >= 2) {
        String msg = "Find more than one project containing the cube " + cube.getName()
                + ". It does't meet the uniqueness requirement!!! ";
        throw new RuntimeException(msg);
    }

    LookupSnapshotBuildJob result = new LookupSnapshotBuildJob();
    SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    format.setTimeZone(TimeZone.getTimeZone(kylinConfig.getTimeZone()));
    result.setDeployEnvName(kylinConfig.getDeployEnv());
    result.setProjectName(projList.get(0).getName());
    CubingExecutableUtil.setCubeName(cube.getName(), result.getParams());
    result.setName(JOB_TYPE + " CUBE - " + cube.getName() + " - " + " TABLE - " + tableName + " - "
            + format.format(new Date(System.currentTimeMillis())));
    result.setSubmitter(submitter);
    result.setNotifyList(cube.getDescriptor().getNotifyList());
    return result;
}
 
Example #7
Source File: TableMetadataManager.java    From kylin with Apache License 2.0 6 votes vote down vote up
public Map<String, TableDesc> getAllTablesMap(String prj) {
    // avoid cyclic locks
    ProjectInstance project = (prj == null) ? null : ProjectManager.getInstance(config).getProject(prj);
    
    try (AutoLock lock = srcTableMapLock.lockForWrite()) {
        //TODO prj == null case is now only used by test case and CubeMetaIngester
        //should refactor these test case and tool ASAP and stop supporting null case
        if (prj == null) {
            Map<String, TableDesc> globalTables = new LinkedHashMap<>();

            for (TableDesc t : srcTableMap.values()) {
                globalTables.put(t.getIdentity(), t);
            }
            return globalTables;
        }

        Set<String> prjTableNames = project.getTables();

        Map<String, TableDesc> ret = new LinkedHashMap<>();
        for (String tableName : prjTableNames) {
            String tableIdentity = getTableIdentity(tableName);
            ret.put(tableIdentity, getProjectSpecificTableDesc(tableIdentity, prj, true));
        }
        return ret;
    }
}
 
Example #8
Source File: LocalWithSparkSessionTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
protected static void populateSSWithCSVData(KylinConfig kylinConfig, String project, SparkSession sparkSession) {

        ProjectInstance projectInstance = ProjectManager.getInstance(kylinConfig).getProject(project);
        Preconditions.checkArgument(projectInstance != null);
        for (String table : projectInstance.getTables()) {

            if ("DEFAULT.STREAMING_TABLE".equals(table)) {
                continue;
            }
            if (!new File(String.format(Locale.ROOT, CSV_TABLE_DIR, table)).exists()) {
                continue;
            }
            TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(table, project);
            ColumnDesc[] columns = tableDesc.getColumns();
            StructType schema = new StructType();
            for (ColumnDesc column : columns) {
                schema = schema.add(column.getName(), convertType(column.getType()), false);
            }
            Dataset<Row> ret = sparkSession.read().schema(schema).csv(String.format(Locale.ROOT, CSV_TABLE_DIR, table));
            ret.createOrReplaceTempView(tableDesc.getName());
        }

    }
 
Example #9
Source File: TableACLService.java    From kylin with Apache License 2.0 6 votes vote down vote up
private <T> List<T> filterByAcl(List<T> tables, String project, AclFilter filter) throws IOException {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());

    if (aclEvaluate.hasProjectAdminPermission(projectManager.getProject(project))) {
        return tables;
    }

    String username = SecurityContextHolder.getContext().getAuthentication().getName();
    Set<String> blockedTables = getBlockedTablesByUser(project, username, TYPE_USER);
    List<T> result = Lists.newArrayList();
    for (T table : tables) {
        if (filter.filter(table, blockedTables)) {
            result.add(table);
        }
    }
    return result;
}
 
Example #10
Source File: CubeManager.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner)
        throws IOException {
    try (AutoLock lock = cubeMapLock.lockForWrite()) {
        logger.info("Creating cube '{}-->{}' from desc '{}'", projectName, cubeName, desc.getName());

        // save cube resource
        CubeInstance cube = CubeInstance.create(cubeName, desc);
        cube.setOwner(owner);
        updateCubeWithRetry(new CubeUpdate(cube), 0);

        ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName,
                owner);

        return cube;
    }
}
 
Example #11
Source File: OLAPSchema.java    From kylin with Apache License 2.0 6 votes vote down vote up
private Map<String, Table> buildTableMap() {
    Map<String, Table> olapTables = new HashMap<String, Table>();

    Collection<TableDesc> projectTables = ProjectManager.getInstance(config).listExposedTables(projectName,
            exposeMore);

    for (TableDesc tableDesc : projectTables) {
        if (tableDesc.getDatabase().equals(schemaName)) {
            final String tableName = tableDesc.getName();//safe to use tableDesc.getName() here, it is in a DB context now
            final OLAPTable table = new OLAPTable(this, tableDesc, exposeMore);
            olapTables.put(tableName, table);
            //logger.debug("Project " + projectName + " exposes table " + tableName);
        }
    }

    return olapTables;
}
 
Example #12
Source File: DataModelManager.java    From kylin with Apache License 2.0 6 votes vote down vote up
protected void init(KylinConfig cfg) throws IOException {
    this.config = cfg;
    this.dataModelDescMap = new CaseInsensitiveStringCache<>(config, "data_model");
    this.crud = new CachedCrudAssist<DataModelDesc>(getStore(), ResourceStore.DATA_MODEL_DESC_RESOURCE_ROOT,
            getDataModelImplClass(), dataModelDescMap) {
        @Override
        protected DataModelDesc initEntityAfterReload(DataModelDesc model, String resourceName) {
            String prj = (null == model.getProjectName()
                    ? ProjectManager.getInstance(config).getProjectOfModel(model.getName()).getName()
                    : model.getProjectName());
            if (!model.isDraft()) {
                model.init(config, getAllTablesMap(prj));
            }
            return model;
        }
    };

    // touch lower level metadata before registering model listener
    TableMetadataManager.getInstance(config);
    crud.reloadAll();
    Broadcaster.getInstance(config).registerListener(new DataModelSyncListener(), "data_model");
}
 
Example #13
Source File: CubeManagerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateAndDrop() throws Exception {

    KylinConfig config = getTestConfig();
    CubeManager cubeMgr = CubeManager.getInstance(config);
    ProjectManager prjMgr = ProjectManager.getInstance(config);
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/a_whole_new_cube.json");

    CubeDescManager cubeDescMgr = getCubeDescManager();
    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("a_whole_new_cube", ProjectInstance.DEFAULT_PROJECT_NAME, desc,
            null);
    assertTrue(createdCube.equals(cubeMgr.getCube("a_whole_new_cube")));

    assertTrue(prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(createdCube));

    CubeInstance droppedCube = CubeManager.getInstance(getTestConfig()).dropCube("a_whole_new_cube", false);
    assertTrue(createdCube.equals(droppedCube));

    assertTrue(!prjMgr.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME).contains(droppedCube));

    assertNull(CubeManager.getInstance(getTestConfig()).getCube("a_whole_new_cube"));
}
 
Example #14
Source File: TableACLService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private <T> List<T> filterByAcl(List<T> tables, String project, AclFilter filter) throws IOException {
    ProjectManager projectManager = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv());

    if (aclEvaluate.hasProjectAdminPermission(projectManager.getProject(project))) {
        return tables;
    }

    String username = SecurityContextHolder.getContext().getAuthentication().getName();
    Set<String> blockedTables = getBlockedTablesByUser(project, username, TYPE_USER);
    List<T> result = Lists.newArrayList();
    for (T table : tables) {
        if (filter.filter(table, blockedTables)) {
            result.add(table);
        }
    }
    return result;
}
 
Example #15
Source File: CubeService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public List<CubeInstance> listAllCubes(String projectName) {
    ProjectManager projectManager = getProjectManager();
    ProjectInstance project = projectManager.getProject(projectName);
    if (project == null) {
        return Collections.emptyList();
    }
    ArrayList<CubeInstance> result = new ArrayList<CubeInstance>();
    for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
        if (projectDataModel.getType() == RealizationType.CUBE) {
            CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
            if (cube != null)
                result.add(cube);
            else
                logger.error("Cube instance " + projectDataModel.getRealization() + " is failed to load");
        }
    }
    return result;
}
 
Example #16
Source File: AccessService.java    From kylin with Apache License 2.0 6 votes vote down vote up
private Map<String, Integer> getProjectPermission(String project) {
    Map<String, Integer> SidWithPermission = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);

    String uuid = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project).getUuid();
    AclEntity ae = getAclEntity(AclEntityType.PROJECT_INSTANCE, uuid);
    Acl acl = getAcl(ae);
    if (acl != null && acl.getEntries() != null) {
        List<AccessControlEntry> aces = acl.getEntries();
        for (AccessControlEntry ace : aces) {
            Sid sid = ace.getSid();
            if (sid instanceof PrincipalSid) {
                String principal = ((PrincipalSid) sid).getPrincipal();
                SidWithPermission.put(principal, ace.getPermission().getMask());
            }
            if (sid instanceof GrantedAuthoritySid) {
                String grantedAuthority = ((GrantedAuthoritySid) sid).getGrantedAuthority();
                SidWithPermission.put(grantedAuthority, ace.getPermission().getMask());
            }
        }
    }
    return SidWithPermission;
}
 
Example #17
Source File: AccessService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private Map<String, Integer> getProjectPermission(String project) {
    Map<String, Integer> SidWithPermission = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);

    String uuid = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()).getProject(project).getUuid();
    AclEntity ae = getAclEntity(AclEntityType.PROJECT_INSTANCE, uuid);
    Acl acl = getAcl(ae);
    if (acl != null && acl.getEntries() != null) {
        List<AccessControlEntry> aces = acl.getEntries();
        for (AccessControlEntry ace : aces) {
            Sid sid = ace.getSid();
            if (sid instanceof PrincipalSid) {
                String principal = ((PrincipalSid) sid).getPrincipal();
                SidWithPermission.put(principal, ace.getPermission().getMask());
            }
            if (sid instanceof GrantedAuthoritySid) {
                String grantedAuthority = ((GrantedAuthoritySid) sid).getGrantedAuthority();
                SidWithPermission.put(grantedAuthority, ace.getPermission().getMask());
            }
        }
    }
    return SidWithPermission;
}
 
Example #18
Source File: TableMetadataManager.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
        throws IOException {
    try (AutoLock lock = srcTableMapLock.lockForWrite()) {
        if (event == Event.DROP)
            srcTableMap.removeLocal(cacheKey);
        else
            srcTableCrud.reloadQuietly(cacheKey);
    }

    TableProject tableProject = TableDesc.parseResourcePath(cacheKey);
    String table = tableProject.getTable();
    String prj = tableProject.getProject();

    if (prj == null) {
        for (ProjectInstance p : ProjectManager.getInstance(config).findProjectsByTable(table)) {
            broadcaster.notifyProjectSchemaUpdate(p.getName());
        }
    } else {
        broadcaster.notifyProjectSchemaUpdate(prj);
    }
}
 
Example #19
Source File: ProjectControllerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void testAddUpdateProject() throws IOException {
    int originalProjectCount = projectController.getProjects(null, null).size();

    //test add project
    ProjectInstance project = new ProjectInstance();
    project.setName("new_project");
    ProjectInstance ret = projectController.saveProject(getProjectRequest(project, null));

    Assert.assertEquals(ret.getOwner(), "ADMIN");
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);

    //test update project description only
    ProjectInstance newProject2 = new ProjectInstance();
    newProject2.setName("new_project");
    newProject2.setDescription("hello world");
    projectController.updateProject(getProjectRequest(newProject2, "new_project"));

    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);
    Assert.assertNotEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null);
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project").getDescription(), "hello world");
}
 
Example #20
Source File: TableMetadataManager.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * the project-specific table desc will be expand by computed columns from the projects' models
 * when the projects' model list changed, project-specific table should be reset and get expanded
 * again
 */
public void resetProjectSpecificTableDesc(String prj) throws IOException {
    // avoid cyclic locks
    ProjectInstance project = ProjectManager.getInstance(config).getProject(prj);
    
    try (AutoLock lock = srcTableMapLock.lockForWrite()) {
        for (String tableName : project.getTables()) {
            String tableIdentity = getTableIdentity(tableName);
            String key = mapKey(tableIdentity, prj);
            TableDesc originTableDesc = srcTableMap.get(key);
            if (originTableDesc == null) {
                continue;
            }

            if (originTableDesc.isBorrowedFromGlobal()) {
                srcTableMap.removeLocal(key);//delete it so that getProjectSpecificTableDesc will create again
            } else {
                srcTableCrud.reload(key);
            }
        }
    }
}
 
Example #21
Source File: JobInstanceExtractor.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public JobInstanceExtractor() {
    config = KylinConfig.getInstanceFromEnv();
    executableManager = ExecutableManager.getInstance(config);
    projectManager = ProjectManager.getInstance(config);

    packageType = "jobInstances";

    options.addOption(OPTION_PROJECT);
    options.addOption(OPTION_CUBE);
    options.addOption(OPTION_PERIOD);
}
 
Example #22
Source File: ProjectManagerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testExistingProject() throws Exception {
    ProjectManager prjMgr = ProjectManager.getInstance(getTestConfig());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeDescManager cubeDescMgr = CubeDescManager.getInstance(getTestConfig());

    int originalProjectCount = prjMgr.listAllProjects().size();
    int originalCubeCount = cubeMgr.listAllCubes().size();
    ResourceStore store = getStore();

    // clean legacy in case last run failed
    store.deleteResource("/cube/new_cube_in_default.json");

    CubeDesc desc = cubeDescMgr.getCubeDesc("test_kylin_cube_with_slr_desc");
    CubeInstance createdCube = cubeMgr.createCube("new_cube_in_default", ProjectInstance.DEFAULT_PROJECT_NAME, desc, null);
    assertTrue(createdCube.equals(cubeMgr.getCube("new_cube_in_default")));

    //System.out.println(JsonUtil.writeValueAsIndentString(createdCube));

    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount + 1);

    CubeInstance droppedCube = cubeMgr.dropCube("new_cube_in_default", true);

    assertTrue(createdCube.equals(droppedCube));
    assertNull(cubeMgr.getCube("new_cube_in_default"));
    assertTrue(prjMgr.listAllProjects().size() == originalProjectCount);
    assertTrue(cubeMgr.listAllCubes().size() == originalCubeCount);
}
 
Example #23
Source File: CubeManager.java    From kylin with Apache License 2.0 5 votes vote down vote up
public CubeInstance createCube(CubeInstance cube, String projectName, String owner) throws IOException {
    try (AutoLock lock = cubeMapLock.lockForWrite()) {
        logger.info("Creating cube '{}-->{}' from instance object. '", projectName, cube.getName());

        // save cube resource
        cube.setOwner(owner);
        updateCubeWithRetry(new CubeUpdate(cube), 0);

        ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cube.getName(),
                projectName, owner);

        return cube;
    }
}
 
Example #24
Source File: CubeMetadataUpgrade.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public void verify() {
    logger.info("=================================================================");
    logger.info("The changes are applied, now it's time to verify the new metadata store by reloading all metadata:");
    logger.info("=================================================================");
    config.clearManagers();
    DataModelManager.getInstance(config);
    CubeDescManager.getInstance(config);
    CubeManager.getInstance(config);
    ProjectManager.getInstance(config);
    //cleanup();
}
 
Example #25
Source File: CubeService.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws UnknownHostException, IOException, JobException {
    final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
    if (!cubingJobs.isEmpty()) {
        throw new JobException("Cube schema shouldn't be changed with running job.");
    }

    try {
        if (!cube.getDescriptor().calculateSignature().equals(cube.getDescriptor().getSignature())) {
            this.releaseAllSegments(cube);
        }

        CubeDesc updatedCubeDesc = getCubeDescManager().updateCubeDesc(desc);

        int cuboidCount = CuboidCLI.simulateCuboidGeneration(updatedCubeDesc);
        logger.info("Updated cube " + cube.getName() + " has " + cuboidCount + " cuboids");

        ProjectManager projectManager = getProjectManager();
        if (!isCubeInProject(newProjectName, cube)) {
            String owner = SecurityContextHolder.getContext().getAuthentication().getName();
            ProjectInstance newProject = projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), newProjectName, owner);
            accessService.inherit(cube, newProject);
        }

        return updatedCubeDesc;
    } catch (IOException e) {
        throw new InternalErrorException("Failed to deal with the request.", e);
    }
}
 
Example #26
Source File: CubeManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public CubeInstance updateCubeStatus(CubeInstance cube, RealizationStatusEnum newStatus) throws IOException {
    try (AutoLock lock = cubeMapLock.lockForWrite()) {
        cube = cube.latestCopyForWrite(); // get a latest copy
        CubeUpdate update = new CubeUpdate(cube);
        update.setStatus(newStatus);
        ProjectManager.getInstance(config).touchProject(cube.getProject());
        return updateCube(update);
    }
}
 
Example #27
Source File: HBaseUsageExtractor.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Override
protected void executeExtract(OptionsHelper optionsHelper, File exportDir) throws Exception {
    if (cachedHMasterUrl == null) {
        return;
    }
    kylinConfig = KylinConfig.getInstanceFromEnv();
    cubeManager = CubeManager.getInstance(kylinConfig);
    realizationRegistry = RealizationRegistry.getInstance(kylinConfig);
    projectManager = ProjectManager.getInstance(kylinConfig);

    if (optionsHelper.hasOption(OPTION_PROJECT)) {
        String projectNames = optionsHelper.getOptionValue(OPTION_PROJECT);
        for (String projectName : StringUtil.splitByComma(projectNames)) {
            ProjectInstance projectInstance = projectManager.getProject(projectName);
            if (projectInstance == null) {
                throw new IllegalArgumentException("Project " + projectName + " does not exist");
            }
            List<RealizationEntry> realizationEntries = projectInstance.getRealizationEntries();
            for (RealizationEntry realizationEntry : realizationEntries) {
                retrieveResourcePath(getRealization(realizationEntry));
            }
        }
    } else if (optionsHelper.hasOption(OPTION_CUBE)) {
        String cubeNames = optionsHelper.getOptionValue(OPTION_CUBE);
        for (String cubeName : StringUtil.splitByComma(cubeNames)) {
            IRealization realization = cubeManager.getRealization(cubeName);
            if (realization != null) {
                retrieveResourcePath(realization);
            } else {
                throw new IllegalArgumentException("No cube found with name of " + cubeName);
            }
        }
    }

    extractCommonInfo(exportDir, kylinConfig);
    extractHTables(exportDir);
}
 
Example #28
Source File: CubeDescManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Override
public void onProjectSchemaChange(Broadcaster broadcaster, String project) throws IOException {
    for (IRealization real : ProjectManager.getInstance(config).listAllRealizations(project)) {
        if (real instanceof CubeInstance) {
            String descName = ((CubeInstance) real).getDescName();
            reloadCubeDescQuietly(descName);
        }
    }
}
 
Example #29
Source File: ProjectControllerTest.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testAddUpdateProject() throws IOException {

    List<ProjectInstance> projects = projectController.getProjects(null, null);

    int originalProjectCount = projects.size();
    CreateProjectRequest request = new CreateProjectRequest();
    request.setName("new_project");
    ProjectInstance ret = projectController.saveProject(request);

    Assert.assertEquals(ret.getOwner(), "ADMIN");
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);

    UpdateProjectRequest updateR = new UpdateProjectRequest();
    updateR.setFormerProjectName("new_project");
    updateR.setNewProjectName("new_project_2");
    projectController.updateProject(updateR);

    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null);

    Assert.assertNotEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project_2"), null);

    // only update desc:
    updateR = new UpdateProjectRequest();
    updateR.setFormerProjectName("new_project_2");
    updateR.setNewProjectName("new_project_2");
    updateR.setNewDescription("hello world");
    projectController.updateProject(updateR);

    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).listAllProjects().size(), originalProjectCount + 1);
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project"), null);
    Assert.assertNotEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project_2"), null);
    Assert.assertEquals(ProjectManager.getInstance(getTestConfig()).getProject("new_project_2").getDescription(), "hello world");
}
 
Example #30
Source File: KylinLogExtractor.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private void beforeExtract() {
    // reload metadata before extract diagnosis info
    logger.info("Start to reload metadata from diagnosis.");
    
    config.clearManagers();

    CubeManager.getInstance(config);
    CubeDescManager.getInstance(config);
    DataModelManager.getInstance(config);
    ProjectManager.getInstance(config);
}