org.apache.kylin.cube.CubeInstance Java Examples

void prepareTestCheckSegmentBuildJobFromMetadata() {
    CubeSegment cubeSegment = stubCubSegment(SegmentStatusEnum.NEW, 100L, 200L);
    CubeInstance cubeInstance = stubCubeInstance(cubeSegment);
    config = stubKylinConfig();
    when(cubeInstance.getConfig()).thenReturn(config);

    cubeManager = stubCubeManager(cubeInstance, false);

    Map<String, CubingJob> cubingJobMap = new HashMap<>();
    cubingJobMap.put(mockBuildJob1, stubCubingJob(ExecutableState.SUCCEED));
    cubingJobMap.put(mockBuildJob2, stubCubingJob(ExecutableState.DISCARDED));
    cubingJobMap.put(mockBuildJob3, stubCubingJob(ExecutableState.DISCARDED));
    cubingJobMap.put(mockBuildJob4, stubCubingJob(ExecutableState.ERROR));

    executableManager = stubExecutableManager(cubingJobMap);
    streamingCoordinator = stubStreamingCoordinator(config, cubeManager, executableManager);
    clusterManager = stubReceiverClusterManager(streamingCoordinator);
    when(streamingCoordinator.getClusterManager()).thenReturn(clusterManager);
}
 

public static void prepareTestDataForStreamingCube(long startTime, long endTime, int numberOfRecords,
        String cubeName, StreamDataLoader streamDataLoader) throws IOException {
    CubeInstance cubeInstance = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
    List<String> data = StreamingTableDataGenerator.generate(numberOfRecords, startTime, endTime,
            cubeInstance.getRootFactTable(), cubeInstance.getProject());
    //load into kafka
    streamDataLoader.loadIntoKafka(data);
    logger.info("Write {} messages into {}", data.size(), streamDataLoader.toString());

    //csv data for H2 use
    TableRef factTable = cubeInstance.getModel().getRootFactTable();
    List<TblColRef> tableColumns = Lists.newArrayList(factTable.getColumns());
    TimedJsonStreamParser timedJsonStreamParser = new TimedJsonStreamParser(tableColumns, null);
    StringBuilder sb = new StringBuilder();
    for (String json : data) {
        List<String> rowColumns = timedJsonStreamParser
                .parse(ByteBuffer.wrap(json.getBytes(StandardCharsets.UTF_8))).get(0).getData();
        sb.append(StringUtils.join(rowColumns, ","));
        sb.append(System.getProperty("line.separator"));
    }
    appendFactTableData(sb.toString(), cubeInstance.getRootFactTable());
}
 

CubeInstance stubCubeInstance(CubeSegment cubSegment) {
    CubeInstance cubeInstance = mock(CubeInstance.class);
    CubeSegment readySegment = stubCubSegment(SegmentStatusEnum.READY, 0L, 1L);
    when(cubeInstance.latestCopyForWrite()).thenReturn(cubeInstance);
    @SuppressWarnings("unchecked")
    Segments<CubeSegment> segmentSegments = mock(Segments.class, RETURNS_DEEP_STUBS);

    Segments<CubeSegment> optimizedSegments = mock(Segments.class, RETURNS_DEEP_STUBS);

    when(segmentSegments.size()).thenReturn(1);
    when(cubeInstance.getBuildingSegments()).thenReturn(segmentSegments);
    when(cubeInstance.getName()).thenReturn(cubeName1);
    when(cubeInstance.getSegment(anyString(), Matchers.any())).thenReturn(cubSegment);

    when(optimizedSegments.size()).thenReturn(0);
    when(cubeInstance.getLatestReadySegment()).thenReturn(readySegment);
    when(cubeInstance.getSegments(SegmentStatusEnum.READY_PENDING)).thenReturn(optimizedSegments);
    when(cubeInstance.getSegments(SegmentStatusEnum.NEW)).thenReturn(segmentSegments);

    return cubeInstance;
}
 

@Test
public void testBasics() throws IOException, PersistentException {
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc("test_kylin_cube_with_slr_left_join_desc");
    CubeInstance cube = cubeManager.createCube(CUBE_NAME, "DEFAULT", cubeDesc, "test");
    assertNotNull(cube);

    JobListRequest jobRequest = new JobListRequest();
    Assert.assertNotNull(jobSchedulerController.list(jobRequest));

    JobBuildRequest jobBuildRequest = new JobBuildRequest();
    jobBuildRequest.setBuildType("BUILD");
    jobBuildRequest.setStartTime(0L);
    jobBuildRequest.setEndTime(new Date().getTime());
    JobInstance job = cubeController.rebuild(CUBE_NAME, jobBuildRequest);

    Assert.assertNotNull(jobSchedulerController.get(job.getId()));
    executableDAO.deleteJob(job.getId());
    if (cubeManager.getCube(CUBE_NAME) != null) {
        cubeManager.dropCube(CUBE_NAME, false);
    }

    // jobSchedulerController.cancel(job.getId());
}
 

private void resetRealizationInContext(OLAPContext olapContext) {
    IRealization realization = olapContext.realization;
    if (realization == null) {
        return;
    }
    KylinConfig config = getConfig();
    HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(realization.getName());
    if (hybridInstance != null) {
        olapContext.realization = hybridInstance;
        return;
    }
    CubeInstance cubeInstance = CubeManager.getInstance(config).getCube(realization.getName());
    if (cubeInstance != null) {
        olapContext.realization = cubeInstance;
    }
}
 

private static LookupSnapshotBuildJob initJob(CubeInstance cube, String tableName, String submitter,
        KylinConfig kylinConfig) {
    List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),
            cube.getName());
    if (projList == null || projList.size() == 0) {
        throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
    } else if (projList.size() >= 2) {
        String msg = "Find more than one project containing the cube " + cube.getName()
                + ". It does't meet the uniqueness requirement!!! ";
        throw new RuntimeException(msg);
    }

    LookupSnapshotBuildJob result = new LookupSnapshotBuildJob();
    SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    format.setTimeZone(TimeZone.getTimeZone(kylinConfig.getTimeZone()));
    result.setDeployEnvName(kylinConfig.getDeployEnv());
    result.setProjectName(projList.get(0).getName());
    CubingExecutableUtil.setCubeName(cube.getName(), result.getParams());
    result.setName(JOB_TYPE + " CUBE - " + cube.getName() + " - " + " TABLE - " + tableName + " - "
            + format.format(new Date(System.currentTimeMillis())));
    result.setSubmitter(submitter);
    result.setNotifyList(cube.getDescriptor().getNotifyList());
    return result;
}
 

public static Set<String> collectCubeMetadata(CubeInstance cube) {
    // cube, model_desc, cube_desc, table
    TableMetadataManager tableMgr = TableMetadataManager.getInstance(cube.getConfig());
    Set<String> dumpList = new LinkedHashSet<>();
    dumpList.add(cube.getResourcePath());
    dumpList.add(cube.getDescriptor().getModel().getResourcePath());
    dumpList.add(cube.getDescriptor().getResourcePath());
    dumpList.add(cube.getProjectInstance().getResourcePath());

    for (TableRef tableRef : cube.getDescriptor().getModel().getAllTables()) {
        TableDesc table = tableRef.getTableDesc();
        dumpList.add(table.getResourcePath());
        dumpList.add(tableMgr.getTableExt(table).getResourcePath());
    }

    return dumpList;
}
 

private boolean isCubeInProject(String projectName, CubeInstance target) {
    ProjectManager projectManager = getProjectManager();
    ProjectInstance project = projectManager.getProject(projectName);
    if (project == null) {
        return false;
    }
    for (RealizationEntry projectDataModel : project.getRealizationEntries()) {
        if (projectDataModel.getType() == RealizationType.CUBE) {
            CubeInstance cube = getCubeManager().getCube(projectDataModel.getRealization());
            assert cube != null;
            if (cube.equals(target)) {
                return true;
            }
        }
    }
    return false;
}
 

@Override
protected void doSetup(Context context) throws IOException, InterruptedException {
    super.bindCurrentConfiguration(context.getConfiguration());

    String cubeName = context.getConfiguration().get(BatchConstants.CFG_CUBE_NAME);
    String segmentID = context.getConfiguration().get(BatchConstants.CFG_CUBE_SEGMENT_ID);

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata();

    CubeManager cubeManager = CubeManager.getInstance(config);
    CubeInstance cube = cubeManager.getCube(cubeName);
    CubeDesc cubeDesc = cube.getDescriptor();
    CubeSegment mergedCubeSegment = cube.getSegmentById(segmentID);

    // decide which source segment
    FileSplit fileSplit = (FileSplit) context.getInputSplit();
    IMROutput2.IMRMergeOutputFormat outputFormat = MRUtil.getBatchMergeOutputSide2(mergedCubeSegment)
            .getOutputFormat();
    CubeSegment sourceCubeSegment = outputFormat.findSourceSegment(fileSplit, cube);
    reEncoder = new SegmentReEncoder(cubeDesc, sourceCubeSegment, mergedCubeSegment, config);
}
 

public static long estimatedCubeSize(String cubeName, long[] cardinality) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    CubeManager cubeManager = CubeManager.getInstance(config);
    CubeInstance cubeInstance = cubeManager.getCube(cubeName);
    CubeDesc cubeDesc = cubeInstance.getDescriptor();

    CuboidScheduler scheduler = new CuboidScheduler(cubeDesc);
    long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
    LinkedList<Long> cuboidQueue = new LinkedList<Long>();
    cuboidQueue.push(baseCuboid);

    long totalSpace = 0;

    while (!cuboidQueue.isEmpty()) {
        long cuboidID = cuboidQueue.pop();
        Collection<Long> spanningCuboid = scheduler.getSpanningCuboid(cuboidID);
        for (Long sc : spanningCuboid) {
            cuboidQueue.push(sc);
        }

        totalSpace += estimateCuboidSpace(cuboidID, cardinality, cubeDesc);
    }
    return totalSpace;
}
 

@Override
public void open(Configuration parameters) throws Exception {
    KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
            .setAndUnsetThreadLocalConfig(kConfig)) {
        CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
        CubeDesc cubeDesc = cubeInstance.getDescriptor();
        CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
        CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich(
                EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
        Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
        baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc,
                AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid),
                MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
    }
}
 

void prepareTestCheckSegmentBuildJobFromMetadata() {
    CubeSegment cubeSegment = stubCubSegment(SegmentStatusEnum.NEW, 100L, 200L);
    CubeInstance cubeInstance = stubCubeInstance(cubeSegment);
    config = stubKylinConfig();
    when(cubeInstance.getConfig()).thenReturn(config);

    cubeManager = stubCubeManager(cubeInstance, false);

    Map<String, CubingJob> cubingJobMap = new HashMap<>();
    cubingJobMap.put(mockBuildJob1, stubCubingJob(ExecutableState.SUCCEED));
    cubingJobMap.put(mockBuildJob2, stubCubingJob(ExecutableState.DISCARDED));
    cubingJobMap.put(mockBuildJob3, stubCubingJob(ExecutableState.DISCARDED));
    cubingJobMap.put(mockBuildJob4, stubCubingJob(ExecutableState.ERROR));

    executableManager = stubExecutableManager(cubingJobMap);
    streamingCoordinator = stubStreamingCoordinator(config, cubeManager, executableManager);
    clusterManager = stubReceiverClusterManager(streamingCoordinator);
    when(streamingCoordinator.getClusterManager()).thenReturn(clusterManager);
}
 

private static List<String> filterByCubes(List<String> allTableNames, List<String> cubeNames) {
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    List<String> result = Lists.newArrayList();
    for (String c : cubeNames) {
        c = c.trim();
        if (c.endsWith(","))
            c = c.substring(0, c.length() - 1);

        CubeInstance cubeInstance = cubeManager.getCube(c);
        for (CubeSegment segment : cubeInstance.getSegments()) {
            String tableName = segment.getStorageLocationIdentifier();
            if (allTableNames.contains(tableName)) {
                result.add(tableName);
            }
        }
    }
    return result;
}
 

private Map<String, String> getCubeFilterMap(CategoryEnum category, String cubeName) {
    HashMap<String, String> filterMap = new HashMap<>();

    if (category == CategoryEnum.QUERY) {
        filterMap.put(QueryPropertyEnum.EXCEPTION.toString() + " = ?", "NULL");

        if (!Strings.isNullOrEmpty(cubeName)) {
            filterMap.put(QueryPropertyEnum.REALIZATION + " = ?", cubeName);
        }
    } else if (category == CategoryEnum.JOB && !Strings.isNullOrEmpty(cubeName)) {
        HybridInstance hybridInstance = getHybridManager().getHybridInstance(cubeName);
        if (null != hybridInstance) {
            StringBuffer cubeNames = new StringBuffer();
            for (CubeInstance cube : getCubeByHybrid(hybridInstance)) {
                cubeNames.append(",'" + cube.getName() + "'");
            }
            filterMap.put(JobPropertyEnum.CUBE.toString() + " IN (?)", cubeNames.substring(1));
        } else {
            filterMap.put(JobPropertyEnum.CUBE.toString() + " = ?", cubeName);
        }
    }
    return filterMap;
}
 

/**
 * Delete a cube segment
 *
 * @throws IOException
 */
@RequestMapping(value = "/{cubeName}/segs/{segmentName}", method = { RequestMethod.DELETE }, produces = {
        "application/json" })
@ResponseBody
public CubeInstance deleteSegment(@PathVariable String cubeName, @PathVariable String segmentName) {
    checkCubeExists(cubeName);
    CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);

    CubeSegment segment = cube.getSegment(segmentName, null);
    if (segment == null) {
        throw new NotFoundException("Cannot find segment '" + segmentName + "'");
    }

    try {
        return cubeService.deleteSegment(cube, segmentName);
    } catch (Exception e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e.getLocalizedMessage(), e);
    }
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
    final CubeInstance cubeCopy = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite();
    final String segmentId = CubingExecutableUtil.getSegmentId(this.getParams());
    final CubeSegment segCopy = cubeCopy.getSegmentById(segmentId);

    Preconditions.checkNotNull(segCopy, "Cube segment '" + segmentId + "' not found.");
    Segments<CubeSegment> mergingSegs = cubeCopy.getMergingSegments(segCopy);

    Preconditions.checkArgument(mergingSegs.size() > 0, "Merging segment not exist.");

    Collections.sort(mergingSegs);
    final CubeSegment first = mergingSegs.get(0);
    final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1);

    segCopy.setSegRange(new SegmentRange(first.getSegRange().start, last.getSegRange().end));
    segCopy.setSourcePartitionOffsetStart(first.getSourcePartitionOffsetStart());
    segCopy.setSourcePartitionOffsetEnd(last.getSourcePartitionOffsetEnd());

    segCopy.setTSRange(new TSRange(mergingSegs.getTSStart(), mergingSegs.getTSEnd()));

    CubeUpdate update = new CubeUpdate(cubeCopy);
    update.setToUpdateSegs(segCopy);
    try {
        cubeManager.updateCube(update);
        return ExecuteResult.createSucceed();
    } catch (IOException e) {
        logger.error("fail to update cube segment offset", e);
        return ExecuteResult.createError(e);
    }
}
 

public BatchOptimizeJobCheckpointBuilder(CubeInstance cube, String submitter) {
    this.cube = cube;
    this.submitter = submitter;

    Preconditions.checkNotNull(cube.getFirstSegment(), "Cube " + cube + " is empty!!!");
    this.outputSide = MRUtil.getBatchOptimizeOutputSide2(cube.getFirstSegment());
}
 

@Test
public void testWithoutSlr() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITHOUT_SLR_READY");

    RowKeySplitter rowKeySplitter = new RowKeySplitter(cube.getFirstSegment(), 10, 20);
    // base cuboid rowkey
    byte[] input = { 0, 0, 0, 0, 0, 0, 0, -1, 11, 55, -13, 13, 22, 34, 121, 70, 80, 45, 71, 84, 67, 9, 9, 9, 9, 9, 9, 0, 10, 5 };
    rowKeySplitter.split(input, input.length);

    assertEquals(9, rowKeySplitter.getBufferSize());
}
 

private void injest(File metaRoot) throws IOException {
    KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath());
    TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig);
    DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig);
    HybridManager srcHybridManager = HybridManager.getInstance(srcConfig);
    CubeManager srcCubeManager = CubeManager.getInstance(srcConfig);
    CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig);

    checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager);
    new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources));

    // clear the cache
    Broadcaster.getInstance(kylinConfig).notifyClearAll();
    
    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        logger.info("add " + tableDesc + " to " + targetProjectName);
        projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName);
    }

    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        logger.info("add " + cube + " to " + targetProjectName);
        projectManager.addModelToProject(cube.getModel().getName(), targetProjectName);
        projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null);
    }

}
 

public static RootPersistentEntity createAclEntity(String entityType, String uuid) {
    // Validate the uuid first, exception will be thrown if the uuid string is not a valid uuid
    UUID uuidObj = UUID.fromString(uuid);
    uuid = uuidObj.toString();

    if (CUBE_INSTANCE.equals(entityType)) {
        CubeInstance cubeInstance = new CubeInstance();
        cubeInstance.setUuid(uuid);

        return cubeInstance;
    }

    if (DATA_MODEL_DESC.equals(entityType)) {
        DataModelDesc modelInstance = new DataModelDesc();
        modelInstance.setUuid(uuid);

        return modelInstance;
    }

    if (JOB_INSTANCE.equals(entityType)) {
        JobInstance jobInstance = new JobInstance();
        jobInstance.setUuid(uuid);

        return jobInstance;
    }

    if (PROJECT_INSTANCE.equals(entityType)) {
        ProjectInstance projectInstance = new ProjectInstance();
        projectInstance.setUuid(uuid);

        return projectInstance;
    }

    throw new RuntimeException("Unsupported entity type!");
}
 

@RequestMapping(value = "/{cubeName}/disable", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public CubeInstance disableCube(@PathVariable String cubeName) {
    try {
        checkCubeExists(cubeName);
        CubeInstance cube = cubeService.getCubeManager().getCube(cubeName);

        return cubeService.disableCube(cube);
    } catch (Exception e) {
        String message = "Failed to disable cube: " + cubeName;
        logger.error(message, e);
        throw new InternalErrorException(message + " Caused by: " + e.getMessage(), e);
    }
}
 

private void checkCubeHoles(List<CubeInstance> cubes) {
    reporter.log("## Checking holes of Cubes");
    for (CubeInstance cube : cubes) {
        if (cube.isReady()) {
            List<CubeSegment> holes = cubeManager.calculateHoles(cube.getName());
            if (holes.size() > 0) {
                reporter.log("{} holes in cube: {}, project: {}", holes.size(), cube.getName(), cube.getProject());
            }
        }
    }
}
 

private static void optimizeSizeMap(Map<Long, Double> sizeMap, CubeSegment cubeSegment) {
    CubeInstance cubeInstance = cubeSegment.getCubeInstance();
    int totalLevels = cubeInstance.getCuboidScheduler().getBuildLevel();
    List<List<Long>> layeredCuboids = cubeInstance.getCuboidScheduler().getCuboidsByLayer();

    logger.info("cube size is {} before optimize", SumHelper.sumDouble(sizeMap.values()));

    List<Double> levelRating = getHistoricalRating(cubeSegment, cubeInstance, totalLevels);

    if (levelRating == null) {
        logger.info("Fail to optimize, use origin.");
        return;
    }

    for (int level = 0; level <= totalLevels; level++) {
        Double rate = levelRating.get(level);

        for (Long cuboidId : layeredCuboids.get(level)) {
            double oriValue = (sizeMap.get(cuboidId) == null ? 0.0 : sizeMap.get(cuboidId));
            sizeMap.put(cuboidId, oriValue * rate);
        }
    }

    logger.info("cube size is {} after optimize", SumHelper.sumDouble(sizeMap.values()));

    return;
}
 

private void saveExtSnapshotIfNeeded(CubeManager cubeManager, CubeInstance cube, CubeSegment segment)
        throws IOException {
    String extLookupSnapshotStr = this.getParam(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO);
    if (extLookupSnapshotStr == null || extLookupSnapshotStr.isEmpty()) {
        return;
    }
    Map<String, String> extLookupSnapshotMap = LookupMaterializeContext.parseLookupSnapshots(extLookupSnapshotStr);
    logger.info("update ext lookup snapshots:{}", extLookupSnapshotMap);
    List<SnapshotTableDesc> snapshotTableDescList = cube.getDescriptor().getSnapshotTableDescList();
    for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescList) {
        String tableName = snapshotTableDesc.getTableName();
        if (snapshotTableDesc.isExtSnapshotTable()) {
            String newSnapshotResPath = extLookupSnapshotMap.get(tableName);
            if (newSnapshotResPath == null || newSnapshotResPath.isEmpty()) {
                continue;
            }

            if (snapshotTableDesc.isGlobal()) {
                if (!newSnapshotResPath.equals(cube.getSnapshotResPath(tableName))) {
                    cubeManager.updateCubeLookupSnapshot(cube, tableName, newSnapshotResPath);
                }
            } else {
                segment.putSnapshotResPath(tableName, newSnapshotResPath);
            }
        }
    }
}
 

/** Trigger cube planner phase two for optimization */
public static Map<Long, Long> getRecommendCuboidList(CubeInstance cube, Map<Long, Long> hitFrequencyMap,
        Map<Long, Map<Long, Pair<Long, Long>>> rollingUpCountSourceMap) throws IOException {

    CuboidScheduler cuboidScheduler = cube.getCuboidScheduler();
    Set<Long> currentCuboids = cuboidScheduler.getAllCuboidIds();
    Pair<Map<Long, Long>, Map<Long, Double>> statsPair = CuboidStatsReaderUtil
            .readCuboidStatsAndSizeFromCube(currentCuboids, cube);
    long baseCuboid = cuboidScheduler.getBaseCuboidId();
    if (statsPair.getFirst().get(baseCuboid) == null || statsPair.getFirst().get(baseCuboid) == 0L) {
        logger.info(BASE_CUBOID_COUNT_IN_CUBOID_STATISTICS_IS_ZERO);
        return null;
    }

    KylinConfig config = cube.getConfig();
    String key = cube.getName();
    double queryUncertaintyRatio = config.getCubePlannerQueryUncertaintyRatio();
    double bpusMinBenefitRatio = config.getCubePlannerBPUSMinBenefitRatio();
    CuboidStats cuboidStats = new CuboidStats.Builder(key, baseCuboid, statsPair.getFirst(),
            statsPair.getSecond()) {
        @Override
        public Map<Long, Double> estimateCuboidsSize(Map<Long, Long> statistics) {
            try {
                return CuboidStatsReaderUtil.readCuboidSizeFromCube(statistics, cube);
            } catch (IOException e) {
                logger.warn("Fail to get cuboid size from cube due to ", e);
                return null;
            }
        }
    }.setQueryUncertaintyRatio(queryUncertaintyRatio) //
            .setBPUSMinBenefitRatio(bpusMinBenefitRatio) //
            .setHitFrequencyMap(hitFrequencyMap) //
            .setRollingUpCountSourceMap(rollingUpCountSourceMap) //
            .build();
    return CuboidRecommender.getInstance().getRecommendCuboidList(cuboidStats, config);
}
 

private JobInstance buildInternal(String cubeName, TSRange tsRange, SegmentRange segRange, //
        Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd,
        String buildType, boolean force, Integer priorityOffset) {
    try {
        String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
        CubeInstance cube = jobService.getCubeManager().getCube(cubeName);

        checkBuildingSegment(cube);
        return jobService.submitJob(cube, tsRange, segRange, sourcePartitionOffsetStart, sourcePartitionOffsetEnd,
                CubeBuildTypeEnum.valueOf(buildType), force, submitter, priorityOffset);
    } catch (Throwable e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e.getLocalizedMessage(), e);
    }
}
 

private List<CubeInstance> findCubeByModel(final String modelName) {
    Iterable<CubeInstance> relatedCubes = Iterables.filter(cubeManager.listAllCubes(), cube -> {
        if (cube == null || cube.allowBrokenDescriptor()) {
            return false;
        }
        DataModelDesc model = cube.getModel();
        if (model == null)
            return false;
        return model.getName().equals(modelName);
    });

    return ImmutableList.copyOf(relatedCubes);
}
 

@Test
public void testGTInfo() {
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube("test_kylin_cube_with_slr_ready");
    CubeSegment segment = cube.getFirstSegment();

    Cuboid baseCuboid = Cuboid.getBaseCuboid(cube.getDescriptor());
    GTInfo info = CubeGridTable.newGTInfo(baseCuboid, new CubeDimEncMap(segment));
    GTInfo.serializer.serialize(info, buffer);
    buffer.flip();

    GTInfo sInfo = GTInfo.serializer.deserialize(buffer);
    this.compareTwoGTInfo(info, sInfo);
}
 

private void saveExtSnapshotIfNeeded(CubeManager cubeManager, CubeInstance cube, CubeSegment segment)
        throws IOException {
    String extLookupSnapshotStr = this.getParam(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO);
    if (extLookupSnapshotStr == null || extLookupSnapshotStr.isEmpty()) {
        return;
    }
    Map<String, String> extLookupSnapshotMap = LookupMaterializeContext.parseLookupSnapshots(extLookupSnapshotStr);
    logger.info("update ext lookup snapshots:{}", extLookupSnapshotMap);
    List<SnapshotTableDesc> snapshotTableDescList = cube.getDescriptor().getSnapshotTableDescList();
    for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescList) {
        String tableName = snapshotTableDesc.getTableName();
        if (snapshotTableDesc.isExtSnapshotTable()) {
            String newSnapshotResPath = extLookupSnapshotMap.get(tableName);
            if (newSnapshotResPath == null || newSnapshotResPath.isEmpty()) {
                continue;
            }

            if (snapshotTableDesc.isGlobal()) {
                if (!newSnapshotResPath.equals(cube.getSnapshotResPath(tableName))) {
                    cubeManager.updateCubeLookupSnapshot(cube, tableName, newSnapshotResPath);
                }
            } else {
                segment.putSnapshotResPath(tableName, newSnapshotResPath);
            }
        }
    }
}
 

public static JobSearchResult parseToJobSearchResult(DefaultChainedExecutable job, Map<String, ExecutableOutputPO> outputs) {
    if (job == null) {
        logger.warn("job is null.");
        return null;
    }

    ExecutableOutputPO output = outputs.get(job.getId());
    if (output == null) {
        logger.warn("job output is null.");
        return null;
    }

    final JobSearchResult result = new JobSearchResult();

    String cubeName = CubingExecutableUtil.getCubeName(job.getParams());

    if (cubeName == null) {
        cubeName = job.getParam("model_name");
    } else {
        CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
        if (cube != null) {
            cubeName = cube.getDisplayName();
        }
    }
    result.setCubeName(cubeName);
    result.setId(job.getId());
    result.setJobName(job.getName());
    result.setLastModified(output.getLastModified());
    result.setJobStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
    return result;
}