Java Code Examples for org.apache.kylin.job.execution.ExecuteResult#createSucceed()

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    String cubeId = getParam(MetadataConstants.P_CUBE_ID);
    String[] segments = StringUtils.split(getParam(MetadataConstants.P_SEGMENT_NAMES), ",");
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    CubeInstance cube = CubeManager.getInstance(config).getCubeByUuid(cubeId);

    updateMetadataAfterMerge(cubeId);

    for (String segmentName : segments) {
        String path = config.getHdfsWorkingDirectory() + cube.getProject() + "/parquet/" + cube.getName() + "/" + segmentName;
        try {
            HadoopUtil.deletePath(HadoopUtil.getCurrentConfiguration(), new Path(path));
        } catch (IOException e) {
            throw new ExecuteException("Can not delete segment: " + segmentName + ", in cube: " + cube.getName());
        }
    }

    return ExecuteResult.createSucceed();
}
 

private ExecuteResult runSparkSubmit(KylinConfig config, String hadoopConf, String jars,
        String kylinJobJar, String appArgs, String jobId) {
    PatternedLogger patternedLogger;
    if (config.isJobLogPrintEnabled()) {
        patternedLogger = new PatternedLogger(logger);
    } else {
        patternedLogger = new PatternedLogger(null);
    }

    try {
        String cmd = generateSparkCmd(config, hadoopConf, jars, kylinJobJar, appArgs);

        CliCommandExecutor exec = new CliCommandExecutor();
        Pair<Integer, String> result = exec.execute(cmd, patternedLogger, jobId);

        Map<String, String> extraInfo = makeExtraInfo(patternedLogger.getInfo());
        ExecuteResult ret = ExecuteResult.createSucceed(result.getSecond());
        ret.getExtraInfo().putAll(extraInfo);
        updateMetaAfterBuilding(config);
        return ret;
    } catch (Exception e) {
        return ExecuteResult.createError(e);
    }
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager mgr = CubeManager.getInstance(context.getConfig());
    final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
    final List<CubeSegment> mergingSegments = getMergingSegments(cube);
    KylinConfig conf = cube.getConfig();

    Collections.sort(mergingSegments);

    try {
        checkLookupSnapshotsMustIncremental(mergingSegments);

        // work on copy instead of cached objects
        CubeInstance cubeCopy = cube.latestCopyForWrite();
        CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid());
        
        makeDictForNewSegment(conf, cubeCopy, newSegCopy, mergingSegments);
        makeSnapshotForNewSegment(cubeCopy, newSegCopy, mergingSegments);

        CubeUpdate update = new CubeUpdate(cubeCopy);
        update.setToUpdateSegs(newSegCopy);
        mgr.updateCube(update);
        return ExecuteResult.createSucceed();
    } catch (IOException e) {
        logger.error("fail to merge dictionary or lookup snapshots", e);
        return ExecuteResult.createError(e);
    }
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    try {
        Thread.sleep(5000);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
    return ExecuteResult.createSucceed();
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    try {
        Thread.sleep(1000);
    } catch (InterruptedException e) {
    }
    return ExecuteResult.createSucceed();
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
    final CubeInstance cubeCopy = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite();
    final String segmentId = CubingExecutableUtil.getSegmentId(this.getParams());
    final CubeSegment segCopy = cubeCopy.getSegmentById(segmentId);

    Preconditions.checkNotNull(segCopy, "Cube segment '" + segmentId + "' not found.");
    Segments<CubeSegment> mergingSegs = cubeCopy.getMergingSegments(segCopy);

    Preconditions.checkArgument(mergingSegs.size() > 0, "Merging segment not exist.");

    Collections.sort(mergingSegs);
    final CubeSegment first = mergingSegs.get(0);
    final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1);

    segCopy.setSegRange(new SegmentRange(first.getSegRange().start, last.getSegRange().end));
    segCopy.setSourcePartitionOffsetStart(first.getSourcePartitionOffsetStart());
    segCopy.setSourcePartitionOffsetEnd(last.getSourcePartitionOffsetEnd());

    segCopy.setTSRange(new TSRange(mergingSegs.getTSStart(), mergingSegs.getTSEnd()));

    CubeUpdate update = new CubeUpdate(cubeCopy);
    update.setToUpdateSegs(segCopy);
    try {
        cubeManager.updateCube(update);
        return ExecuteResult.createSucceed();
    } catch (IOException e) {
        logger.error("fail to update cube segment offset", e);
        return ExecuteResult.createError(e);
    }
}
 

private ExecuteResult runLocalMode(String appArgs, KylinConfig config) {
    try {
        Class<? extends Object> appClz = ClassUtil.forName(getSparkSubmitClassName(), Object.class);
        appClz.getMethod("main", String[].class).invoke(null, (Object) new String[] { appArgs });
        updateMetaAfterBuilding(config);
        return ExecuteResult.createSucceed();
    } catch (Exception e) {
        return ExecuteResult.createError(e);
    }
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager mgr = CubeManager.getInstance(context.getConfig());
    final CubeInstance cube = mgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
    final List<CubeSegment> mergingSegments = getMergingSegments(cube);
    KylinConfig conf = cube.getConfig();

    Collections.sort(mergingSegments);

    try {
        checkLookupSnapshotsMustIncremental(mergingSegments);

        // work on copy instead of cached objects
        CubeInstance cubeCopy = cube.latestCopyForWrite();
        CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid());
        
        makeDictForNewSegment(conf, cubeCopy, newSegCopy, mergingSegments);
        makeSnapshotForNewSegment(cubeCopy, newSegCopy, mergingSegments);

        CubeUpdate update = new CubeUpdate(cubeCopy);
        update.setToUpdateSegs(newSegCopy);
        mgr.updateCube(update);
        return ExecuteResult.createSucceed();
    } catch (IOException e) {
        logger.error("fail to merge dictionary or lookup snapshots", e);
        return ExecuteResult.createError(e);
    }
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    try {
        Thread.sleep(5000);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
    return ExecuteResult.createSucceed();
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    try {
        Thread.sleep(1000);
    } catch (InterruptedException e) {
    }
    return ExecuteResult.createSucceed();
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
    final CubeInstance cubeCopy = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams())).latestCopyForWrite();
    final String segmentId = CubingExecutableUtil.getSegmentId(this.getParams());
    final CubeSegment segCopy = cubeCopy.getSegmentById(segmentId);

    Preconditions.checkNotNull(segCopy, "Cube segment '" + segmentId + "' not found.");
    Segments<CubeSegment> mergingSegs = cubeCopy.getMergingSegments(segCopy);

    Preconditions.checkArgument(mergingSegs.size() > 0, "Merging segment not exist.");

    Collections.sort(mergingSegs);
    final CubeSegment first = mergingSegs.get(0);
    final CubeSegment last = mergingSegs.get(mergingSegs.size() - 1);

    segCopy.setSegRange(new SegmentRange(first.getSegRange().start, last.getSegRange().end));
    segCopy.setSourcePartitionOffsetStart(first.getSourcePartitionOffsetStart());
    segCopy.setSourcePartitionOffsetEnd(last.getSourcePartitionOffsetEnd());

    segCopy.setTSRange(new TSRange(mergingSegs.getTSStart(), mergingSegs.getTSEnd()));

    CubeUpdate update = new CubeUpdate(cubeCopy);
    update.setToUpdateSegs(segCopy);
    try {
        cubeManager.updateCube(update);
        return ExecuteResult.createSucceed();
    } catch (IOException e) {
        logger.error("fail to update cube segment offset", e);
        return ExecuteResult.createError(e);
    }
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeMgr = CubeManager.getInstance(context.getConfig());
    final DictionaryManager dictMgrHdfs;
    final DictionaryManager dictMgrHbase;
    final CubeInstance cube = cubeMgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
    final List<CubeSegment> mergingSegments = getMergingSegments(cube);
    final String dictInfoPath = this.getParams().get(BatchConstants.ARG_DICT_PATH);
    final String metadataUrl = this.getParams().get(BatchConstants.ARG_META_URL);

    final KylinConfig kylinConfHbase = cube.getConfig();
    final KylinConfig kylinConfHdfs = AbstractHadoopJob.loadKylinConfigFromHdfs(metadataUrl);

    Collections.sort(mergingSegments);

    try {
        Configuration conf = HadoopUtil.getCurrentConfiguration();
        FileSystem fs = HadoopUtil.getWorkingFileSystem();
        ResourceStore hbaseRS = ResourceStore.getStore(kylinConfHbase);
        ResourceStore hdfsRS = ResourceStore.getStore(kylinConfHdfs);
        dictMgrHdfs = DictionaryManager.getInstance(kylinConfHdfs);
        dictMgrHbase = DictionaryManager.getInstance(kylinConfHbase);

        // work on copy instead of cached objects
        CubeInstance cubeCopy = cube.latestCopyForWrite();
        CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid());

        // update cube segment dictionary

        FileStatus[] fileStatuss = fs.listStatus(new Path(dictInfoPath), new PathFilter() {
            @Override
            public boolean accept(Path path) {
                return path.getName().startsWith("part") || path.getName().startsWith("tmp");
            }
        });

        for (FileStatus fileStatus : fileStatuss) {
            Path filePath = fileStatus.getPath();

            SequenceFile.Reader reader = new SequenceFile.Reader(fs, filePath, conf);
            Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
            Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf);

            while (reader.next(key, value)) {
                String tblCol = key.toString();
                String dictInfoResource = value.toString();

                if (StringUtils.isNotEmpty(dictInfoResource)) {
                    logger.info(dictInfoResource);
                    // put dictionary file to metadata store
                    DictionaryInfo dictInfoHdfs = dictMgrHdfs.getDictionaryInfo(dictInfoResource);
                    DictionaryInfo dicInfoHbase = dictMgrHbase.trySaveNewDict(dictInfoHdfs.getDictionaryObject(), dictInfoHdfs);

                    if (dicInfoHbase != null){
                        TblColRef tblColRef = cube.getDescriptor().findColumnRef(tblCol.split(":")[0], tblCol.split(":")[1]);
                        newSegCopy.putDictResPath(tblColRef, dicInfoHbase.getResourcePath());
                    }
                }
            }

            IOUtils.closeStream(reader);
        }

        CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
        for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
            newSegCopy.putSnapshotResPath(entry.getKey(), entry.getValue());
        }

        // update statistics
        // put the statistics to metadata store
        String statisticsFileName = newSegment.getStatisticsResourcePath();
        hbaseRS.putResource(statisticsFileName, hdfsRS.getResource(newSegment.getStatisticsResourcePath()).content(), System.currentTimeMillis());

        CubeUpdate update = new CubeUpdate(cubeCopy);
        update.setToUpdateSegs(newSegCopy);
        cubeMgr.updateCube(update);

        return ExecuteResult.createSucceed();
    } catch (IOException e) {
        logger.error("fail to merge dictionary", e);
        return ExecuteResult.createError(e);
    }
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    CubeSegment newSegment = CubingExecutableUtil.findSegment(context,
            CubingExecutableUtil.getCubeName(this.getParams()),
            CubingExecutableUtil.getSegmentId(this.getParams()));
    KylinConfig kylinConf = newSegment.getConfig();

    ResourceStore rs = ResourceStore.getStore(kylinConf);
    try {

        FileSystem fs = HadoopUtil.getWorkingFileSystem();
        Configuration hadoopConf = HadoopUtil.getCurrentConfiguration();
        Path statisticsDir = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams()));
        Path[] statisticsFiles = HadoopUtil.getFilteredPath(fs, statisticsDir,
                BatchConstants.CFG_OUTPUT_STATISTICS);
        if (statisticsFiles == null) {
            throw new IOException("fail to find the statistics file in base dir: " + statisticsDir);
        }

        Map<Long, HLLCounter> cuboidHLLMap = Maps.newHashMap();
        long totalRowsBeforeMerge = 0;
        long grantTotal = 0;
        int samplingPercentage = -1;
        int mapperNumber = -1;
        for (Path item : statisticsFiles) {
            CubeStatsReader.CubeStatsResult cubeStatsResult = new CubeStatsReader.CubeStatsResult(item,
                    kylinConf.getCubeStatsHLLPrecision());
            cuboidHLLMap.putAll(cubeStatsResult.getCounterMap());
            long pGrantTotal = 0L;
            for (HLLCounter hll : cubeStatsResult.getCounterMap().values()) {
                pGrantTotal += hll.getCountEstimate();
            }
            totalRowsBeforeMerge += pGrantTotal * cubeStatsResult.getMapperOverlapRatio();
            grantTotal += pGrantTotal;
            int pMapperNumber = cubeStatsResult.getMapperNumber();
            if (pMapperNumber > 0) {
                if (mapperNumber < 0) {
                    mapperNumber = pMapperNumber;
                } else {
                    throw new RuntimeException(
                            "Base cuboid has been distributed to multiple reducers at step FactDistinctColumnsReducer!!!");
                }
            }
            int pSamplingPercentage = cubeStatsResult.getPercentage();
            if (samplingPercentage < 0) {
                samplingPercentage = pSamplingPercentage;
            } else if (samplingPercentage != pSamplingPercentage) {
                throw new RuntimeException(
                        "The sampling percentage should be same among all of the reducer of FactDistinctColumnsReducer!!!");
            }
        }
        if (samplingPercentage < 0) {
            logger.warn("The sampling percentage should be set!!!");
        }
        if (mapperNumber < 0) {
            logger.warn("The mapper number should be set!!!");
        }

        if (logger.isDebugEnabled()) {
            logMapperAndCuboidStatistics(cuboidHLLMap, samplingPercentage, mapperNumber, grantTotal,
                    totalRowsBeforeMerge);
        }
        double mapperOverlapRatio = grantTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grantTotal;
        CubingJob cubingJob = (CubingJob) getManager()
                .getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
        long sourceRecordCount = cubingJob.findSourceRecordCount();
        CubeStatsWriter.writeCuboidStatistics(hadoopConf, statisticsDir, cuboidHLLMap, samplingPercentage,
                mapperNumber, mapperOverlapRatio, sourceRecordCount);

        Path statisticsFile = new Path(statisticsDir, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
        logger.info("{} stats saved to hdfs {}", newSegment, statisticsFile);

        FSDataInputStream is = fs.open(statisticsFile);
        try {
            // put the statistics to metadata store
            String resPath = newSegment.getStatisticsResourcePath();
            rs.putResource(resPath, is, System.currentTimeMillis());
            logger.info("{} stats saved to resource {}", newSegment, resPath);

            StatisticsDecisionUtil.decideCubingAlgorithm(cubingJob, newSegment);
            StatisticsDecisionUtil.optimizeCubingPlan(newSegment);
        } finally {
            IOUtils.closeStream(is);
        }

        return ExecuteResult.createSucceed();
    } catch (IOException e) {
        logger.error("fail to save cuboid statistics", e);
        return ExecuteResult.createError(e);
    }
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeMgr = CubeManager.getInstance(context.getConfig());
    final DictionaryManager dictMgrHdfs;
    final DictionaryManager dictMgrHbase;
    final CubeInstance cube = cubeMgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
    final List<CubeSegment> mergingSegments = getMergingSegments(cube);
    final String dictInfoPath = this.getParams().get(BatchConstants.ARG_DICT_PATH);
    final String metadataUrl = this.getParams().get(BatchConstants.ARG_META_URL);

    final KylinConfig kylinConfHbase = cube.getConfig();
    final KylinConfig kylinConfHdfs = AbstractHadoopJob.loadKylinConfigFromHdfs(metadataUrl);

    Collections.sort(mergingSegments);

    try {
        Configuration conf = HadoopUtil.getCurrentConfiguration();
        FileSystem fs = HadoopUtil.getWorkingFileSystem();
        ResourceStore hbaseRS = ResourceStore.getStore(kylinConfHbase);
        ResourceStore hdfsRS = ResourceStore.getStore(kylinConfHdfs);
        dictMgrHdfs = DictionaryManager.getInstance(kylinConfHdfs);
        dictMgrHbase = DictionaryManager.getInstance(kylinConfHbase);

        // work on copy instead of cached objects
        CubeInstance cubeCopy = cube.latestCopyForWrite();
        CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid());

        // update cube segment dictionary

        FileStatus[] fileStatuss = fs.listStatus(new Path(dictInfoPath), new PathFilter() {
            @Override
            public boolean accept(Path path) {
                return path.getName().startsWith("part") || path.getName().startsWith("tmp");
            }
        });

        for (FileStatus fileStatus : fileStatuss) {
            Path filePath = fileStatus.getPath();

            SequenceFile.Reader reader = new SequenceFile.Reader(fs, filePath, conf);
            Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
            Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf);

            while (reader.next(key, value)) {
                String tblCol = key.toString();
                String dictInfoResource = value.toString();

                if (StringUtils.isNotEmpty(dictInfoResource)) {
                    logger.info(dictInfoResource);
                    // put dictionary file to metadata store
                    DictionaryInfo dictInfoHdfs = dictMgrHdfs.getDictionaryInfo(dictInfoResource);
                    DictionaryInfo dicInfoHbase = dictMgrHbase.trySaveNewDict(dictInfoHdfs.getDictionaryObject(), dictInfoHdfs);

                    if (dicInfoHbase != null){
                        TblColRef tblColRef = cube.getDescriptor().findColumnRef(tblCol.split(":")[0], tblCol.split(":")[1]);
                        newSegCopy.putDictResPath(tblColRef, dicInfoHbase.getResourcePath());
                    }
                }
            }

            IOUtils.closeStream(reader);
        }

        CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
        for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
            newSegCopy.putSnapshotResPath(entry.getKey(), entry.getValue());
        }

        // update statistics
        // put the statistics to metadata store
        String statisticsFileName = newSegment.getStatisticsResourcePath();
        hbaseRS.putResource(statisticsFileName, hdfsRS.getResource(newSegment.getStatisticsResourcePath()).content(), System.currentTimeMillis());

        CubeUpdate update = new CubeUpdate(cubeCopy);
        update.setToUpdateSegs(newSegCopy);
        cubeMgr.updateCube(update);

        return ExecuteResult.createSucceed();
    } catch (IOException e) {
        logger.error("fail to merge dictionary", e);
        return ExecuteResult.createError(e);
    }
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    CubeSegment newSegment = CubingExecutableUtil.findSegment(context,
            CubingExecutableUtil.getCubeName(this.getParams()),
            CubingExecutableUtil.getSegmentId(this.getParams()));
    KylinConfig kylinConf = newSegment.getConfig();

    ResourceStore rs = ResourceStore.getStore(kylinConf);
    try {

        FileSystem fs = HadoopUtil.getWorkingFileSystem();
        Configuration hadoopConf = HadoopUtil.getCurrentConfiguration();
        Path statisticsDir = new Path(CubingExecutableUtil.getStatisticsPath(this.getParams()));
        Path[] statisticsFiles = HadoopUtil.getFilteredPath(fs, statisticsDir,
                BatchConstants.CFG_OUTPUT_STATISTICS);
        if (statisticsFiles == null) {
            throw new IOException("fail to find the statistics file in base dir: " + statisticsDir);
        }

        Map<Long, HLLCounter> cuboidHLLMap = Maps.newHashMap();
        long totalRowsBeforeMerge = 0;
        long grantTotal = 0;
        int samplingPercentage = -1;
        int mapperNumber = -1;
        for (Path item : statisticsFiles) {
            CubeStatsReader.CubeStatsResult cubeStatsResult = new CubeStatsReader.CubeStatsResult(item,
                    kylinConf.getCubeStatsHLLPrecision());
            cuboidHLLMap.putAll(cubeStatsResult.getCounterMap());
            long pGrantTotal = 0L;
            for (HLLCounter hll : cubeStatsResult.getCounterMap().values()) {
                pGrantTotal += hll.getCountEstimate();
            }
            totalRowsBeforeMerge += pGrantTotal * cubeStatsResult.getMapperOverlapRatio();
            grantTotal += pGrantTotal;
            int pMapperNumber = cubeStatsResult.getMapperNumber();
            if (pMapperNumber > 0) {
                if (mapperNumber < 0) {
                    mapperNumber = pMapperNumber;
                } else {
                    throw new RuntimeException(
                            "Base cuboid has been distributed to multiple reducers at step FactDistinctColumnsReducer!!!");
                }
            }
            int pSamplingPercentage = cubeStatsResult.getPercentage();
            if (samplingPercentage < 0) {
                samplingPercentage = pSamplingPercentage;
            } else if (samplingPercentage != pSamplingPercentage) {
                throw new RuntimeException(
                        "The sampling percentage should be same among all of the reducer of FactDistinctColumnsReducer!!!");
            }
        }
        if (samplingPercentage < 0) {
            logger.warn("The sampling percentage should be set!!!");
        }
        if (mapperNumber < 0) {
            logger.warn("The mapper number should be set!!!");
        }

        if (logger.isDebugEnabled()) {
            logMapperAndCuboidStatistics(cuboidHLLMap, samplingPercentage, mapperNumber, grantTotal,
                    totalRowsBeforeMerge);
        }
        double mapperOverlapRatio = grantTotal == 0 ? 0 : (double) totalRowsBeforeMerge / grantTotal;
        CubingJob cubingJob = (CubingJob) getManager()
                .getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
        long sourceRecordCount = cubingJob.findSourceRecordCount();
        CubeStatsWriter.writeCuboidStatistics(hadoopConf, statisticsDir, cuboidHLLMap, samplingPercentage,
                mapperNumber, mapperOverlapRatio, sourceRecordCount);

        Path statisticsFile = new Path(statisticsDir, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION_FILENAME);
        logger.info("{} stats saved to hdfs {}", newSegment, statisticsFile);

        FSDataInputStream is = fs.open(statisticsFile);
        try {
            // put the statistics to metadata store
            String resPath = newSegment.getStatisticsResourcePath();
            rs.putResource(resPath, is, System.currentTimeMillis());
            logger.info("{} stats saved to resource {}", newSegment, resPath);

            StatisticsDecisionUtil.decideCubingAlgorithm(cubingJob, newSegment);
            StatisticsDecisionUtil.optimizeCubingPlan(newSegment);
        } finally {
            IOUtils.closeStream(is);
        }

        return ExecuteResult.createSucceed();
    } catch (IOException e) {
        logger.error("fail to save cuboid statistics", e);
        return ExecuteResult.createError(e);
    }
}