Java Code Examples for org.apache.kylin.cube.CubeSegment#setLastBuildJobID()

The following examples show how to use org.apache.kylin.cube.CubeSegment#setLastBuildJobID() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BuildCubeWithEngine.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private Boolean optimizeCube(String cubeName) throws Exception {
    CubeInstance cubeInstance = cubeManager.getCube(cubeName);
    Set<Long> cuboidsRecommend = mockRecommendCuboids(cubeInstance, 0.05, 255);
    CubeSegment[] optimizeSegments = cubeManager.optimizeSegments(cubeInstance, cuboidsRecommend);
    List<AbstractExecutable> optimizeJobList = Lists.newArrayListWithExpectedSize(optimizeSegments.length);
    for (CubeSegment optimizeSegment : optimizeSegments) {
        DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, "TEST");
        jobService.addJob(optimizeJob);
        optimizeJobList.add(optimizeJob);
        optimizeSegment.setLastBuildJobID(optimizeJob.getId());
    }
    CheckpointExecutable checkpointJob = new BatchOptimizeJobCheckpointBuilder(cubeInstance, "TEST").build();
    checkpointJob.addTaskListForCheck(optimizeJobList);
    jobService.addJob(checkpointJob);
    if (fastBuildMode) {
        return true;
    }
    ExecutableState state = waitForJob(checkpointJob.getId());
    return Boolean.valueOf(ExecutableState.SUCCEED == state);
}
Example 2
Source File: BuildCubeWithEngine.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private Boolean optimizeCube(String cubeName) throws Exception {
    CubeInstance cubeInstance = cubeManager.getCube(cubeName);
    Set<Long> cuboidsRecommend = mockRecommendCuboids(cubeInstance, 0.05, 255);
    CubeSegment[] optimizeSegments = cubeManager.optimizeSegments(cubeInstance, cuboidsRecommend);
    List<AbstractExecutable> optimizeJobList = Lists.newArrayListWithExpectedSize(optimizeSegments.length);
    for (CubeSegment optimizeSegment : optimizeSegments) {
        DefaultChainedExecutable optimizeJob = EngineFactory.createBatchOptimizeJob(optimizeSegment, "TEST");
        jobService.addJob(optimizeJob);
        optimizeJobList.add(optimizeJob);
        optimizeSegment.setLastBuildJobID(optimizeJob.getId());
    }
    CheckpointExecutable checkpointJob = new BatchOptimizeJobCheckpointBuilder(cubeInstance, "TEST").build();
    checkpointJob.addTaskListForCheck(optimizeJobList);
    jobService.addJob(checkpointJob);
    if (fastBuildMode) {
        return true;
    }
    ExecutableState state = waitForJob(checkpointJob.getId());
    return Boolean.valueOf(ExecutableState.SUCCEED == state);
}
Example 3
Source File: CubeBuildJob.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private void updateSegmentInfo(String cubeId, SegmentInfo segmentInfo, long sourceRowCount) throws IOException {
    CubeInstance cubeInstance = cubeManager.getCubeByUuid(cubeId);
    CubeInstance cubeCopy = cubeInstance.latestCopyForWrite();
    CubeUpdate update = new CubeUpdate(cubeCopy);

    List<CubeSegment> cubeSegments = Lists.newArrayList();
    CubeSegment segment = cubeCopy.getSegmentById(segmentInfo.id());
    segment.setSizeKB(segmentInfo.getAllLayoutSize() / 1024);
    segment.setLastBuildTime(System.currentTimeMillis());
    segment.setLastBuildJobID(getParam(MetadataConstants.P_JOB_ID));
    segment.setInputRecords(sourceRowCount);
    segment.setSnapshots(new ConcurrentHashMap<>(segmentInfo.getSnapShot2JavaMap()));
    segment.setCuboidShardNums(cuboidShardNum);
    Map<String, String> additionalInfo = segment.getAdditionalInfo();
    additionalInfo.put("storageType", "" + IStorageAware.ID_PARQUET);
    segment.setAdditionalInfo(additionalInfo);
    cubeSegments.add(segment);
    update.setToUpdateSegs(cubeSegments.toArray(new CubeSegment[0]));
    cubeManager.updateCube(update);
}
Example 4
Source File: Coordinator.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private boolean triggerSegmentBuild(String cubeName, String segmentName) {
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeInstance cubeInstance = cubeManager.getCube(cubeName);
    try {
        Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
        logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
        CubeSegment newSeg = getCubeManager().appendSegment(cubeInstance,
                new TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
        DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg,
                "SYSTEM");
        getExecutableManager().addJob(executable);
        CubingJob cubingJob = (CubingJob) executable;
        newSeg.setLastBuildJobID(cubingJob.getId());

        SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, cubingJob.getId());
        jobStatusChecker.addSegmentBuildJob(segmentJobBuildInfo);
        SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
        state.setBuildStartTime(System.currentTimeMillis());
        state.setState(SegmentBuildState.BuildState.State.BUILDING);
        state.setJobId(cubingJob.getId());
        streamMetadataStore.updateSegmentBuildState(cubeName, segmentName, state);
        return true;
    } catch (Exception e) {
        logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
        return false;
    }
}
Example 5
Source File: UpdateCubeInfoAfterMergeStep.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeInstance cube = cubeManager.getCube(getCubeName());
    
    CubeSegment mergedSegment = cube.getSegmentById(getSegmentId());
    if (mergedSegment == null) {
        return new ExecuteResult(ExecuteResult.State.FAILED, "there is no segment with id:" + getSegmentId());
    }
    String cubeSizeString = executableManager.getOutput(getConvertToHfileStepId()).getExtra().get(ExecutableConstants.HDFS_BYTES_WRITTEN);
    Preconditions.checkState(StringUtils.isNotEmpty(cubeSizeString), "Can't get cube segment size.");
    long cubeSize = Long.parseLong(cubeSizeString) / 1024;

    // collect source statistics
    List<String> mergingSegmentIds = getMergingSegmentIds();
    if (mergingSegmentIds.isEmpty()) {
        return new ExecuteResult(ExecuteResult.State.FAILED, "there are no merging segments");
    }
    long sourceCount = 0L;
    long sourceSize = 0L;
    for (String id : mergingSegmentIds) {
        CubeSegment segment = cube.getSegmentById(id);
        sourceCount += segment.getInputRecords();
        sourceSize += segment.getInputRecordsSize();
    }
    
    // update segment info
    mergedSegment.setSizeKB(cubeSize);
    mergedSegment.setInputRecords(sourceCount);
    mergedSegment.setInputRecordsSize(sourceSize);
    mergedSegment.setLastBuildJobID(getCubingJobId());
    mergedSegment.setLastBuildTime(System.currentTimeMillis());
    
    try {
        cubeManager.promoteNewlyBuiltSegments(cube, mergedSegment);
        return new ExecuteResult(ExecuteResult.State.SUCCEED);
    } catch (IOException e) {
        logger.error("fail to update cube after merge", e);
        return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
    }
}
Example 6
Source File: JobService.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public void resubmitJob(JobInstance job) throws IOException {
    aclEvaluate.checkProjectOperationPermission(job);

    Coordinator coordinator = Coordinator.getInstance();
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    String cubeName = job.getRelatedCube();
    CubeInstance cubeInstance = cubeManager.getCube(cubeName);

    String segmentName = job.getRelatedSegmentName();
    try {
        Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
        logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
        CubeSegment newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
                new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));

        DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg, aclEvaluate.getCurrentUserName());
        coordinator.getExecutableManager().addJob(executable);
        CubingJob cubingJob = (CubingJob) executable;
        newSeg.setLastBuildJobID(cubingJob.getId());

        SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
        state.setBuildStartTime(System.currentTimeMillis());
        state.setState(SegmentBuildState.BuildState.State.BUILDING);
        state.setJobId(cubingJob.getId());
        coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
    } catch (Exception e) {
        logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
        throw e;
    }
}
Example 7
Source File: UpdateCubeInfoAfterBuildStep.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
    final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()))
            .latestCopyForWrite();
    final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));

    CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
    long sourceCount = cubingJob.findSourceRecordCount();
    long sourceSizeBytes = cubingJob.findSourceSizeBytes();
    long cubeSizeBytes = cubingJob.findCubeSizeBytes();

    KylinConfig config = KylinConfig.getInstanceFromEnv();
    List<Double> cuboidEstimateRatio = cubingJob.findEstimateRatio(segment, config);

    segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
    segment.setLastBuildTime(System.currentTimeMillis());
    segment.setSizeKB(cubeSizeBytes / 1024);
    segment.setInputRecords(sourceCount);
    segment.setInputRecordsSize(sourceSizeBytes);
    segment.setEstimateRatio(cuboidEstimateRatio);

    try {
        deleteDictionaryIfNeeded(segment);
        saveExtSnapshotIfNeeded(cubeManager, cube, segment);
        updateSegment(segment);

        cubeManager.promoteNewlyBuiltSegments(cube, segment);
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to update cube after build", e);
        return ExecuteResult.createError(e);
    }
}
Example 8
Source File: UpdateCubeInfoAfterOptimizeStep.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
    final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));

    CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(segment);
    long sourceCount = originalSegment.getInputRecords();
    long sourceSizeBytes = originalSegment.getInputRecordsSize();

    CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
    long cubeSizeBytes = cubingJob.findCubeSizeBytes();

    segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
    segment.setLastBuildTime(System.currentTimeMillis());
    segment.setSizeKB(cubeSizeBytes / 1024);
    segment.setInputRecords(sourceCount);
    segment.setInputRecordsSize(sourceSizeBytes);
    segment.setDimensionRangeInfoMap(originalSegment.getDimensionRangeInfoMap());

    try {
        cubeManager.promoteNewlyOptimizeSegments(cube, segment);
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to update cube after build", e);
        return ExecuteResult.createError(e);
    }
}
Example 9
Source File: ReceiverClusterManager.java    From kylin with Apache License 2.0 5 votes vote down vote up 
/**
 * Promote a segment from realtime part into historical part.
 */
void promoteNewSegment(CubingJob cubingJob, CubeInstance cubeInstance, CubeSegment cubeSegment) throws IOException {
    logger.debug("Try transfer segment's {} state to ready.", cubeSegment.getName());
    long sourceCount = cubingJob.findSourceRecordCount();
    long sourceSizeBytes = cubingJob.findSourceSizeBytes();
    long cubeSizeBytes = cubingJob.findCubeSizeBytes();
    Map<Integer, String> sourceCheckpoint = getCoordinator().getStreamMetadataStore()
            .getSourceCheckpoint(cubeInstance.getName(), cubeSegment.getName());

    ISourcePositionHandler positionOperator = StreamingSourceFactory.getStreamingSource(cubeInstance)
            .getSourcePositionHandler();
    Collection<ISourcePosition> sourcePositions = Collections2.transform(sourceCheckpoint.values(),
            new Function<String, ISourcePosition>() {
                @Nullable
                @Override
                public ISourcePosition apply(@Nullable String input) {
                    return positionOperator.parsePosition(input);
                }
            });
    ISourcePosition sourcePosition = positionOperator.mergePositions(sourcePositions,
            ISourcePositionHandler.MergeStrategy.KEEP_SMALL);
    cubeSegment.setLastBuildJobID(cubingJob.getId());
    cubeSegment.setLastBuildTime(System.currentTimeMillis());
    cubeSegment.setSizeKB(cubeSizeBytes / 1024);
    cubeSegment.setInputRecords(sourceCount);
    cubeSegment.setInputRecordsSize(sourceSizeBytes);
    cubeSegment.setStreamSourceCheckpoint(positionOperator.serializePosition(sourcePosition));
    getCoordinator().getCubeManager().promoteNewlyBuiltSegments(cubeInstance, cubeSegment);
}
Example 10
Source File: Coordinator.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private boolean triggerSegmentBuild(String cubeName, String segmentName) {
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeInstance cubeInstance = cubeManager.getCube(cubeName);
    try {
        Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
        logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
        CubeSegment newSeg = getCubeManager().appendSegment(cubeInstance,
                new TSRange(segmentRange.getFirst(), segmentRange.getSecond()));
        DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg,
                "SYSTEM");
        getExecutableManager().addJob(executable);
        CubingJob cubingJob = (CubingJob) executable;
        newSeg.setLastBuildJobID(cubingJob.getId());

        SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, cubingJob.getId());
        jobStatusChecker.addSegmentBuildJob(segmentJobBuildInfo);
        SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
        state.setBuildStartTime(System.currentTimeMillis());
        state.setState(SegmentBuildState.BuildState.State.BUILDING);
        state.setJobId(cubingJob.getId());
        streamMetadataStore.updateSegmentBuildState(cubeName, segmentName, state);
        return true;
    } catch (Exception e) {
        logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
        return false;
    }
}
Example 11
Source File: Coordinator.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void promoteNewSegment(CubingJob cubingJob, CubeInstance cubeInstance, CubeSegment cubeSegment)
        throws IOException {
    long sourceCount = cubingJob.findSourceRecordCount();
    long sourceSizeBytes = cubingJob.findSourceSizeBytes();
    long cubeSizeBytes = cubingJob.findCubeSizeBytes();
    Map<Integer, String> sourceCheckpoint = streamMetadataStore.getSourceCheckpoint(cubeInstance.getName(),
            cubeSegment.getName());

    ISourcePositionHandler positionOperator = StreamingSourceFactory.getStreamingSource(cubeInstance)
            .getSourcePositionHandler();
    Collection<ISourcePosition> sourcePositions = Collections2.transform(sourceCheckpoint.values(),
            new Function<String, ISourcePosition>() {
                @Nullable
                @Override
                public ISourcePosition apply(@Nullable String input) {
                    return positionOperator.parsePosition(input);
                }
            });
    ISourcePosition sourcePosition = positionOperator.mergePositions(sourcePositions, MergeStrategy.KEEP_SMALL);
    cubeSegment.setLastBuildJobID(cubingJob.getId());
    cubeSegment.setLastBuildTime(System.currentTimeMillis());
    cubeSegment.setSizeKB(cubeSizeBytes / 1024);
    cubeSegment.setInputRecords(sourceCount);
    cubeSegment.setInputRecordsSize(sourceSizeBytes);
    cubeSegment.setStreamSourceCheckpoint(positionOperator.serializePosition(sourcePosition));
    getCubeManager().promoteNewlyBuiltSegments(cubeInstance, cubeSegment);
}
Example 12
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public void resubmitJob(JobInstance job) throws IOException {
    aclEvaluate.checkProjectOperationPermission(job);

    Coordinator coordinator = Coordinator.getInstance();
    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    String cubeName = job.getRelatedCube();
    CubeInstance cubeInstance = cubeManager.getCube(cubeName);

    String segmentName = job.getRelatedSegmentName();
    try {
        Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
        logger.info("submit streaming segment build, cube:{} segment:{}", cubeName, segmentName);
        CubeSegment newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
                new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));

        DefaultChainedExecutable executable = new StreamingCubingEngine().createStreamingCubingJob(newSeg, aclEvaluate.getCurrentUserName());
        coordinator.getExecutableManager().addJob(executable);
        CubingJob cubingJob = (CubingJob) executable;
        newSeg.setLastBuildJobID(cubingJob.getId());

        SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
        state.setBuildStartTime(System.currentTimeMillis());
        state.setState(SegmentBuildState.BuildState.State.BUILDING);
        state.setJobId(cubingJob.getId());
        coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
    } catch (Exception e) {
        logger.error("streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
        throw e;
    }
}
Example 13
Source File: UpdateCubeInfoAfterBuildStep.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
    final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()))
            .latestCopyForWrite();
    final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));

    CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
    long sourceCount = cubingJob.findSourceRecordCount();
    long sourceSizeBytes = cubingJob.findSourceSizeBytes();
    long cubeSizeBytes = cubingJob.findCubeSizeBytes();

    KylinConfig config = KylinConfig.getInstanceFromEnv();
    List<Double> cuboidEstimateRatio = cubingJob.findEstimateRatio(segment, config);

    segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
    segment.setLastBuildTime(System.currentTimeMillis());
    segment.setSizeKB(cubeSizeBytes / 1024);
    segment.setInputRecords(sourceCount);
    segment.setInputRecordsSize(sourceSizeBytes);
    segment.setEstimateRatio(cuboidEstimateRatio);

    try {
        saveExtSnapshotIfNeeded(cubeManager, cube, segment);
        updateSegment(segment);

        cubeManager.promoteNewlyBuiltSegments(cube, segment);
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to update cube after build", e);
        return ExecuteResult.createError(e);
    }
}
Example 14
Source File: UpdateCubeInfoAfterOptimizeStep.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
    final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));

    CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(segment);
    long sourceCount = originalSegment.getInputRecords();
    long sourceSizeBytes = originalSegment.getInputRecordsSize();

    CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
    long cubeSizeBytes = cubingJob.findCubeSizeBytes();

    segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
    segment.setLastBuildTime(System.currentTimeMillis());
    segment.setSizeKB(cubeSizeBytes / 1024);
    segment.setInputRecords(sourceCount);
    segment.setInputRecordsSize(sourceSizeBytes);
    segment.setDimensionRangeInfoMap(originalSegment.getDimensionRangeInfoMap());

    try {
        cubeManager.promoteNewlyOptimizeSegments(cube, segment);
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to update cube after build", e);
        return ExecuteResult.createError(e);
    }
}
Example 15
Source File: ReceiverClusterManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
/**
 * Promote a segment from realtime part into historical part.
 */
void promoteNewSegment(CubingJob cubingJob, CubeInstance cubeInstance, CubeSegment cubeSegment) throws IOException {
    logger.debug("Try transfer segment's {} state to ready.", cubeSegment.getName());
    long sourceCount = cubingJob.findSourceRecordCount();
    long sourceSizeBytes = cubingJob.findSourceSizeBytes();
    long cubeSizeBytes = cubingJob.findCubeSizeBytes();
    Map<Integer, String> sourceCheckpoint = getCoordinator().getStreamMetadataStore()
            .getSourceCheckpoint(cubeInstance.getName(), cubeSegment.getName());

    ISourcePositionHandler positionOperator = StreamingSourceFactory.getStreamingSource(cubeInstance)
            .getSourcePositionHandler();
    Collection<ISourcePosition> sourcePositions = Collections2.transform(sourceCheckpoint.values(),
            new Function<String, ISourcePosition>() {
                @Nullable
                @Override
                public ISourcePosition apply(@Nullable String input) {
                    return positionOperator.parsePosition(input);
                }
            });
    ISourcePosition sourcePosition = positionOperator.mergePositions(sourcePositions,
            ISourcePositionHandler.MergeStrategy.KEEP_SMALL);
    cubeSegment.setLastBuildJobID(cubingJob.getId());
    cubeSegment.setLastBuildTime(System.currentTimeMillis());
    cubeSegment.setSizeKB(cubeSizeBytes / 1024);
    cubeSegment.setInputRecords(sourceCount);
    cubeSegment.setInputRecordsSize(sourceSizeBytes);
    cubeSegment.setStreamSourceCheckpoint(positionOperator.serializePosition(sourcePosition));
    getCoordinator().getCubeManager().promoteNewlyBuiltSegments(cubeInstance, cubeSegment);
}
Example 16
Source File: Coordinator.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void promoteNewSegment(CubingJob cubingJob, CubeInstance cubeInstance, CubeSegment cubeSegment)
        throws IOException {
    long sourceCount = cubingJob.findSourceRecordCount();
    long sourceSizeBytes = cubingJob.findSourceSizeBytes();
    long cubeSizeBytes = cubingJob.findCubeSizeBytes();
    Map<Integer, String> sourceCheckpoint = streamMetadataStore.getSourceCheckpoint(cubeInstance.getName(),
            cubeSegment.getName());

    ISourcePositionHandler positionOperator = StreamingSourceFactory.getStreamingSource(cubeInstance)
            .getSourcePositionHandler();
    Collection<ISourcePosition> sourcePositions = Collections2.transform(sourceCheckpoint.values(),
            new Function<String, ISourcePosition>() {
                @Nullable
                @Override
                public ISourcePosition apply(@Nullable String input) {
                    return positionOperator.parsePosition(input);
                }
            });
    ISourcePosition sourcePosition = positionOperator.mergePositions(sourcePositions, MergeStrategy.KEEP_SMALL);
    cubeSegment.setLastBuildJobID(cubingJob.getId());
    cubeSegment.setLastBuildTime(System.currentTimeMillis());
    cubeSegment.setSizeKB(cubeSizeBytes / 1024);
    cubeSegment.setInputRecords(sourceCount);
    cubeSegment.setInputRecordsSize(sourceSizeBytes);
    cubeSegment.setStreamSourceCheckpoint(positionOperator.serializePosition(sourcePosition));
    getCubeManager().promoteNewlyBuiltSegments(cubeInstance, cubeSegment);
}
Example 17
Source File: BuildJobSubmitter.java    From kylin with Apache License 2.0 4 votes vote down vote up 
/**
 * Submit a build job for streaming segment
 *
 * @return true if submit succeed ; else false
 */
@NotAtomicIdempotent
boolean submitSegmentBuildJob(String cubeName, String segmentName) {
    logger.info("Try submit streaming segment build job, cube:{} segment:{}", cubeName, segmentName);
    CubeInstance cubeInstance = coordinator.getCubeManager().getCube(cubeName);
    try {
        // Step 1. create a new segment if not exists
        CubeSegment newSeg = null;
        Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
        boolean segmentExists = false;
        for (CubeSegment segment : cubeInstance.getSegments()) {
            SegmentRange.TSRange tsRange = segment.getTSRange();
            if (tsRange.start.v.equals(segmentRange.getFirst()) && segmentRange.getSecond().equals(tsRange.end.v)) {
                segmentExists = true;
                newSeg = segment;
            }
        }

        if (segmentExists) {
            logger.warn("Segment {} exists, it will be forced deleted.", segmentName);
            coordinator.getCubeManager().updateCubeDropSegments(cubeInstance, newSeg);
        }
        
        logger.debug("Create segment for {} {} .", cubeName, segmentName);
        newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
                new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));

        // Step 2. create and submit new build job
        DefaultChainedExecutable executable = getStreamingCubingJob(newSeg);
        coordinator.getExecutableManager().addJob(executable);
        String jobId = executable.getId();
        newSeg.setLastBuildJobID(jobId);

        // Step 3. add it to job trigger list
        SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, jobId);
        addToJobTrackList(segmentJobBuildInfo);

        // Step 4. add job to stream metadata in case of current node dead
        SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
        state.setBuildStartTime(System.currentTimeMillis());
        state.setState(SegmentBuildState.BuildState.State.BUILDING);
        state.setJobId(jobId);
        logger.debug("Commit building job {} for {} {} .", jobId, cubeName, segmentName);
        coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
        return true;
    } catch (Exception e) {
        logger.error("Streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
        return false;
    }
}
Example 18
Source File: BuildJobSubmitter.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
/**
 * Submit a build job for streaming segment
 *
 * @return true if submit succeed ; else false
 */
@NotAtomicIdempotent
boolean submitSegmentBuildJob(String cubeName, String segmentName) {
    logger.info("Try submit streaming segment build job, cube:{} segment:{}", cubeName, segmentName);
    CubeInstance cubeInstance = coordinator.getCubeManager().getCube(cubeName);
    try {
        // Step 1. create a new segment if not exists
        CubeSegment newSeg = null;
        Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
        boolean segmentExists = false;
        for (CubeSegment segment : cubeInstance.getSegments()) {
            SegmentRange.TSRange tsRange = segment.getTSRange();
            if (tsRange.start.v.equals(segmentRange.getFirst()) && segmentRange.getSecond().equals(tsRange.end.v)) {
                segmentExists = true;
                newSeg = segment;
            }
        }

        if (segmentExists) {
            logger.warn("Segment {} exists, it will be forced deleted.", segmentName);
            coordinator.getCubeManager().updateCubeDropSegments(cubeInstance, newSeg);
        }
        
        logger.debug("Create segment for {} {} .", cubeName, segmentName);
        newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
                new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));

        // Step 2. create and submit new build job
        DefaultChainedExecutable executable = getStreamingCubingJob(newSeg);
        coordinator.getExecutableManager().addJob(executable);
        String jobId = executable.getId();
        newSeg.setLastBuildJobID(jobId);

        // Step 3. add it to job trigger list
        SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, jobId);
        addToJobTrackList(segmentJobBuildInfo);

        // Step 4. add job to stream metadata in case of current node dead
        SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
        state.setBuildStartTime(System.currentTimeMillis());
        state.setState(SegmentBuildState.BuildState.State.BUILDING);
        state.setJobId(jobId);
        logger.debug("Commit building job {} for {} {} .", jobId, cubeName, segmentName);
        coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
        return true;
    } catch (Exception e) {
        logger.error("Streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
        return false;
    }
}
Example 19
Source File: UpdateCubeInfoAfterBuildStep.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
    final CubeInstance cube = cubeManager.getCube(getCubeName());
    final CubeSegment segment = cube.getSegmentById(getSegmentId());

    Output baseCuboidOutput = executableManager.getOutput(getBaseCuboidStepId());
    String sourceRecordsCount = baseCuboidOutput.getExtra().get(ExecutableConstants.SOURCE_RECORDS_COUNT);
    Preconditions.checkState(StringUtils.isNotEmpty(sourceRecordsCount), "Can't get cube source record count.");
    long sourceCount = Long.parseLong(sourceRecordsCount);

    String sourceRecordsSize = baseCuboidOutput.getExtra().get(ExecutableConstants.SOURCE_RECORDS_SIZE);
    Preconditions.checkState(StringUtils.isNotEmpty(sourceRecordsSize), "Can't get cube source record size.");
    long sourceSize = Long.parseLong(sourceRecordsSize);

    long size = 0;
    boolean segmentReady = true;
    if (!StringUtils.isBlank(getConvertToHfileStepId())) {
        String cubeSizeString = executableManager.getOutput(getConvertToHfileStepId()).getExtra().get(ExecutableConstants.HDFS_BYTES_WRITTEN);
        Preconditions.checkState(StringUtils.isNotEmpty(cubeSizeString), "Can't get cube segment size.");
        size = Long.parseLong(cubeSizeString) / 1024;
    } else {
        // for the increment & merge case, the increment segment is only built to be merged, won't serve query by itself
        segmentReady = false;
    }

    segment.setLastBuildJobID(getCubingJobId());
    segment.setLastBuildTime(System.currentTimeMillis());
    segment.setSizeKB(size);
    segment.setInputRecords(sourceCount);
    segment.setInputRecordsSize(sourceSize);

    try {
        if (segmentReady) {
            cubeManager.promoteNewlyBuiltSegments(cube, segment);
        } else {
            cubeManager.updateCube(cube);
        }
        return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
    } catch (IOException e) {
        logger.error("fail to update cube after build", e);
        return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
    }
}