Java Code Examples for org.apache.kylin.job.execution.DefaultChainedExecutable#getId()

The following examples show how to use org.apache.kylin.job.execution.DefaultChainedExecutable#getId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HBaseJobSteps.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
public void addCubingGarbageCollectionSteps(DefaultChainedExecutable jobFlow) {
    String jobId = jobFlow.getId();

    List<String> toDeletePaths = new ArrayList<>();
    toDeletePaths.add(getFactDistinctColumnsPath(jobId));
    toDeletePaths.add(getHFilePath(jobId));
    toDeletePaths.add(getShrunkenDictionaryPath(jobId));

    CubeSegment oldSegment = ((CubeInstance)seg.getRealization()).getOriginalSegmentToRefresh(seg);

    // refresh segment
    if (oldSegment != null && KylinConfig.getInstanceFromEnv().cleanStorageAfterDelOperation()) {
        // delete old hdfs job
        toDeletePaths.addAll(getRefreshingHDFSPaths());

        // drop old htables
        MergeGCStep hBaseGCStep = createHBaseGCStep(getRefreshingHTables());
        jobFlow.addTask(hBaseGCStep);
    }

    HDFSPathGarbageCollectionStep step = createHDFSPathGCStep(toDeletePaths, jobId);
    jobFlow.addTask(step);
}
Example 2
Source File: HBaseJobSteps.java    From kylin with Apache License 2.0 6 votes vote down vote up 
public void addCubingGarbageCollectionSteps(DefaultChainedExecutable jobFlow) {
    String jobId = jobFlow.getId();

    List<String> toDeletePaths = new ArrayList<>();
    toDeletePaths.add(getFactDistinctColumnsPath(jobId));
    toDeletePaths.add(getHFilePath(jobId));
    toDeletePaths.add(getShrunkenDictionaryPath(jobId));

    CubeSegment oldSegment = ((CubeInstance)seg.getRealization()).getOriginalSegmentToRefresh(seg);

    // refresh segment
    if (oldSegment != null && KylinConfig.getInstanceFromEnv().cleanStorageAfterDelOperation()) {
        // delete old hdfs job
        toDeletePaths.addAll(getRefreshingHDFSPaths());

        // drop old htables
        MergeGCStep hBaseGCStep = createHBaseGCStep(getRefreshingHTables());
        jobFlow.addTask(hBaseGCStep);
    }

    HDFSPathGarbageCollectionStep step = createHDFSPathGCStep(toDeletePaths, jobId);
    jobFlow.addTask(step);
}
Example 3
Source File: BuildCubeWithStream.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private String refreshSegment(String cubeName, SegmentRange segRange) throws Exception {
    CubeSegment segment = cubeManager.refreshSegment(cubeManager.getCube(cubeName), null, segRange);
    DefaultChainedExecutable job = EngineFactory.createBatchCubingJob(segment, "TEST");
    jobService.addJob(job);
    waitForJob(job.getId());
    return job.getId();
}
Example 4
Source File: HBaseJobSteps.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public void addMergingGarbageCollectionSteps(DefaultChainedExecutable jobFlow) {
    String jobId = jobFlow.getId();

    MergeGCStep hBaseGCStep = createHBaseGCStep(getMergingHTables());
    jobFlow.addTask(hBaseGCStep);

    List<String> toDeletePaths = new ArrayList<>();
    toDeletePaths.addAll(getMergingHDFSPaths());
    toDeletePaths.add(getHFilePath(jobId));

    HDFSPathGarbageCollectionStep step = createHDFSPathGCStep(toDeletePaths, jobId);

    jobFlow.addTask(step);
}
Example 5
Source File: CubeService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public String mergeCubeSegment(String cubeName) {
    CubeInstance cube = getCubeManager().getCube(cubeName);
    if (!cube.needAutoMerge())
        return null;

    if (!cube.isReady()) {
        logger.info("The cube: {} is disabled", cubeName);
        return null;
    }

    synchronized (CubeService.class) {
        try {
            cube = getCubeManager().getCube(cubeName);
            SegmentRange offsets = cube.autoMergeCubeSegments();
            if (offsets != null && !isMergingJobBeenDiscarded(cube, cubeName, cube.getProject(), offsets)) {
                CubeSegment newSeg = getCubeManager().mergeSegments(cube, null, offsets, true);
                logger.info("Will submit merge job on " + newSeg);
                DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(newSeg, "SYSTEM");
                getExecutableManager().addJob(job);
                return job.getId();
            } else {
                logger.info("Not ready for merge on cube " + cubeName);
            }
        } catch (IOException e) {
            logger.error("Failed to auto merge cube " + cubeName, e);
        }
    }
    return null;
}
Example 6
Source File: BuildCubeWithStream.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private String refreshSegment(String cubeName, SegmentRange segRange) throws Exception {
    CubeSegment segment = cubeManager.refreshSegment(cubeManager.getCube(cubeName), null, segRange);
    DefaultChainedExecutable job = EngineFactory.createBatchCubingJob(segment, "TEST");
    jobService.addJob(job);
    waitForJob(job.getId());
    return job.getId();
}
Example 7
Source File: HBaseJobSteps.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public void addMergingGarbageCollectionSteps(DefaultChainedExecutable jobFlow) {
    String jobId = jobFlow.getId();

    MergeGCStep hBaseGCStep = createHBaseGCStep(getMergingHTables());
    jobFlow.addTask(hBaseGCStep);

    List<String> toDeletePaths = new ArrayList<>();
    toDeletePaths.addAll(getMergingHDFSPaths());
    toDeletePaths.add(getHFilePath(jobId));

    HDFSPathGarbageCollectionStep step = createHDFSPathGCStep(toDeletePaths, jobId);

    jobFlow.addTask(step);
}
Example 8
Source File: CubeService.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public String mergeCubeSegment(String cubeName) {
    CubeInstance cube = getCubeManager().getCube(cubeName);
    if (!cube.needAutoMerge())
        return null;

    if (!cube.isReady()) {
        logger.info("The cube: {} is disabled", cubeName);
        return null;
    }

    synchronized (CubeService.class) {
        try {
            cube = getCubeManager().getCube(cubeName);
            SegmentRange offsets = cube.autoMergeCubeSegments();
            if (offsets != null && !isMergingJobBeenDiscarded(cube, cubeName, cube.getProject(), offsets)) {
                CubeSegment newSeg = getCubeManager().mergeSegments(cube, null, offsets, true);
                logger.info("Will submit merge job on " + newSeg);
                DefaultChainedExecutable job = EngineFactory.createBatchMergeJob(newSeg, "SYSTEM");
                getExecutableManager().addJob(job);
                return job.getId();
            } else {
                logger.info("Not ready for merge on cube " + cubeName);
            }
        } catch (IOException e) {
            logger.error("Failed to auto merge cube " + cubeName, e);
        }
    }
    return null;
}
Example 9
Source File: BuildJobSubmitter.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
/**
 * Submit a build job for streaming segment
 *
 * @return true if submit succeed ; else false
 */
@NotAtomicIdempotent
boolean submitSegmentBuildJob(String cubeName, String segmentName) {
    logger.info("Try submit streaming segment build job, cube:{} segment:{}", cubeName, segmentName);
    CubeInstance cubeInstance = coordinator.getCubeManager().getCube(cubeName);
    try {
        // Step 1. create a new segment if not exists
        CubeSegment newSeg = null;
        Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
        boolean segmentExists = false;
        for (CubeSegment segment : cubeInstance.getSegments()) {
            SegmentRange.TSRange tsRange = segment.getTSRange();
            if (tsRange.start.v.equals(segmentRange.getFirst()) && segmentRange.getSecond().equals(tsRange.end.v)) {
                segmentExists = true;
                newSeg = segment;
            }
        }

        if (segmentExists) {
            logger.warn("Segment {} exists, it will be forced deleted.", segmentName);
            coordinator.getCubeManager().updateCubeDropSegments(cubeInstance, newSeg);
        }
        
        logger.debug("Create segment for {} {} .", cubeName, segmentName);
        newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
                new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));

        // Step 2. create and submit new build job
        DefaultChainedExecutable executable = getStreamingCubingJob(newSeg);
        coordinator.getExecutableManager().addJob(executable);
        String jobId = executable.getId();
        newSeg.setLastBuildJobID(jobId);

        // Step 3. add it to job trigger list
        SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, jobId);
        addToJobTrackList(segmentJobBuildInfo);

        // Step 4. add job to stream metadata in case of current node dead
        SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
        state.setBuildStartTime(System.currentTimeMillis());
        state.setState(SegmentBuildState.BuildState.State.BUILDING);
        state.setJobId(jobId);
        logger.debug("Commit building job {} for {} {} .", jobId, cubeName, segmentName);
        coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
        return true;
    } catch (Exception e) {
        logger.error("Streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
        return false;
    }
}
Example 10
Source File: BuildJobSubmitter.java    From kylin with Apache License 2.0 4 votes vote down vote up 
/**
 * Submit a build job for streaming segment
 *
 * @return true if submit succeed ; else false
 */
@NotAtomicIdempotent
boolean submitSegmentBuildJob(String cubeName, String segmentName) {
    logger.info("Try submit streaming segment build job, cube:{} segment:{}", cubeName, segmentName);
    CubeInstance cubeInstance = coordinator.getCubeManager().getCube(cubeName);
    try {
        // Step 1. create a new segment if not exists
        CubeSegment newSeg = null;
        Pair<Long, Long> segmentRange = CubeSegment.parseSegmentName(segmentName);
        boolean segmentExists = false;
        for (CubeSegment segment : cubeInstance.getSegments()) {
            SegmentRange.TSRange tsRange = segment.getTSRange();
            if (tsRange.start.v.equals(segmentRange.getFirst()) && segmentRange.getSecond().equals(tsRange.end.v)) {
                segmentExists = true;
                newSeg = segment;
            }
        }

        if (segmentExists) {
            logger.warn("Segment {} exists, it will be forced deleted.", segmentName);
            coordinator.getCubeManager().updateCubeDropSegments(cubeInstance, newSeg);
        }
        
        logger.debug("Create segment for {} {} .", cubeName, segmentName);
        newSeg = coordinator.getCubeManager().appendSegment(cubeInstance,
                new SegmentRange.TSRange(segmentRange.getFirst(), segmentRange.getSecond()));

        // Step 2. create and submit new build job
        DefaultChainedExecutable executable = getStreamingCubingJob(newSeg);
        coordinator.getExecutableManager().addJob(executable);
        String jobId = executable.getId();
        newSeg.setLastBuildJobID(jobId);

        // Step 3. add it to job trigger list
        SegmentJobBuildInfo segmentJobBuildInfo = new SegmentJobBuildInfo(cubeName, segmentName, jobId);
        addToJobTrackList(segmentJobBuildInfo);

        // Step 4. add job to stream metadata in case of current node dead
        SegmentBuildState.BuildState state = new SegmentBuildState.BuildState();
        state.setBuildStartTime(System.currentTimeMillis());
        state.setState(SegmentBuildState.BuildState.State.BUILDING);
        state.setJobId(jobId);
        logger.debug("Commit building job {} for {} {} .", jobId, cubeName, segmentName);
        coordinator.getStreamMetadataStore().updateSegmentBuildState(cubeName, segmentName, state);
        return true;
    } catch (Exception e) {
        logger.error("Streaming job submit fail, cubeName:" + cubeName + " segment:" + segmentName, e);
        return false;
    }
}
Example 11
Source File: HBaseJobSteps.java    From kylin-on-parquet-v2 with Apache License 2.0 3 votes vote down vote up 
public void addOptimizeGarbageCollectionSteps(DefaultChainedExecutable jobFlow) {
    String jobId = jobFlow.getId();

    List<String> toDeletePaths = new ArrayList<>();
    toDeletePaths.add(getOptimizationRootPath(jobId));

    HDFSPathGarbageCollectionStep step =createHDFSPathGCStep(toDeletePaths, jobId);

    jobFlow.addTask(step);
}
Example 12
Source File: HBaseJobSteps.java    From kylin-on-parquet-v2 with Apache License 2.0 3 votes vote down vote up 
public void addCheckpointGarbageCollectionSteps(DefaultChainedExecutable jobFlow) {
    String jobId = jobFlow.getId();

    MergeGCStep hBaseGCStep = createHBaseGCStep(getOptimizeHTables());
    jobFlow.addTask(hBaseGCStep);

    List<String> toDeletePaths = new ArrayList<>();
    toDeletePaths.addAll(getOptimizeHDFSPaths());

    HDFSPathGarbageCollectionStep step = createHDFSPathGCStep(toDeletePaths, jobId);

    jobFlow.addTask(step);
}
Example 13
Source File: HBaseJobSteps.java    From kylin with Apache License 2.0 3 votes vote down vote up 
public void addOptimizeGarbageCollectionSteps(DefaultChainedExecutable jobFlow) {
    String jobId = jobFlow.getId();

    List<String> toDeletePaths = new ArrayList<>();
    toDeletePaths.add(getOptimizationRootPath(jobId));

    HDFSPathGarbageCollectionStep step =createHDFSPathGCStep(toDeletePaths, jobId);

    jobFlow.addTask(step);
}
Example 14
Source File: HBaseJobSteps.java    From kylin with Apache License 2.0 3 votes vote down vote up 
public void addCheckpointGarbageCollectionSteps(DefaultChainedExecutable jobFlow) {
    String jobId = jobFlow.getId();

    MergeGCStep hBaseGCStep = createHBaseGCStep(getOptimizeHTables());
    jobFlow.addTask(hBaseGCStep);

    List<String> toDeletePaths = new ArrayList<>();
    toDeletePaths.addAll(getOptimizeHDFSPaths());

    HDFSPathGarbageCollectionStep step = createHDFSPathGCStep(toDeletePaths, jobId);

    jobFlow.addTask(step);
}