Java Code Examples for org.apache.kylin.cube.CubeSegment#getInputRecordsSize()

The following examples show how to use org.apache.kylin.cube.CubeSegment#getInputRecordsSize() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: UpdateCubeInfoAfterOptimizeStep.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
    final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));

    CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(segment);
    long sourceCount = originalSegment.getInputRecords();
    long sourceSizeBytes = originalSegment.getInputRecordsSize();

    CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
    long cubeSizeBytes = cubingJob.findCubeSizeBytes();

    segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
    segment.setLastBuildTime(System.currentTimeMillis());
    segment.setSizeKB(cubeSizeBytes / 1024);
    segment.setInputRecords(sourceCount);
    segment.setInputRecordsSize(sourceSizeBytes);
    segment.setDimensionRangeInfoMap(originalSegment.getDimensionRangeInfoMap());

    try {
        cubeManager.promoteNewlyOptimizeSegments(cube, segment);
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to update cube after build", e);
        return ExecuteResult.createError(e);
    }
}
Example 2
Source File: KylinHealthCheckJob.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void checkStaleSegments(List<CubeInstance> cubes) {
    for (CubeInstance cube : cubes) {
        for (CubeSegment segment : cube.getSegments()) {
            if (segment.getInputRecordsSize() == 0) {
                // TODO: add stale segment to report
                logger.info("Segment: {} in project: {} may be stale", segment, cube.getProject());
            }
        }
    }
}
Example 3
Source File: UpdateCubeInfoAfterOptimizeStep.java    From kylin with Apache License 2.0 5 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeManager = CubeManager.getInstance(context.getConfig());
    final CubeInstance cube = cubeManager.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment segment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));

    CubeSegment originalSegment = cube.getOriginalSegmentToOptimize(segment);
    long sourceCount = originalSegment.getInputRecords();
    long sourceSizeBytes = originalSegment.getInputRecordsSize();

    CubingJob cubingJob = (CubingJob) getManager().getJob(CubingExecutableUtil.getCubingJobId(this.getParams()));
    long cubeSizeBytes = cubingJob.findCubeSizeBytes();

    segment.setLastBuildJobID(CubingExecutableUtil.getCubingJobId(this.getParams()));
    segment.setLastBuildTime(System.currentTimeMillis());
    segment.setSizeKB(cubeSizeBytes / 1024);
    segment.setInputRecords(sourceCount);
    segment.setInputRecordsSize(sourceSizeBytes);
    segment.setDimensionRangeInfoMap(originalSegment.getDimensionRangeInfoMap());

    try {
        cubeManager.promoteNewlyOptimizeSegments(cube, segment);
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to update cube after build", e);
        return ExecuteResult.createError(e);
    }
}
Example 4
Source File: KylinHealthCheckJob.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void checkStaleSegments(List<CubeInstance> cubes) {
    for (CubeInstance cube : cubes) {
        for (CubeSegment segment : cube.getSegments()) {
            if (segment.getInputRecordsSize() == 0) {
                // TODO: add stale segment to report
                logger.info("Segment: {} in project: {} may be stale", segment, cube.getProject());
            }
        }
    }
}
Example 5
Source File: UpdateCubeInfoAfterMergeStep.java    From Kylin with Apache License 2.0 5 votes vote down vote up 
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeInstance cube = cubeManager.getCube(getCubeName());
    
    CubeSegment mergedSegment = cube.getSegmentById(getSegmentId());
    if (mergedSegment == null) {
        return new ExecuteResult(ExecuteResult.State.FAILED, "there is no segment with id:" + getSegmentId());
    }
    String cubeSizeString = executableManager.getOutput(getConvertToHfileStepId()).getExtra().get(ExecutableConstants.HDFS_BYTES_WRITTEN);
    Preconditions.checkState(StringUtils.isNotEmpty(cubeSizeString), "Can't get cube segment size.");
    long cubeSize = Long.parseLong(cubeSizeString) / 1024;

    // collect source statistics
    List<String> mergingSegmentIds = getMergingSegmentIds();
    if (mergingSegmentIds.isEmpty()) {
        return new ExecuteResult(ExecuteResult.State.FAILED, "there are no merging segments");
    }
    long sourceCount = 0L;
    long sourceSize = 0L;
    for (String id : mergingSegmentIds) {
        CubeSegment segment = cube.getSegmentById(id);
        sourceCount += segment.getInputRecords();
        sourceSize += segment.getInputRecordsSize();
    }
    
    // update segment info
    mergedSegment.setSizeKB(cubeSize);
    mergedSegment.setInputRecords(sourceCount);
    mergedSegment.setInputRecordsSize(sourceSize);
    mergedSegment.setLastBuildJobID(getCubingJobId());
    mergedSegment.setLastBuildTime(System.currentTimeMillis());
    
    try {
        cubeManager.promoteNewlyBuiltSegments(cube, mergedSegment);
        return new ExecuteResult(ExecuteResult.State.SUCCEED);
    } catch (IOException e) {
        logger.error("fail to update cube after merge", e);
        return new ExecuteResult(ExecuteResult.State.ERROR, e.getLocalizedMessage());
    }
}