Java Code Examples for org.apache.kylin.metadata.model.SegmentStatusEnum#NEW

The following examples show how to use org.apache.kylin.metadata.model.SegmentStatusEnum#NEW . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: KylinHealthCheckJob.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private void checkHBaseTables(List<CubeInstance> cubes) throws IOException {
    reporter.log("## Checking HBase Table of segments");
    HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create());
    try {
        for (CubeInstance cube : cubes) {
            for (CubeSegment segment : cube.getSegments()) {
                if (segment.getStatus() != SegmentStatusEnum.NEW) {
                    String tableName = segment.getStorageLocationIdentifier();
                    if ((!hbaseAdmin.tableExists(tableName)) || (!hbaseAdmin.isTableEnabled(tableName))) {
                        reporter.log("HBase table: {} not exist for segment: {}, project: {}", tableName, segment,
                                cube.getProject());
                        reporter.log(
                                "The rebuild url: -d '{\"startTime\":{}, \"endTime\":{}, \"buildType\":\"REFRESH\"}' /kylin/api/cubes/{}/build",
                                segment.getTSRange().start, segment.getTSRange().end, cube.getName());
                    }
                }
            }
        }
    } finally {
        if (null != hbaseAdmin) {
            hbaseAdmin.close();
        }
    }

}

Example 2

Source File: JobService.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private void cancelCubingJobInner(CubingJob cubingJob) throws IOException {
    CubeInstance cubeInstance = getCubeManager().getCube(CubingExecutableUtil.getCubeName(cubingJob.getParams()));
    // might not a cube job
    final String segmentIds = CubingExecutableUtil.getSegmentId(cubingJob.getParams());
    if (!StringUtils.isEmpty(segmentIds)) {
        for (String segmentId : StringUtils.split(segmentIds)) {
            final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
            if (segment != null
                    && (segment.getStatus() == SegmentStatusEnum.NEW || segment.getTSRange().end.v == 0)) {
                // Remove this segment
                getCubeManager().updateCubeDropSegments(cubeInstance, segment);
            }
        }
    }
    getExecutableManager().discardJob(cubingJob.getId());
}

Example 3

Source File: AfterMergeOrRefreshResourceMerger.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

List<CubeSegment> getToRemoveSegs(CubeInstance cube, CubeSegment mergedSegment) {
    Segments tobe = cube.calculateToBeSegments(mergedSegment);

    if (!tobe.contains(mergedSegment))
        throw new IllegalStateException(
                "For Cube " + cube + ", segment " + mergedSegment + " is expected but not in the tobe " + tobe);

    if (mergedSegment.getStatus() == SegmentStatusEnum.NEW)
        mergedSegment.setStatus(SegmentStatusEnum.READY);

    List<CubeSegment> toRemoveSegs = Lists.newArrayList();
    for (CubeSegment s : cube.getSegments()) {
        if (!tobe.contains(s))
            toRemoveSegs.add(s);
    }

    return toRemoveSegs;
}

Example 4

Source File: KylinHealthCheckJob.java From kylin with Apache License 2.0

6 votes

private void checkHBaseTables(List<CubeInstance> cubes) throws IOException {
    reporter.log("## Checking HBase Table of segments");
    HBaseAdmin hbaseAdmin = new HBaseAdmin(HBaseConfiguration.create());
    try {
        for (CubeInstance cube : cubes) {
            for (CubeSegment segment : cube.getSegments()) {
                if (segment.getStatus() != SegmentStatusEnum.NEW) {
                    String tableName = segment.getStorageLocationIdentifier();
                    if ((!hbaseAdmin.tableExists(tableName)) || (!hbaseAdmin.isTableEnabled(tableName))) {
                        reporter.log("HBase table: {} not exist for segment: {}, project: {}", tableName, segment,
                                cube.getProject());
                        reporter.log(
                                "The rebuild url: -d '{\"startTime\":{}, \"endTime\":{}, \"buildType\":\"REFRESH\"}' /kylin/api/cubes/{}/build",
                                segment.getTSRange().start, segment.getTSRange().end, cube.getName());
                    }
                }
            }
        }
    } finally {
        if (null != hbaseAdmin) {
            hbaseAdmin.close();
        }
    }

}

Example 5

Source File: JobService.java From kylin with Apache License 2.0

6 votes

private void cancelCubingJobInner(CubingJob cubingJob) throws IOException {
    CubeInstance cubeInstance = getCubeManager().getCube(CubingExecutableUtil.getCubeName(cubingJob.getParams()));
    // might not a cube job
    final String segmentIds = CubingExecutableUtil.getSegmentId(cubingJob.getParams());
    if (!StringUtils.isEmpty(segmentIds)) {
        for (String segmentId : StringUtils.split(segmentIds)) {
            final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
            if (segment != null
                    && (segment.getStatus() == SegmentStatusEnum.NEW || segment.getTSRange().end.v == 0)) {
                // Remove this segment
                getCubeManager().updateCubeDropSegments(cubeInstance, segment);
            }
        }
    }
    getExecutableManager().discardJob(cubingJob.getId());
}

Example 6

Source File: JobService.java From Kylin with Apache License 2.0

6 votes

@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#job, 'ADMINISTRATION') or hasPermission(#job, 'OPERATION') or hasPermission(#job, 'MANAGEMENT')")
public JobInstance cancelJob(String jobId) throws IOException, JobException {
    //        CubeInstance cube = this.getCubeManager().getCube(job.getRelatedCube());
    //        for (BuildCubeJob cubeJob: listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING))) {
    //            getExecutableManager().stopJob(cubeJob.getId());
    //        }
    final JobInstance jobInstance = getJobInstance(jobId);
    final String segmentId = jobInstance.getRelatedSegment();
    CubeInstance cubeInstance = getCubeManager().getCube(jobInstance.getRelatedCube());
    final CubeSegment segment = cubeInstance.getSegmentById(segmentId);
    if (segment.getStatus() == SegmentStatusEnum.NEW) {
        cubeInstance.getSegments().remove(segment);
        getCubeManager().updateCube(cubeInstance);
    }
    getExecutableManager().discardJob(jobId);
    return jobInstance;
}

Example 7

Source File: CubeInstance.java From Kylin with Apache License 2.0

5 votes

public List<CubeSegment> getBuildingSegments() {
    List<CubeSegment> buildingSegments = new ArrayList<CubeSegment>();
    if (null != segments) {
        for (CubeSegment segment : segments) {
            if (SegmentStatusEnum.NEW == segment.getStatus() || SegmentStatusEnum.READY_PENDING == segment.getStatus()) {
                buildingSegments.add(segment);
            }
        }
    }

    return buildingSegments;
}

Example 8

Source File: CubeService.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.READY) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getDELETE_SEG_FROM_READY_CUBE(), segmentName, cube.getName()));
    }

    CubeSegment toDelete = null;
    for (CubeSegment seg : cube.getSegments()) {
        if (seg.getName().equals(segmentName)) {
            toDelete = seg;
            break;
        }
    }

    if (toDelete == null) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getSEG_NOT_FOUND(), segmentName));
    }

    if (toDelete.getStatus() != SegmentStatusEnum.READY) {
        if (toDelete.getStatus() == SegmentStatusEnum.NEW) {
            if (!isOrphonSegment(cube, toDelete.getUuid())) {
                throw new BadRequestException(
                        String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
            }
        } else {
            throw new BadRequestException(String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
        }
    }

    if (!segmentName.equals(cube.getSegments().get(0).getName())
            && !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) {
        logger.warn(String.format(Locale.ROOT, msg.getDELETE_SEGMENT_CAUSE_GAPS(), cube.getName(), segmentName));
    }

    CubeInstance cubeInstance = CubeManager.getInstance(getConfig()).updateCubeDropSegments(cube, toDelete);

    cleanSegmentStorage(Collections.singletonList(toDelete));

    return cubeInstance;
}

Example 9

Source File: CubeService.java From kylin with Apache License 2.0

4 votes

public CubeInstance deleteSegment(CubeInstance cube, String segmentName) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.READY) {
        throw new BadRequestException(
                String.format(Locale.ROOT, msg.getDELETE_SEG_FROM_READY_CUBE(), segmentName, cube.getName()));
    }

    CubeSegment toDelete = null;
    for (CubeSegment seg : cube.getSegments()) {
        if (seg.getName().equals(segmentName)) {
            toDelete = seg;
            break;
        }
    }

    if (toDelete == null) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getSEG_NOT_FOUND(), segmentName));
    }

    if (toDelete.getStatus() != SegmentStatusEnum.READY) {
        if (toDelete.getStatus() == SegmentStatusEnum.NEW) {
            if (!isOrphonSegment(cube, toDelete.getUuid())) {
                throw new BadRequestException(
                        String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
            }
        } else {
            throw new BadRequestException(String.format(Locale.ROOT, msg.getDELETE_NOT_READY_SEG(), segmentName));
        }
    }

    if (!segmentName.equals(cube.getSegments().get(0).getName())
            && !segmentName.equals(cube.getSegments().get(cube.getSegments().size() - 1).getName())) {
        logger.warn(String.format(Locale.ROOT, msg.getDELETE_SEGMENT_CAUSE_GAPS(), cube.getName(), segmentName));
    }

    CubeInstance cubeInstance = CubeManager.getInstance(getConfig()).updateCubeDropSegments(cube, toDelete);

    cleanSegmentStorage(Collections.singletonList(toDelete));

    return cubeInstance;
}

Example 10

Source File: CubeManager.java From Kylin with Apache License 2.0

4 votes

private boolean isNew(CubeSegment seg) {
    return seg.getStatus() == SegmentStatusEnum.NEW || seg.getStatus() == SegmentStatusEnum.READY_PENDING;
}