org.apache.kylin.cube.model.CubeBuildTypeEnum Java Examples

The following examples show how to use org.apache.kylin.cube.model.CubeBuildTypeEnum. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JobService.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private JobInstance parseToJobInstance(AbstractExecutable job) {
    if (job == null) {
        return null;
    }
    Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId());
    CubingJob cubeJob = (CubingJob) job;
    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setRelatedCube(cubeJob.getCubeName());
    result.setRelatedSegment(cubeJob.getSegmentId());
    result.setLastModified(cubeJob.getLastModified());
    result.setSubmitter(cubeJob.getSubmitter());
    result.setUuid(cubeJob.getId());
    result.setType(CubeBuildTypeEnum.BUILD);
    result.setStatus(parseToJobStatus(job.getStatus()));
    result.setMrWaiting(cubeJob.getMapReduceWaitTime() / 1000);
    result.setDuration(cubeJob.getDuration() / 1000);
    for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
        AbstractExecutable task = cubeJob.getTasks().get(i);
        result.addStep(parseToJobStep(task, i));
    }
    return result;
}
Example #2
Source File: JobInfoConverterTest.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Test
public void testParseToJobInstance4CheckpointJob() {
    Test2Job task = new Test2Job();
    String jobId = UUID.randomUUID().toString();
    String cubeName = "cube1";
    task.setId(jobId);
    task.setParam(CubingExecutableUtil.CUBE_NAME, cubeName);
    Map<String, Output> outPutMap = Maps.newHashMap();
    DefaultOutput executeOutput = new DefaultOutput();
    executeOutput.setState(ExecutableState.READY);
    Map<String, String> extraMap = Maps.newHashMap();
    executeOutput.setExtra(extraMap);
    outPutMap.put(jobId, executeOutput);

    JobInstance instance3 = JobInfoConverter.parseToJobInstanceQuietly(task, outPutMap);
    // no exception thrown is expected
    assertEquals(jobId, instance3.getId());
    assertEquals(CubeBuildTypeEnum.CHECKPOINT, instance3.getType());
    assertEquals(cubeName, instance3.getRelatedCube());
    assertEquals(JobStatusEnum.PENDING, instance3.getStatus());
}
Example #3
Source File: JobInfoConverterTest.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Test
public void testParseToJobInstance4CuboidJob() {
    TestJob task = new TestJob();
    String jobId = UUID.randomUUID().toString();
    String cubeName = "cube1";
    task.setId(jobId);
    task.setParam(CubingExecutableUtil.CUBE_NAME, cubeName);
    Map<String, Output> outPutMap = Maps.newHashMap();
    DefaultOutput executeOutput = new DefaultOutput();
    executeOutput.setState(ExecutableState.READY);
    Map<String, String> extraMap = Maps.newHashMap();
    executeOutput.setExtra(extraMap);
    outPutMap.put(jobId, executeOutput);

    JobInstance instance3 = JobInfoConverter.parseToJobInstanceQuietly(task, outPutMap);
    // no exception thrown is expected
    assertEquals(jobId, instance3.getId());
    assertEquals(CubeBuildTypeEnum.BUILD, instance3.getType());
    assertEquals(cubeName, instance3.getRelatedCube());
    assertEquals(JobStatusEnum.PENDING, instance3.getStatus());
}
Example #4
Source File: JobInfoConverter.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public static JobInstance parseToJobInstance(CheckpointExecutable job, Map<String, Output> outputs) {
    if (job == null) {
        logger.warn("job is null.");
        return null;
    }

    Output output = outputs.get(job.getId());
    if (output == null) {
        logger.warn("job output is null.");
        return null;
    }

    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setProjectName(job.getProjectName());
    result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
    result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams()));
    result.setLastModified(output.getLastModified());
    result.setSubmitter(job.getSubmitter());
    result.setUuid(job.getId());
    result.setType(CubeBuildTypeEnum.CHECKPOINT);
    result.setStatus(parseToJobStatus(output.getState()));
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setExecStartTime(AbstractExecutable.getStartTime(output));
    result.setExecEndTime(AbstractExecutable.getEndTime(output));
    result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
    result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(),
            result.getExecInterruptTime()) / 1000);
    for (int i = 0; i < job.getTasks().size(); ++i) {
        AbstractExecutable task = job.getTasks().get(i);
        result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
    }
    return result;
}
Example #5
Source File: JobService.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected JobInstance getCheckpointJobInstance(AbstractExecutable job) {
    Message msg = MsgPicker.getMsg();

    if (job == null) {
        return null;
    }
    if (!(job instanceof CheckpointExecutable)) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_JOB_TYPE(), job.getId()));
    }

    CheckpointExecutable checkpointExecutable = (CheckpointExecutable) job;
    Output output = checkpointExecutable.getOutput();
    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setProjectName(checkpointExecutable.getProjectName());
    result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
    result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams()));
    result.setLastModified(job.getLastModified());
    result.setSubmitter(job.getSubmitter());
    result.setUuid(job.getId());
    result.setExecStartTime(job.getStartTime());
    result.setExecEndTime(job.getEndTime());
    result.setExecInterruptTime(job.getInterruptTime());
    result.setType(CubeBuildTypeEnum.CHECKPOINT);
    result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setDuration(job.getDuration() / 1000);
    for (int i = 0; i < checkpointExecutable.getTasks().size(); ++i) {
        AbstractExecutable task = checkpointExecutable.getTasks().get(i);
        result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
    }
    return result;
}
Example #6
Source File: JobService.java    From kylin with Apache License 2.0 5 votes vote down vote up 
protected JobInstance getLookupSnapshotBuildJobInstance(LookupSnapshotBuildJob job) {
    if (job == null) {
        return null;
    }
    Output output = job.getOutput();
    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setProjectName(job.getProjectName());
    result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
    result.setRelatedSegment(CubingExecutableUtil.getSegmentId(job.getParams()));
    result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(job.getParams()));
    result.setLastModified(job.getLastModified());
    result.setSubmitter(job.getSubmitter());
    result.setUuid(job.getId());
    result.setExecStartTime(job.getStartTime());
    result.setExecEndTime(job.getEndTime());
    result.setExecInterruptTime(job.getInterruptTime());
    result.setType(CubeBuildTypeEnum.BUILD);
    result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setDuration(job.getDuration() / 1000);
    for (int i = 0; i < job.getTasks().size(); ++i) {
        AbstractExecutable task = job.getTasks().get(i);
        result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
    }
    return result;
}
Example #7
Source File: JobService.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public JobInstance submitJob(CubeInstance cube, TSRange tsRange, SegmentRange segRange, //
        Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd,
        CubeBuildTypeEnum buildType, boolean force, String submitter, Integer priorityOffset) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    JobInstance jobInstance = submitJobInternal(cube, tsRange, segRange, sourcePartitionOffsetStart,
            sourcePartitionOffsetEnd, buildType, force, submitter, priorityOffset);

    return jobInstance;
}
Example #8
Source File: CubeController.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private JobInstance buildInternal(String cubeName, TSRange tsRange, SegmentRange segRange, //
        Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd,
        String buildType, boolean force, Integer priorityOffset) {
    try {
        String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
        CubeInstance cube = jobService.getCubeManager().getCube(cubeName);

        checkBuildingSegment(cube);
        return jobService.submitJob(cube, tsRange, segRange, sourcePartitionOffsetStart, sourcePartitionOffsetEnd,
                CubeBuildTypeEnum.valueOf(buildType), force, submitter, priorityOffset);
    } catch (Throwable e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e.getLocalizedMessage(), e);
    }
}
Example #9
Source File: CubeBuildingCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private void run(String cubeName, long startDate, long endDate, String buildType) throws IOException, JobException {
    CubeInstance cube = cubeManager.getCube(cubeName);
    Preconditions.checkArgument(cube != null, "Cube named " + cubeName + " does not exist!!!");
    CubeBuildTypeEnum buildTypeEnum = CubeBuildTypeEnum.valueOf(buildType);
    Preconditions.checkArgument(buildTypeEnum != null, "Build type named " + buildType + " does not exist!!!");
    submitJob(cube, new TSRange(startDate, endDate), buildTypeEnum, false, "SYSTEM");
}
Example #10
Source File: JobInstanceExtractor.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private JobInstance parseToJobInstance(CubingJob cubeJob, Map<String, Output> outputs) {
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));

    Output output = outputs.get(cubeJob.getId());
    final JobInstance result = new JobInstance();
    result.setName(cubeJob.getName());
    result.setProjectName(cubeJob.getProjectName());
    if (cube != null) {
        result.setRelatedCube(cube.getName());
        result.setDisplayCubeName(cube.getDisplayName());
    } else {
        String cubeName = CubingExecutableUtil.getCubeName(cubeJob.getParams());
        result.setRelatedCube(cubeName);
        result.setDisplayCubeName(cubeName);
    }
    result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
    result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(cubeJob.getParams()));
    result.setLastModified(output.getLastModified());
    result.setSubmitter(cubeJob.getSubmitter());
    result.setUuid(cubeJob.getId());
    result.setType(CubeBuildTypeEnum.BUILD);
    result.setStatus(parseToJobStatus(output.getState()));
    result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setExecStartTime(AbstractExecutable.getStartTime(output));
    result.setExecEndTime(AbstractExecutable.getEndTime(output));
    result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
    result.setDuration(AbstractExecutable.getDuration(AbstractExecutable.getStartTime(output), AbstractExecutable.getEndTime(output), AbstractExecutable.getInterruptTime(output)) / 1000);
    for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
        AbstractExecutable task = cubeJob.getTasks().get(i);
        result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
    }
    return result;
}
Example #11
Source File: CubeBuildingCLI.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private void run(String cubeName, long startDate, long endDate, String buildType) throws IOException, JobException {
    CubeInstance cube = cubeManager.getCube(cubeName);
    Preconditions.checkArgument(cube != null, "Cube named " + cubeName + " does not exist!!!");
    CubeBuildTypeEnum buildTypeEnum = CubeBuildTypeEnum.valueOf(buildType);
    Preconditions.checkArgument(buildTypeEnum != null, "Build type named " + buildType + " does not exist!!!");
    submitJob(cube, new TSRange(startDate, endDate), buildTypeEnum, false, "SYSTEM");
}
Example #12
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected JobInstance getLookupSnapshotBuildJobInstance(LookupSnapshotBuildJob job) {
    if (job == null) {
        return null;
    }
    Output output = job.getOutput();
    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setProjectName(job.getProjectName());
    result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
    result.setRelatedSegment(CubingExecutableUtil.getSegmentId(job.getParams()));
    result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(job.getParams()));
    result.setLastModified(job.getLastModified());
    result.setSubmitter(job.getSubmitter());
    result.setUuid(job.getId());
    result.setExecStartTime(job.getStartTime());
    result.setExecEndTime(job.getEndTime());
    result.setExecInterruptTime(job.getInterruptTime());
    result.setType(CubeBuildTypeEnum.BUILD);
    result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setDuration(job.getDuration() / 1000);
    for (int i = 0; i < job.getTasks().size(); ++i) {
        AbstractExecutable task = job.getTasks().get(i);
        result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
    }
    return result;
}
Example #13
Source File: JobInstanceExtractor.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private JobInstance parseToJobInstance(CubingJob cubeJob, Map<String, Output> outputs) {
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));

    Output output = outputs.get(cubeJob.getId());
    final JobInstance result = new JobInstance();
    result.setName(cubeJob.getName());
    result.setProjectName(cubeJob.getProjectName());
    if (cube != null) {
        result.setRelatedCube(cube.getName());
        result.setDisplayCubeName(cube.getDisplayName());
    } else {
        String cubeName = CubingExecutableUtil.getCubeName(cubeJob.getParams());
        result.setRelatedCube(cubeName);
        result.setDisplayCubeName(cubeName);
    }
    result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
    result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(cubeJob.getParams()));
    result.setLastModified(output.getLastModified());
    result.setSubmitter(cubeJob.getSubmitter());
    result.setUuid(cubeJob.getId());
    result.setType(CubeBuildTypeEnum.BUILD);
    result.setStatus(parseToJobStatus(output.getState()));
    result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setExecStartTime(AbstractExecutable.getStartTime(output));
    result.setExecEndTime(AbstractExecutable.getEndTime(output));
    result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
    result.setDuration(AbstractExecutable.getDuration(AbstractExecutable.getStartTime(output), AbstractExecutable.getEndTime(output), AbstractExecutable.getInterruptTime(output)) / 1000);
    for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
        AbstractExecutable task = cubeJob.getTasks().get(i);
        result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
    }
    return result;
}
Example #14
Source File: JobInfoConverter.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public static JobInstance parseToJobInstance(CheckpointExecutable job, Map<String, Output> outputs) {
    if (job == null) {
        logger.warn("job is null.");
        return null;
    }

    Output output = outputs.get(job.getId());
    if (output == null) {
        logger.warn("job output is null.");
        return null;
    }

    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setProjectName(job.getProjectName());
    result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
    result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams()));
    result.setLastModified(output.getLastModified());
    result.setSubmitter(job.getSubmitter());
    result.setUuid(job.getId());
    result.setType(CubeBuildTypeEnum.CHECKPOINT);
    result.setStatus(parseToJobStatus(output.getState()));
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setExecStartTime(AbstractExecutable.getStartTime(output));
    result.setExecEndTime(AbstractExecutable.getEndTime(output));
    result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
    result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(),
            result.getExecInterruptTime()) / 1000);
    for (int i = 0; i < job.getTasks().size(); ++i) {
        AbstractExecutable task = job.getTasks().get(i);
        result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
    }
    return result;
}
Example #15
Source File: CubeController.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private JobInstance buildInternal(String cubeName, TSRange tsRange, SegmentRange segRange, //
        Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd,
        String buildType, boolean force, Integer priorityOffset) {
    try {
        String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
        CubeInstance cube = jobService.getCubeManager().getCube(cubeName);

        checkBuildingSegment(cube);
        return jobService.submitJob(cube, tsRange, segRange, sourcePartitionOffsetStart, sourcePartitionOffsetEnd,
                CubeBuildTypeEnum.valueOf(buildType), force, submitter, priorityOffset);
    } catch (Throwable e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e.getLocalizedMessage(), e);
    }
}
Example #16
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public JobInstance submitJob(CubeInstance cube, TSRange tsRange, SegmentRange segRange, //
        Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd,
        CubeBuildTypeEnum buildType, boolean force, String submitter, Integer priorityOffset) throws IOException {
    aclEvaluate.checkProjectOperationPermission(cube);
    JobInstance jobInstance = submitJobInternal(cube, tsRange, segRange, sourcePartitionOffsetStart,
            sourcePartitionOffsetEnd, buildType, force, submitter, priorityOffset);

    return jobInstance;
}
Example #17
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
protected JobInstance getCheckpointJobInstance(AbstractExecutable job) {
    Message msg = MsgPicker.getMsg();

    if (job == null) {
        return null;
    }
    if (!(job instanceof CheckpointExecutable)) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_JOB_TYPE(), job.getId()));
    }

    CheckpointExecutable checkpointExecutable = (CheckpointExecutable) job;
    Output output = checkpointExecutable.getOutput();
    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setProjectName(checkpointExecutable.getProjectName());
    result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
    result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams()));
    result.setLastModified(job.getLastModified());
    result.setSubmitter(job.getSubmitter());
    result.setUuid(job.getId());
    result.setExecStartTime(job.getStartTime());
    result.setExecEndTime(job.getEndTime());
    result.setExecInterruptTime(job.getInterruptTime());
    result.setType(CubeBuildTypeEnum.CHECKPOINT);
    result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setDuration(job.getDuration() / 1000);
    for (int i = 0; i < checkpointExecutable.getTasks().size(); ++i) {
        AbstractExecutable task = checkpointExecutable.getTasks().get(i);
        result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
    }
    return result;
}
Example #18
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
protected JobInstance getSingleJobInstance(AbstractExecutable job) {
    Message msg = MsgPicker.getMsg();

    if (job == null) {
        return null;
    }

    DefaultChainedExecutable cubeJob;

    if (job instanceof CubingJob) {
        cubeJob = (CubingJob) job;
    } else {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_JOB_TYPE(), job.getId()));
    }

    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
    Output output = cubeJob.getOutput();
    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setProjectName(cubeJob.getProjectName());
    if (cube != null) {
        result.setRelatedCube(cube.getName());
        result.setDisplayCubeName(cube.getDisplayName());
    } else {
        String cubeName = CubingExecutableUtil.getCubeName(cubeJob.getParams());
        result.setRelatedCube(cubeName);
        result.setDisplayCubeName(cubeName);
    }
    result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
    result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(cubeJob.getParams()));
    result.setLastModified(cubeJob.getLastModified());
    result.setSubmitter(cubeJob.getSubmitter());
    result.setUuid(cubeJob.getId());
    result.setExecStartTime(cubeJob.getStartTime());
    result.setExecEndTime(cubeJob.getEndTime());
    result.setExecInterruptTime(cubeJob.getInterruptTime());
    result.setType(CubeBuildTypeEnum.BUILD);
    result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
    result.setMrWaiting(cubeJob.getMapReduceWaitTime() / 1000);
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setDuration(cubeJob.getDuration() / 1000);
    for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
        AbstractExecutable task = cubeJob.getTasks().get(i);
        result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
    }
    return result;
}
Example #19
Source File: JobInstance.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public CubeBuildTypeEnum getType() {
    return type;
}
Example #20
Source File: JobInstance.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public void setType(CubeBuildTypeEnum type) {
    this.type = type;
}
Example #21
Source File: JobInstance.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public void setType(CubeBuildTypeEnum type) {
    this.type = type;
}
Example #22
Source File: JobInstance.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public CubeBuildTypeEnum getType() {
    return type;
}
Example #23
Source File: JobService.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public JobInstance submitJobInternal(CubeInstance cube, TSRange tsRange, SegmentRange segRange, //
        Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd, //
        CubeBuildTypeEnum buildType, boolean force, String submitter, Integer priorityOffset) throws IOException {
    Message msg = MsgPicker.getMsg();

    if (cube.getStatus() == RealizationStatusEnum.DESCBROKEN) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getBUILD_BROKEN_CUBE(), cube.getName()));
    }

    checkCubeDescSignature(cube);
    checkAllowBuilding(cube);

    if (buildType == CubeBuildTypeEnum.BUILD || buildType == CubeBuildTypeEnum.REFRESH) {
        checkAllowParallelBuilding(cube);
    }

    DefaultChainedExecutable job;

    CubeSegment newSeg = null;
    try {
        if (buildType == CubeBuildTypeEnum.BUILD) {
            ISource source = SourceManager.getSource(cube);
            SourcePartition src = new SourcePartition(tsRange, segRange, sourcePartitionOffsetStart,
                    sourcePartitionOffsetEnd);
            src = source.enrichSourcePartitionBeforeBuild(cube, src);
            newSeg = getCubeManager().appendSegment(cube, src);
            job = EngineFactory.createBatchCubingJob(newSeg, submitter, priorityOffset);
        } else if (buildType == CubeBuildTypeEnum.MERGE) {
            newSeg = getCubeManager().mergeSegments(cube, tsRange, segRange, force);
            job = EngineFactory.createBatchMergeJob(newSeg, submitter);
        } else if (buildType == CubeBuildTypeEnum.REFRESH) {
            newSeg = getCubeManager().refreshSegment(cube, tsRange, segRange);
            job = EngineFactory.createBatchCubingJob(newSeg, submitter, priorityOffset);
        } else {
            throw new BadRequestException(String.format(Locale.ROOT, msg.getINVALID_BUILD_TYPE(), buildType));
        }

        getExecutableManager().addJob(job);

    } catch (Exception e) {
        if (newSeg != null) {
            logger.error("Job submission might failed for NEW segment {}, will clean the NEW segment from cube",
                    newSeg.getName());
            try {
                // Remove this segment
                getCubeManager().updateCubeDropSegments(cube, newSeg);
            } catch (Exception ee) {
                // swallow the exception
                logger.error("Clean New segment failed, ignoring it", e);
            }
        }
        throw e;
    }

    JobInstance jobInstance = getSingleJobInstance(job);

    return jobInstance;
}
Example #24
Source File: JobService.java    From kylin with Apache License 2.0 4 votes vote down vote up 
protected JobInstance getSingleJobInstance(AbstractExecutable job) {
    Message msg = MsgPicker.getMsg();

    if (job == null) {
        return null;
    }
    if (!(job instanceof CubingJob)) {
        throw new BadRequestException(String.format(Locale.ROOT, msg.getILLEGAL_JOB_TYPE(), job.getId()));
    }

    CubingJob cubeJob = (CubingJob) job;
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));
    Output output = cubeJob.getOutput();
    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setProjectName(cubeJob.getProjectName());
    if (cube != null) {
        result.setRelatedCube(cube.getName());
        result.setDisplayCubeName(cube.getDisplayName());
    } else {
        String cubeName = CubingExecutableUtil.getCubeName(cubeJob.getParams());
        result.setRelatedCube(cubeName);
        result.setDisplayCubeName(cubeName);
    }
    result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
    result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(cubeJob.getParams()));
    result.setLastModified(cubeJob.getLastModified());
    result.setSubmitter(cubeJob.getSubmitter());
    result.setUuid(cubeJob.getId());
    result.setExecStartTime(cubeJob.getStartTime());
    result.setExecEndTime(cubeJob.getEndTime());
    result.setExecInterruptTime(cubeJob.getInterruptTime());
    result.setType(CubeBuildTypeEnum.BUILD);
    result.setStatus(JobInfoConverter.parseToJobStatus(job.getStatus()));
    result.setMrWaiting(cubeJob.getMapReduceWaitTime() / 1000);
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setDuration(cubeJob.getDuration() / 1000);
    for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
        AbstractExecutable task = cubeJob.getTasks().get(i);
        result.addStep(JobInfoConverter.parseToJobStep(task, i, getExecutableManager().getOutput(task.getId())));
    }
    return result;
}
Example #25
Source File: JobInfoConverter.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static JobInstance parseToJobInstance(CubingJob job, Map<String, Output> outputs) {
    if (job == null) {
        logger.warn("job is null.");
        return null;
    }

    Output output = outputs.get(job.getId());
    if (output == null) {
        logger.warn("job output is null.");
        return null;
    }

    CubingJob cubeJob = job;
    String cubeName = CubingExecutableUtil.getCubeName(cubeJob.getParams());
    String displayCubeName = cubeName;
    try {
        CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()).getCube(cubeName);
        if (cube != null) {
            cubeName = cube.getName();
            displayCubeName = cube.getDisplayName();
        }
    } catch (Exception e) {
        logger.warn("Fail to get cube instance for {}.", cubeName);
    }

    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setProjectName(cubeJob.getProjectName());
    result.setRelatedCube(cubeName);
    result.setDisplayCubeName(displayCubeName);
    result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
    result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(cubeJob.getParams()));
    result.setLastModified(output.getLastModified());
    result.setSubmitter(job.getSubmitter());
    result.setUuid(job.getId());
    result.setType(CubeBuildTypeEnum.BUILD);
    result.setStatus(parseToJobStatus(output.getState()));
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
    result.setExecStartTime(AbstractExecutable.getStartTime(output));
    result.setExecEndTime(AbstractExecutable.getEndTime(output));
    result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
    result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(),
            result.getExecInterruptTime()) / 1000);
    for (int i = 0; i < job.getTasks().size(); ++i) {
        AbstractExecutable task = job.getTasks().get(i);
        result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
    }
    return result;
}
Example #26
Source File: JobInfoConverter.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static JobInstance parseToJobInstance(CubingJob job, Map<String, Output> outputs) {
    if (job == null) {
        logger.warn("job is null.");
        return null;
    }

    Output output = outputs.get(job.getId());
    if (output == null) {
        logger.warn("job output is null.");
        return null;
    }

    CubingJob cubeJob = (CubingJob) job;
    CubeInstance cube = CubeManager.getInstance(KylinConfig.getInstanceFromEnv())
            .getCube(CubingExecutableUtil.getCubeName(cubeJob.getParams()));

    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setProjectName(cubeJob.getProjectName());
    result.setRelatedCube(cube != null ? cube.getName() : CubingExecutableUtil.getCubeName(cubeJob.getParams()));
    result.setDisplayCubeName(cube != null ? cube.getDisplayName() : CubingExecutableUtil.getCubeName(cubeJob.getParams()));
    result.setRelatedSegment(CubingExecutableUtil.getSegmentId(cubeJob.getParams()));
    result.setRelatedSegmentName(CubingExecutableUtil.getSegmentName(cubeJob.getParams()));
    result.setLastModified(output.getLastModified());
    result.setSubmitter(job.getSubmitter());
    result.setUuid(job.getId());
    result.setType(CubeBuildTypeEnum.BUILD);
    result.setStatus(parseToJobStatus(output.getState()));
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setMrWaiting(AbstractExecutable.getExtraInfoAsLong(output, CubingJob.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
    result.setExecStartTime(AbstractExecutable.getStartTime(output));
    result.setExecEndTime(AbstractExecutable.getEndTime(output));
    result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
    result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(),
            result.getExecInterruptTime()) / 1000);
    for (int i = 0; i < job.getTasks().size(); ++i) {
        AbstractExecutable task = job.getTasks().get(i);
        result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
    }
    return result;
}
Example #27
Source File: JobInstance.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
public CubeBuildTypeEnum getType() {
    return type;
}
Example #28
Source File: JobInstance.java    From Kylin with Apache License 2.0 4 votes vote down vote up 
public void setType(CubeBuildTypeEnum type) {
    this.type = type;
}