org.apache.kylin.job.JobInstance Java Examples

The following examples show how to use org.apache.kylin.job.JobInstance. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JobInstanceExtractor.java    From kylin with Apache License 2.0 6 votes vote down vote up
private List<JobInstance> listJobInstances(String project, String cube, long startTime, long endTime) {
    final List<JobInstance> result = Lists.newArrayList();
    final List<AbstractExecutable> executables = executableManager.getAllExecutables(startTime, endTime);
    final Map<String, Output> allOutputs = executableManager.getAllOutputs();
    for (AbstractExecutable executable : executables) {
        if (executable instanceof CubingJob) {
            String cubeName = CubingExecutableUtil.getCubeName(executable.getParams());
            boolean shouldExtract = false;
            if (cube == null || cube.equalsIgnoreCase(cubeName)) {
                if (project == null) {
                    shouldExtract = true;
                } else {
                    ProjectInstance projectInstance = projectManager.getProject(project);
                    if (projectInstance != null && projectInstance.containsRealization(RealizationType.CUBE, cubeName)) {
                        shouldExtract = true;
                    }
                }
            }

            if (shouldExtract) {
                result.add(parseToJobInstance((CubingJob) executable, allOutputs));
            }
        }
    }
    return result;
}
 
Example #2
Source File: JobInstanceExtractor.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output stepOutput) {
    Preconditions.checkNotNull(stepOutput);
    JobInstance.JobStep result = new JobInstance.JobStep();
    result.setId(task.getId());
    result.setName(task.getName());
    result.setSequenceID(i);
    result.setStatus(parseToJobStepStatus(stepOutput.getState()));
    for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) {
        if (entry.getKey() != null && entry.getValue() != null) {
            result.putInfo(entry.getKey(), entry.getValue());
        }
    }
    result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput));
    result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput));
    if (task instanceof ShellExecutable) {
        result.setExecCmd(((ShellExecutable) task).getCmd());
    }
    if (task instanceof MapReduceExecutable) {
        result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
        result.setExecWaitTime(AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
    }
    if (task instanceof HadoopShellExecutable) {
        result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
    }
    return result;
}
 
Example #3
Source File: JobController.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * Resume a cube job
 * 
 * @return
 * @throws IOException
 */
@RequestMapping(value = "/{jobId}/resume", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance resume(@PathVariable String jobId) {
    try {
        final JobInstance jobInstance = jobService.getJobInstance(jobId);
        if (jobInstance == null) {
            throw new BadRequestException("Cannot find job: " + jobId);
        }
        jobService.resumeJob(jobInstance);
        return jobService.getJobInstance(jobId);
    } catch (Exception e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e);
    }
}
 
Example #4
Source File: AclEntityFactory.java    From Kylin with Apache License 2.0 6 votes vote down vote up
public static RootPersistentEntity createAclEntity(String entityType, String uuid) {
    if ("CubeInstance".equals(entityType)) {
        CubeInstance cubeInstance = new CubeInstance();
        cubeInstance.setUuid(uuid);

        return cubeInstance;
    }

    if ("JobInstance".equals(entityType)) {
        JobInstance jobInstance = new JobInstance();
        jobInstance.setUuid(uuid);

        return jobInstance;
    }

    if ("ProjectInstance".equals(entityType)) {
        ProjectInstance projectInstance = new ProjectInstance();
        projectInstance.setUuid(uuid);

        return projectInstance;
    }

    throw new RuntimeException("Unsupported entity type!");
}
 
Example #5
Source File: JobController.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * Drop a cube job
 *
 * @return
 * @throws IOException
 */
@RequestMapping(value = "/{jobId}/drop", method = { RequestMethod.DELETE }, produces = { "application/json" })
@ResponseBody
public JobInstance dropJob(@PathVariable String jobId) {
    JobInstance jobInstance = null;
    try {
        jobInstance = jobService.getJobInstance(jobId);
        if (jobInstance == null) {
            throw new BadRequestException("Cannot find job: " + jobId);
        }
        jobService.dropJob(jobInstance);
    } catch (Exception e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e);
    }

    return jobInstance;
}
 
Example #6
Source File: JobService.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private JobInstance parseToJobInstance(AbstractExecutable job) {
    if (job == null) {
        return null;
    }
    Preconditions.checkState(job instanceof CubingJob, "illegal job type, id:" + job.getId());
    CubingJob cubeJob = (CubingJob) job;
    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setRelatedCube(cubeJob.getCubeName());
    result.setRelatedSegment(cubeJob.getSegmentId());
    result.setLastModified(cubeJob.getLastModified());
    result.setSubmitter(cubeJob.getSubmitter());
    result.setUuid(cubeJob.getId());
    result.setType(CubeBuildTypeEnum.BUILD);
    result.setStatus(parseToJobStatus(job.getStatus()));
    result.setMrWaiting(cubeJob.getMapReduceWaitTime() / 1000);
    result.setDuration(cubeJob.getDuration() / 1000);
    for (int i = 0; i < cubeJob.getTasks().size(); ++i) {
        AbstractExecutable task = cubeJob.getTasks().get(i);
        result.addStep(parseToJobStep(task, i));
    }
    return result;
}
 
Example #7
Source File: CubeController.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * Force rebuild a cube's lookup table snapshot
 *
 * @throws IOException
 */
@RequestMapping(value = "/{cubeName}/refresh_lookup", method = { RequestMethod.PUT }, produces = {
        "application/json" })
@ResponseBody
public JobInstance rebuildLookupSnapshot(@PathVariable String cubeName,
        @RequestBody LookupSnapshotBuildRequest request) {
    try {
        final CubeManager cubeMgr = cubeService.getCubeManager();
        final CubeInstance cube = cubeMgr.getCube(cubeName);
        String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
        return jobService.submitLookupSnapshotJob(cube, request.getLookupTableName(), request.getSegmentIDs(),
                submitter);
    } catch (IOException e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e.getLocalizedMessage(), e);
    }
}
 
Example #8
Source File: JobInstanceExtractor.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private List<JobInstance> listJobInstances(String project, String cube, long startTime, long endTime) {
    final List<JobInstance> result = Lists.newArrayList();
    final List<AbstractExecutable> executables = executableManager.getAllExecutables(startTime, endTime);
    final Map<String, Output> allOutputs = executableManager.getAllOutputs();
    for (AbstractExecutable executable : executables) {
        if (executable instanceof CubingJob) {
            String cubeName = CubingExecutableUtil.getCubeName(executable.getParams());
            boolean shouldExtract = false;
            if (cube == null || cube.equalsIgnoreCase(cubeName)) {
                if (project == null) {
                    shouldExtract = true;
                } else {
                    ProjectInstance projectInstance = projectManager.getProject(project);
                    if (projectInstance != null && projectInstance.containsRealization(RealizationType.CUBE, cubeName)) {
                        shouldExtract = true;
                    }
                }
            }

            if (shouldExtract) {
                result.add(parseToJobInstance((CubingJob) executable, allOutputs));
            }
        }
    }
    return result;
}
 
Example #9
Source File: JobInstanceExtractor.java    From kylin with Apache License 2.0 6 votes vote down vote up
private JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i, Output stepOutput) {
    Preconditions.checkNotNull(stepOutput);
    JobInstance.JobStep result = new JobInstance.JobStep();
    result.setId(task.getId());
    result.setName(task.getName());
    result.setSequenceID(i);
    result.setStatus(parseToJobStepStatus(stepOutput.getState()));
    for (Map.Entry<String, String> entry : stepOutput.getExtra().entrySet()) {
        if (entry.getKey() != null && entry.getValue() != null) {
            result.putInfo(entry.getKey(), entry.getValue());
        }
    }
    result.setExecStartTime(AbstractExecutable.getStartTime(stepOutput));
    result.setExecEndTime(AbstractExecutable.getEndTime(stepOutput));
    if (task instanceof ShellExecutable) {
        result.setExecCmd(((ShellExecutable) task).getCmd());
    }
    if (task instanceof MapReduceExecutable) {
        result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
        result.setExecWaitTime(AbstractExecutable.getExtraInfoAsLong(stepOutput, MapReduceExecutable.MAP_REDUCE_WAIT_TIME, 0L) / 1000);
    }
    if (task instanceof HadoopShellExecutable) {
        result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
    }
    return result;
}
 
Example #10
Source File: JobService.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private JobInstance.JobStep parseToJobStep(AbstractExecutable task, int i) {
    JobInstance.JobStep result = new JobInstance.JobStep();
    result.setId(task.getId());
    result.setName(task.getName());
    result.setSequenceID(i);
    result.setStatus(parseToJobStepStatus(task.getStatus()));
    final Output output = getExecutableManager().getOutput(task.getId());
    for (Map.Entry<String, String> entry : output.getExtra().entrySet()) {
        if (entry.getKey() != null && entry.getValue() != null) {
            result.putInfo(entry.getKey(), entry.getValue());
        }
    }
    result.setExecStartTime(task.getStartTime());
    result.setExecEndTime(task.getEndTime());
    if (task instanceof ShellExecutable) {
        result.setExecCmd(((ShellExecutable) task).getCmd());
    }
    if (task instanceof MapReduceExecutable) {
        result.setExecCmd(((MapReduceExecutable) task).getMapReduceParams());
        result.setExecWaitTime(((MapReduceExecutable) task).getMapReduceWaitTime() / 1000);
    }
    if (task instanceof HadoopShellExecutable) {
        result.setExecCmd(((HadoopShellExecutable) task).getJobParams());
    }
    return result;
}
 
Example #11
Source File: JobInstanceExtractor.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
protected void executeExtract(OptionsHelper optionsHelper, File exportDir) throws Exception {
    String cube = optionsHelper.hasOption(OPTION_CUBE) ? optionsHelper.getOptionValue(OPTION_CUBE) : null;
    String project = optionsHelper.hasOption(OPTION_PROJECT) ? optionsHelper.getOptionValue(OPTION_PROJECT) : null;
    int period = optionsHelper.hasOption(OPTION_PERIOD) ? Integer.parseInt(optionsHelper.getOptionValue(OPTION_PERIOD)) : DEFAULT_PERIOD;
    // maybe use start time and end time to instead of period is better
    long endTime = System.currentTimeMillis();
    long startTime = endTime - period * 24 * 3600 * 1000; // time in Millis
    List<JobInstance> jobInstances = listJobInstances(project, cube, startTime, endTime);
    logger.info("There are {} jobInstances to extract.", jobInstances.size());

    ObjectMapper mapper = new ObjectMapper();
    for (JobInstance jobInstance : jobInstances) {
        mapper.writeValue(new File(exportDir, jobInstance.getUuid() + ".json"), jobInstance);
    }
}
 
Example #12
Source File: JobController.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
/**
 * Resume a cube job
 * 
 * @return
 * @throws IOException
 */
@RequestMapping(value = "/{jobId}/resume", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance resume(@PathVariable String jobId) {
    try {
        final JobInstance jobInstance = jobService.getJobInstance(jobId);
        if (jobInstance == null) {
            throw new BadRequestException("Cannot find job: " + jobId);
        }
        jobService.resumeJob(jobInstance);
        return jobService.getJobInstance(jobId);
    } catch (Exception e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e);
    }
}
 
Example #13
Source File: JobController.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * Rollback a job to the given step
 *
 * @return
 * @throws IOException
 */
@RequestMapping(value = "/{jobId}/steps/{stepId}/rollback", method = { RequestMethod.PUT }, produces = {
        "application/json" })
@ResponseBody
public JobInstance rollback(@PathVariable String jobId, @PathVariable String stepId) {
    try {
        final JobInstance jobInstance = jobService.getJobInstance(jobId);
        if (jobInstance == null) {
            throw new BadRequestException("Cannot find job: " + jobId);
        }
        jobService.rollbackJob(jobInstance, stepId);
        return jobService.getJobInstance(jobId);
    } catch (Exception e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e);
    }
}
 
Example #14
Source File: JobController.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
/**
 * Pause a job
 *
 * @return
 * @throws IOException
 */
@RequestMapping(value = "/{jobId}/pause", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance pause(@PathVariable String jobId) {

    try {
        final JobInstance jobInstance = jobService.getJobInstance(jobId);
        if (jobInstance == null) {
            throw new BadRequestException("Cannot find job: " + jobId);
        }
        jobService.pauseJob(jobInstance);
        return jobService.getJobInstance(jobId);
    } catch (Exception e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e);
    }

}
 
Example #15
Source File: JobController.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
/**
 * Rollback a job to the given step
 *
 * @return
 * @throws IOException
 */
@RequestMapping(value = "/{jobId}/steps/{stepId}/rollback", method = { RequestMethod.PUT }, produces = {
        "application/json" })
@ResponseBody
public JobInstance rollback(@PathVariable String jobId, @PathVariable String stepId) {
    try {
        final JobInstance jobInstance = jobService.getJobInstance(jobId);
        if (jobInstance == null) {
            throw new BadRequestException("Cannot find job: " + jobId);
        }
        jobService.rollbackJob(jobInstance, stepId);
        return jobService.getJobInstance(jobId);
    } catch (Exception e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e);
    }
}
 
Example #16
Source File: JobService.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private List<JobInstance> listCubeJobInstance(final String cubeName, final String projectName, List<JobStatusEnum> statusList) {
    Set<ExecutableState> states;
    if (statusList == null || statusList.isEmpty()) {
        states = EnumSet.allOf(ExecutableState.class);
    } else {
        states = Sets.newHashSet();
        for (JobStatusEnum status : statusList) {
            states.add(parseToExecutableState(status));
        }
    }
    return Lists.newArrayList(FluentIterable.from(listAllCubingJobs(cubeName, projectName, states)).transform(new Function<CubingJob, JobInstance>() {
        @Override
        public JobInstance apply(CubingJob cubingJob) {
            return parseToJobInstance(cubingJob);
        }
    }));
}
 
Example #17
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public List<JobInstance> innerSearchCheckpointJobs(final String cubeName, final String jobName,
        final String projectName, final List<JobStatusEnum> statusList, final JobTimeFilterEnum timeFilter) {
    // TODO: use cache of jobs for this method
    // prepare time range
    Calendar calendar = Calendar.getInstance(TimeZone.getDefault(), Locale.ROOT);
    calendar.setTime(new Date());
    long timeStartInMillis = getTimeStartInMillis(calendar, timeFilter);
    long timeEndInMillis = Long.MAX_VALUE;
    Set<ExecutableState> states = convertStatusEnumToStates(statusList);
    final Map<String, Output> allOutputs = getExecutableManager().getAllOutputs(timeStartInMillis, timeEndInMillis);

    return Lists
            .newArrayList(FluentIterable
                    .from(innerSearchCheckpointJobs(cubeName, jobName, states, timeStartInMillis, timeEndInMillis,
                            allOutputs, false, projectName))
                    .transform(new Function<CheckpointExecutable, JobInstance>() {
                        @Override
                        public JobInstance apply(CheckpointExecutable checkpointExecutable) {
                            return JobInfoConverter.parseToJobInstanceQuietly(checkpointExecutable, allOutputs);
                        }
                    }));
}
 
Example #18
Source File: JobController.java    From Kylin with Apache License 2.0 6 votes vote down vote up
/**
 * get all cube jobs
 * 
 * @return
 * @throws IOException
 */
@RequestMapping(value = "", method = { RequestMethod.GET })
@ResponseBody
public List<JobInstance> list(JobListRequest jobRequest) {

    List<JobInstance> jobInstanceList = Collections.emptyList();
    List<JobStatusEnum> statusList = new ArrayList<JobStatusEnum>();

    if (null != jobRequest.getStatus()) {
        for (int status : jobRequest.getStatus()) {
            statusList.add(JobStatusEnum.getByCode(status));
        }
    }

    try {
        jobInstanceList = jobService.listAllJobs(jobRequest.getCubeName(), jobRequest.getProjectName(), statusList, jobRequest.getLimit(), jobRequest.getOffset());
    } catch (Exception e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e);
    }
    return jobInstanceList;
}
 
Example #19
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
/**
* currently only support substring match
*
* @return
*/
public List<JobInstance> searchJobs(final String cubeNameSubstring, final String projectName,
        final List<JobStatusEnum> statusList, final Integer limitValue, final Integer offsetValue,
        final JobTimeFilterEnum timeFilter, JobSearchMode jobSearchMode) {
    Integer limit = (null == limitValue) ? 30 : limitValue;
    Integer offset = (null == offsetValue) ? 0 : offsetValue;
    List<JobInstance> jobs = searchJobsByCubeName(cubeNameSubstring, projectName, statusList, timeFilter,
            jobSearchMode);

    Collections.sort(jobs);

    if (jobs.size() <= offset) {
        return Collections.emptyList();
    }

    if ((jobs.size() - offset) < limit) {
        return jobs.subList(offset, jobs.size());
    }

    return jobs.subList(offset, offset + limit);
}
 
Example #20
Source File: CubeController.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * Send a auto merge cube job
 *
 * @param cubeName Cube ID
 * @return JobInstance of merging cube
 */
@RequestMapping(value = "/{cubeName}/automerge", method = { RequestMethod.PUT })
@ResponseBody
public JobInstance autoMerge(@PathVariable String cubeName) {
    try {
        checkCubeExists(cubeName);

        CubeInstance cube = jobService.getCubeManager().getCube(cubeName);
        aclEvaluate.checkProjectAdminPermission(cube.getProject());

        String jobID = cubeService.mergeCubeSegment(cubeName);
        if (jobID == null) {
            throw new BadRequestException(String.format(Locale.ROOT,
                    "Cube: %s merging is not supported or no segments to merge", cubeName));
        }
        return jobService.getJobInstance(jobID);
    } catch (Exception e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e.getLocalizedMessage());
    }
}
 
Example #21
Source File: JobControllerTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void testBasics() throws IOException, PersistentException {
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc("test_kylin_cube_with_slr_left_join_desc");
    CubeInstance cube = cubeManager.createCube(CUBE_NAME, "DEFAULT", cubeDesc, "test");
    assertNotNull(cube);

    JobListRequest jobRequest = new JobListRequest();
    Assert.assertNotNull(jobSchedulerController.list(jobRequest));

    JobBuildRequest jobBuildRequest = new JobBuildRequest();
    jobBuildRequest.setBuildType("BUILD");
    jobBuildRequest.setStartTime(0L);
    jobBuildRequest.setEndTime(new Date().getTime());
    JobInstance job = cubeController.rebuild(CUBE_NAME, jobBuildRequest);

    Assert.assertNotNull(jobSchedulerController.get(job.getId()));
    executableDAO.deleteJob(job.getId());
    if (cubeManager.getCube(CUBE_NAME) != null) {
        cubeManager.dropCube(CUBE_NAME, false);
    }

    // jobSchedulerController.cancel(job.getId());
}
 
Example #22
Source File: JobInfoConverter.java    From kylin with Apache License 2.0 5 votes vote down vote up
public static JobInstance parseToJobInstanceQuietly(CubingJob job, Map<String, Output> outputs) {
    try {
        return parseToJobInstance(job, outputs);
    } catch (Exception e) {
        logger.error("Failed to parse job instance: uuid={}", job, e);
        return null;
    }
}
 
Example #23
Source File: CubeController.java    From kylin with Apache License 2.0 5 votes vote down vote up
/**
 * Build/Rebuild a cube segment by source offset
 */
@RequestMapping(value = "/{cubeName}/build2", method = { RequestMethod.PUT }, produces = { "application/json" })
@ResponseBody
public JobInstance build2(@PathVariable String cubeName, @RequestBody JobBuildRequest2 req) {
    try {
        Class<?> clazz = Class.forName("org.apache.kafka.clients.consumer.KafkaConsumer");
        if (clazz == null) {
            throw new ClassNotFoundException();
        }
    } catch (ClassNotFoundException e) {
        throw new InternalErrorException("Could not find Kafka dependency");
    }
    return rebuild2(cubeName, req);
}
 
Example #24
Source File: JobController.java    From Kylin with Apache License 2.0 5 votes vote down vote up
/**
 * Get a cube job
 * 
 * @return
 * @throws IOException
 */
@RequestMapping(value = "/{jobId}", method = { RequestMethod.GET })
@ResponseBody
public JobInstance get(@PathVariable String jobId) {
    JobInstance jobInstance = null;
    try {
        jobInstance = jobService.getJobInstance(jobId);
    } catch (Exception e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e);
    }

    return jobInstance;
}
 
Example #25
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public void dropJob(JobInstance job) {
    aclEvaluate.checkProjectOperationPermission(job);
    if (job.getRelatedCube() != null && getCubeManager().getCube(job.getRelatedCube()) != null) {
        if (job.getStatus() != JobStatusEnum.FINISHED && job.getStatus() != JobStatusEnum.DISCARDED) {
            throw new BadRequestException(
                    "Only FINISHED and DISCARDED job can be deleted. Please wait for the job finishing or discard the job!!!");
        }
    }
    getExecutableManager().deleteJob(job.getId());
    logger.info("Delete job [" + job.getId() + "] trigger by + "
            + SecurityContextHolder.getContext().getAuthentication().getName());
}
 
Example #26
Source File: JobService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public void pauseJob(JobInstance job) {
    aclEvaluate.checkProjectOperationPermission(job);
    logger.info("Pause job [" + job.getId() + "] trigger by "
            + SecurityContextHolder.getContext().getAuthentication().getName());
    if (job.getStatus().isComplete()) {
        throw new IllegalStateException(
                "The job " + job.getId() + " has already been finished and cannot be stopped.");
    }
    getExecutableManager().pauseJob(job.getId());
}
 
Example #27
Source File: JobController.java    From Kylin with Apache License 2.0 5 votes vote down vote up
/**
 * Cancel a job
 * 
 * @return
 * @throws IOException
 */
@RequestMapping(value = "/{jobId}/cancel", method = { RequestMethod.PUT })
@ResponseBody
public JobInstance cancel(@PathVariable String jobId) {

    try {
        return jobService.cancelJob(jobId);
    } catch (Exception e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e);
    }

}
 
Example #28
Source File: CubeMigrationCLI.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private static void renameFoldersInHdfs(CubeInstance cube) {
    for (CubeSegment segment : cube.getSegments()) {

        String jobUuid = segment.getLastBuildJobID();
        String src = JobInstance.getJobWorkingDir(jobUuid, srcConfig.getHdfsWorkingDirectory());
        String tgt = JobInstance.getJobWorkingDir(jobUuid, dstConfig.getHdfsWorkingDirectory());

        operations.add(new Opt(OptType.RENAME_FOLDER_IN_HDFS, new Object[] { src, tgt }));
    }

}
 
Example #29
Source File: JobInfoConverter.java    From kylin with Apache License 2.0 5 votes vote down vote up
public static JobInstance parseToJobInstance(CheckpointExecutable job, Map<String, Output> outputs) {
    if (job == null) {
        logger.warn("job is null.");
        return null;
    }

    Output output = outputs.get(job.getId());
    if (output == null) {
        logger.warn("job output is null.");
        return null;
    }

    final JobInstance result = new JobInstance();
    result.setName(job.getName());
    result.setProjectName(job.getProjectName());
    result.setRelatedCube(CubingExecutableUtil.getCubeName(job.getParams()));
    result.setDisplayCubeName(CubingExecutableUtil.getCubeName(job.getParams()));
    result.setLastModified(output.getLastModified());
    result.setSubmitter(job.getSubmitter());
    result.setUuid(job.getId());
    result.setType(CubeBuildTypeEnum.CHECKPOINT);
    result.setStatus(parseToJobStatus(output.getState()));
    result.setBuildInstance(AbstractExecutable.getBuildInstance(output));
    result.setExecStartTime(AbstractExecutable.getStartTime(output));
    result.setExecEndTime(AbstractExecutable.getEndTime(output));
    result.setExecInterruptTime(AbstractExecutable.getInterruptTime(output));
    result.setDuration(AbstractExecutable.getDuration(result.getExecStartTime(), result.getExecEndTime(),
            result.getExecInterruptTime()) / 1000);
    for (int i = 0; i < job.getTasks().size(); ++i) {
        AbstractExecutable task = job.getTasks().get(i);
        result.addStep(parseToJobStep(task, i, outputs.get(task.getId())));
    }
    return result;
}
 
Example #30
Source File: CubeController.java    From kylin with Apache License 2.0 5 votes vote down vote up
private JobInstance buildInternal(String cubeName, TSRange tsRange, SegmentRange segRange, //
        Map<Integer, Long> sourcePartitionOffsetStart, Map<Integer, Long> sourcePartitionOffsetEnd,
        String buildType, boolean force, Integer priorityOffset) {
    try {
        String submitter = SecurityContextHolder.getContext().getAuthentication().getName();
        CubeInstance cube = jobService.getCubeManager().getCube(cubeName);

        checkBuildingSegment(cube);
        return jobService.submitJob(cube, tsRange, segRange, sourcePartitionOffsetStart, sourcePartitionOffsetEnd,
                CubeBuildTypeEnum.valueOf(buildType), force, submitter, priorityOffset);
    } catch (Throwable e) {
        logger.error(e.getLocalizedMessage(), e);
        throw new InternalErrorException(e.getLocalizedMessage(), e);
    }
}