org.apache.kylin.job.dao.ExecutablePO Java Examples

The following examples show how to use org.apache.kylin.job.dao.ExecutablePO. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: YarnLogExtractor.java From kylin with Apache License 2.0

6 votes

public void extractYarnLogAndMRJob(String jobId, File yarnLogDir) throws Exception {
    logger.info("Collecting Yarn logs and MR counters for the Job {}", jobId);
    kylinConfig = KylinConfig.getInstanceFromEnv();
    executableDao = ExecutableDao.getInstance(kylinConfig);
    ExecutablePO executablePO = null;
    executablePO = executableDao.getJob(jobId);

    if (executablePO == null) {
        logger.error("Can not find executablePO.");
        return;
    }

    for (ExecutablePO task : executablePO.getTasks()) {
        yarnLogsResources.add(task.getUuid());
    }

    for (String stepId : yarnLogsResources) {
        logger.info("Checking step {}", stepId);
        extractYarnLog(stepId, new File(yarnLogDir, stepId));
        extractMRJob(stepId, new File(yarnLogDir, stepId));
    }
}

Example #2

Source File: ExecutableManager.java From Kylin with Apache License 2.0

6 votes

private static AbstractExecutable parseTo(ExecutablePO executablePO) {
    if (executablePO == null) {
        return null;
    }
    String type = executablePO.getType();
    try {
        Class<? extends AbstractExecutable> clazz = ClassUtil.forName(type, AbstractExecutable.class);
        Constructor<? extends AbstractExecutable> constructor = clazz.getConstructor();
        AbstractExecutable result = constructor.newInstance();
        result.setId(executablePO.getUuid());
        result.setName(executablePO.getName());
        result.setParams(executablePO.getParams());
        List<ExecutablePO> tasks = executablePO.getTasks();
        if (tasks != null && !tasks.isEmpty()) {
            Preconditions.checkArgument(result instanceof DefaultChainedExecutable);
            for (ExecutablePO subTask: tasks) {
                ((DefaultChainedExecutable) result).addTask(parseTo(subTask));
            }
        }
        return result;
    } catch (ReflectiveOperationException e) {
        throw new IllegalArgumentException("cannot parse this job:" + executablePO.getId(), e);
    }
}

Example #3

Source File: KylinHealthCheckJob.java From kylin with Apache License 2.0

6 votes

private void checkStoppedJob() throws Exception {
    reporter.log("## Cleanup stopped job");
    int staleJobThresholdInDays = config.getStaleJobThresholdInDays();
    long outdatedJobTimeCut = System.currentTimeMillis() - 1L * staleJobThresholdInDays * 24 * 60 * 60 * 1000;
    ExecutableDao executableDao = ExecutableDao.getInstance(config);
    // discard all expired ERROR or STOPPED jobs
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        String jobStatus = executableDao.getJobOutput(executable.getUuid()).getStatus();
        if (lastModified < outdatedJobTimeCut && (ExecutableState.ERROR.toString().equals(jobStatus)
                || ExecutableState.STOPPED.toString().equals(jobStatus))) {
            // ExecutableManager.getInstance(config).discardJob(executable.getId());
            if (executable.getType().equals(CubingJob.class.getName())
                    || executable.getType().equals(CheckpointExecutable.class.getName())) {
                reporter.log("Should discard job: {}, which in ERROR/STOPPED state for {} days", executable.getId(),
                        staleJobThresholdInDays);
            } else {
                logger.warn("Unknown out of date job: {} with type: {}, which in ERROR/STOPPED state for {} days",
                        executable.getId(), executable.getType(), staleJobThresholdInDays);
            }
        }
    }
}

Example #4

Source File: MetadataCleanupJob.java From kylin with Apache License 2.0

6 votes

private boolean isJobComplete(ExecutableDao executableDao, ExecutablePO job) {
    String jobId = job.getUuid();
    boolean isComplete = false;
    try {
        ExecutableOutputPO output = executableDao.getJobOutput(jobId);
        String status = output.getStatus();
        String jobType = job.getType();
        if (jobType.equals(CubingJob.class.getName())
                || jobType.equals(CheckpointExecutable.class.getName())) {
            if (StringUtils.equals(status, ExecutableState.SUCCEED.toString())
                    || StringUtils.equals(status, ExecutableState.DISCARDED.toString())) {
                isComplete = true;
            }
        } else if (jobType.equals(CardinalityExecutable.class.getName())) {
            // Ignore state of DefaultChainedExecutable
            isComplete = true;
        }
    } catch (PersistentException e) {
        logger.error("Get job output failed for job uuid: {}", jobId, e);
        isComplete = true; // job output broken --> will be treat as complete
    }

    return isComplete;
}

Example #5

Source File: ExecutableManager.java From kylin with Apache License 2.0

6 votes

public void forceKillJob(String jobId) {
    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
        List<ExecutablePO> tasks = executableDao.getJob(jobId).getTasks();

        for (ExecutablePO task : tasks) {
            if (executableDao.getJobOutput(task.getId()).getStatus().equals("SUCCEED")) {
                continue;
            } else if (executableDao.getJobOutput(task.getId()).getStatus().equals("RUNNING")) {
                updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), "");
            }
            break;
        }

        if (!jobOutput.getStatus().equals(ExecutableState.ERROR.toString())) {
            jobOutput.setStatus(ExecutableState.ERROR.toString());
            executableDao.updateJobOutput(jobOutput);
        }
    } catch (PersistentException e) {
        throw new RuntimeException(e);
    }
}

Example #6

Source File: ExecutableManager.java From kylin with Apache License 2.0

6 votes

public void updateCheckpointJob(String jobId, List<AbstractExecutable> subTasksForCheck) {
    try {
        jobId = jobId.replaceAll("[./]", "");
        final ExecutablePO job = executableDao.getJob(jobId);
        Preconditions.checkArgument(job != null, "there is no related job for job id:" + jobId);

        List<ExecutablePO> tasksForCheck = Lists.newArrayListWithExpectedSize(subTasksForCheck.size());
        for (AbstractExecutable taskForCheck : subTasksForCheck) {
            tasksForCheck.add(parse(taskForCheck));
        }
        job.setTasksForCheck(tasksForCheck);
        executableDao.updateJob(job);
    } catch (PersistentException e) {
        logger.error("fail to update checkpoint job:" + jobId, e);
        throw new RuntimeException(e);
    }
}

Example #7

Source File: ExecutableManager.java From kylin with Apache License 2.0

6 votes

private static ExecutablePO parse(AbstractExecutable executable) {
    ExecutablePO result = new ExecutablePO();
    result.setName(executable.getName());
    result.setUuid(executable.getId());
    result.setType(executable.getClass().getName());
    result.setParams(executable.getParams());
    result.setPriority(executable.getPriority());
    if (executable instanceof ChainedExecutable) {
        List<ExecutablePO> tasks = Lists.newArrayList();
        for (AbstractExecutable task : ((ChainedExecutable) executable).getTasks()) {
            tasks.add(parse(task));
        }
        result.setTasks(tasks);
    }
    if (executable instanceof CheckpointExecutable) {
        List<ExecutablePO> tasksForCheck = Lists.newArrayList();
        for (AbstractExecutable taskForCheck : ((CheckpointExecutable) executable).getSubTasksForCheck()) {
            tasksForCheck.add(parse(taskForCheck));
        }
        result.setTasksForCheck(tasksForCheck);
    }
    return result;
}

Example #8

Source File: KylinHealthCheckJob.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private void checkStoppedJob() throws Exception {
    reporter.log("## Cleanup stopped job");
    int staleJobThresholdInDays = config.getStaleJobThresholdInDays();
    long outdatedJobTimeCut = System.currentTimeMillis() - 1L * staleJobThresholdInDays * 24 * 60 * 60 * 1000;
    ExecutableDao executableDao = ExecutableDao.getInstance(config);
    // discard all expired ERROR or STOPPED jobs
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        String jobStatus = executableDao.getJobOutput(executable.getUuid()).getStatus();
        if (lastModified < outdatedJobTimeCut && (ExecutableState.ERROR.toString().equals(jobStatus)
                || ExecutableState.STOPPED.toString().equals(jobStatus))) {
            // ExecutableManager.getInstance(config).discardJob(executable.getId());
            if (executable.getType().equals(CubingJob.class.getName())
                    || executable.getType().equals(CheckpointExecutable.class.getName())) {
                reporter.log("Should discard job: {}, which in ERROR/STOPPED state for {} days", executable.getId(),
                        staleJobThresholdInDays);
            } else {
                logger.warn("Unknown out of date job: {} with type: {}, which in ERROR/STOPPED state for {} days",
                        executable.getId(), executable.getType(), staleJobThresholdInDays);
            }
        }
    }
}

Example #9

Source File: MetadataCleanupJob.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private boolean isJobComplete(ExecutableDao executableDao, ExecutablePO job) {
    String jobId = job.getUuid();
    boolean isComplete = false;
    try {
        ExecutableOutputPO output = executableDao.getJobOutput(jobId);
        String status = output.getStatus();
        String jobType = job.getType();
        if (jobType.equals(CubingJob.class.getName())
                || jobType.equals(CheckpointExecutable.class.getName())) {
            if (StringUtils.equals(status, ExecutableState.SUCCEED.toString())
                    || StringUtils.equals(status, ExecutableState.DISCARDED.toString())) {
                isComplete = true;
            }
        } else if (jobType.equals(CardinalityExecutable.class.getName())) {
            // Ignore state of DefaultChainedExecutable
            isComplete = true;
        }
    } catch (PersistentException e) {
        logger.error("Get job output failed for job uuid: {}", jobId, e);
        isComplete = true; // job output broken --> will be treat as complete
    }

    return isComplete;
}

Example #10

Source File: ExecutableManager.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public void forceKillJob(String jobId) {
    try {
        final ExecutableOutputPO jobOutput = executableDao.getJobOutput(jobId);
        List<ExecutablePO> tasks = executableDao.getJob(jobId).getTasks();

        for (ExecutablePO task : tasks) {
            if (executableDao.getJobOutput(task.getId()).getStatus().equals("SUCCEED")) {
                continue;
            } else if (executableDao.getJobOutput(task.getId()).getStatus().equals("RUNNING")) {
                updateJobOutput(task.getId(), ExecutableState.READY, Maps.<String, String> newHashMap(), "");
            }
            break;
        }

        if (!jobOutput.getStatus().equals(ExecutableState.ERROR.toString())) {
            jobOutput.setStatus(ExecutableState.ERROR.toString());
            executableDao.updateJobOutput(jobOutput);
        }
    } catch (PersistentException e) {
        throw new RuntimeException(e);
    }
}

Example #11

Source File: ExecutableManager.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public void updateCheckpointJob(String jobId, List<AbstractExecutable> subTasksForCheck) {
    try {
        jobId = jobId.replaceAll("[./]", "");
        final ExecutablePO job = executableDao.getJob(jobId);
        Preconditions.checkArgument(job != null, "there is no related job for job id:" + jobId);

        List<ExecutablePO> tasksForCheck = Lists.newArrayListWithExpectedSize(subTasksForCheck.size());
        for (AbstractExecutable taskForCheck : subTasksForCheck) {
            tasksForCheck.add(parse(taskForCheck));
        }
        job.setTasksForCheck(tasksForCheck);
        executableDao.updateJob(job);
    } catch (PersistentException e) {
        logger.error("fail to update checkpoint job:" + jobId, e);
        throw new RuntimeException(e);
    }
}

Example #12

Source File: ExecutableManager.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private static ExecutablePO parse(AbstractExecutable executable) {
    ExecutablePO result = new ExecutablePO();
    result.setName(executable.getName());
    result.setUuid(executable.getId());
    result.setType(executable.getClass().getName());
    result.setParams(executable.getParams());
    result.setPriority(executable.getPriority());
    if (executable instanceof ChainedExecutable) {
        List<ExecutablePO> tasks = Lists.newArrayList();
        for (AbstractExecutable task : ((ChainedExecutable) executable).getTasks()) {
            tasks.add(parse(task));
        }
        result.setTasks(tasks);
    }
    if (executable instanceof CheckpointExecutable) {
        List<ExecutablePO> tasksForCheck = Lists.newArrayList();
        for (AbstractExecutable taskForCheck : ((CheckpointExecutable) executable).getSubTasksForCheck()) {
            tasksForCheck.add(parse(taskForCheck));
        }
        result.setTasksForCheck(tasksForCheck);
    }
    return result;
}

Example #13

Source File: YarnLogExtractor.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public void extractYarnLogAndMRJob(String jobId, File yarnLogDir) throws Exception {
    logger.info("Collecting Yarn logs and MR counters for the Job {}", jobId);
    kylinConfig = KylinConfig.getInstanceFromEnv();
    executableDao = ExecutableDao.getInstance(kylinConfig);
    ExecutablePO executablePO = null;
    executablePO = executableDao.getJob(jobId);

    if (executablePO == null) {
        logger.error("Can not find executablePO.");
        return;
    }

    for (ExecutablePO task : executablePO.getTasks()) {
        yarnLogsResources.add(task.getUuid());
    }

    for (String stepId : yarnLogsResources) {
        logger.info("Checking step {}", stepId);
        extractYarnLog(stepId, new File(yarnLogDir, stepId));
        extractMRJob(stepId, new File(yarnLogDir, stepId));
    }
}

Example #14

Source File: ExecutableManager.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private AbstractExecutable parseTo(ExecutablePO executablePO) {
    if (executablePO == null) {
        logger.warn("executablePO is null");
        return null;
    }
    String type = executablePO.getType();
    AbstractExecutable result = newExecutable(type);
    result.initConfig(config);
    result.setId(executablePO.getUuid());
    result.setName(executablePO.getName());
    result.setParams(executablePO.getParams());
    result.setPriority(executablePO.getPriority());

    if (!(result instanceof BrokenExecutable)) {
        List<ExecutablePO> tasks = executablePO.getTasks();
        if (tasks != null && !tasks.isEmpty()) {
            Preconditions.checkArgument(result instanceof ChainedExecutable);
            for (ExecutablePO subTask : tasks) {
                AbstractExecutable subTaskExecutable = parseTo(subTask);
                if (subTaskExecutable != null) {
                    subTaskExecutable.setParentExecutable(result);
                }
                ((ChainedExecutable) result).addTask(parseTo(subTask));
            }
        }
        List<ExecutablePO> tasksForCheck = executablePO.getTasksForCheck();
        if (tasksForCheck != null && !tasksForCheck.isEmpty()) {
            Preconditions.checkArgument(result instanceof CheckpointExecutable);
            for (ExecutablePO subTaskForCheck : tasksForCheck) {
                ((CheckpointExecutable) result).addTaskForCheck(parseTo(subTaskForCheck));
            }
        }
    }

    return result;
}

Example #15

Source File: ExecutableManager.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public List<AbstractExecutable> getAllExecutableDigests(long timeStartInMillis, long timeEndInMillis) {
    List<AbstractExecutable> ret = Lists.newArrayList();
    for (ExecutablePO po : executableDao.getJobDigests(timeStartInMillis, timeEndInMillis)) {
        try {
            AbstractExecutable ae = parseTo(po);
            ret.add(ae);
        } catch (IllegalArgumentException e) {
            logger.error("error parsing one executabePO: ", e);
        }
    }
    return ret;
}

Example #16

Source File: ExecutableManager.java From kylin with Apache License 2.0

5 votes

public List<AbstractExecutable> getAllExecutableDigests(long timeStartInMillis, long timeEndInMillis) {
    List<AbstractExecutable> ret = Lists.newArrayList();
    for (ExecutablePO po : executableDao.getJobDigests(timeStartInMillis, timeEndInMillis)) {
        try {
            AbstractExecutable ae = parseTo(po);
            ret.add(ae);
        } catch (IllegalArgumentException e) {
            logger.error("error parsing one executabePO: ", e);
        }
    }
    return ret;
}

Example #17

Source File: ExecutableManager.java From kylin with Apache License 2.0

5 votes

private AbstractExecutable parseTo(ExecutablePO executablePO) {
    if (executablePO == null) {
        logger.warn("executablePO is null");
        return null;
    }
    String type = executablePO.getType();
    AbstractExecutable result = newExecutable(type);
    result.initConfig(config);
    result.setId(executablePO.getUuid());
    result.setName(executablePO.getName());
    result.setParams(executablePO.getParams());
    result.setPriority(executablePO.getPriority());

    if (!(result instanceof BrokenExecutable)) {
        List<ExecutablePO> tasks = executablePO.getTasks();
        if (tasks != null && !tasks.isEmpty()) {
            Preconditions.checkArgument(result instanceof ChainedExecutable);
            for (ExecutablePO subTask : tasks) {
                AbstractExecutable subTaskExecutable = parseTo(subTask);
                if (subTaskExecutable != null) {
                    subTaskExecutable.setParentExecutable(result);
                }
                ((ChainedExecutable) result).addTask(parseTo(subTask));
            }
        }
        List<ExecutablePO> tasksForCheck = executablePO.getTasksForCheck();
        if (tasksForCheck != null && !tasksForCheck.isEmpty()) {
            Preconditions.checkArgument(result instanceof CheckpointExecutable);
            for (ExecutablePO subTaskForCheck : tasksForCheck) {
                ((CheckpointExecutable) result).addTaskForCheck(parseTo(subTaskForCheck));
            }
        }
    }

    return result;
}

Example #18

Source File: ExecutableManager.java From Kylin with Apache License 2.0

5 votes

public List<AbstractExecutable> getAllExecutables() {
    try {
        return Lists.transform(executableDao.getJobs(), new Function<ExecutablePO, AbstractExecutable>() {
            @Nullable
            @Override
            public AbstractExecutable apply(ExecutablePO input) {
                    return parseTo(input);
            }
        });
    } catch (PersistentException e) {
        logger.error("error get All Jobs", e);
        throw new RuntimeException(e);
    }
}

Example #19

Source File: ExecutableManager.java From Kylin with Apache License 2.0

5 votes

private static ExecutablePO parse(AbstractExecutable executable) {
    ExecutablePO result = new ExecutablePO();
    result.setName(executable.getName());
    result.setUuid(executable.getId());
    result.setType(executable.getClass().getName());
    result.setParams(executable.getParams());
    if (executable instanceof DefaultChainedExecutable) {
        List<ExecutablePO> tasks = Lists.newArrayList();
        for (AbstractExecutable task : ((DefaultChainedExecutable) executable).getTasks()) {
            tasks.add(parse(task));
        }
        result.setTasks(tasks);
    }
    return result;
}

Example #20

Source File: CubeMetaExtractor.java From kylin with Apache License 2.0

4 votes

private void addSegAndJob(CubeInstance cube) {
    if (includeSegments) {
        addRequired(CubeInstance.concatResourcePath(cube.getName()));
        for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
            addRequired(CubeSegment.getStatisticsResourcePath(cube.getName(), segment.getUuid()));
            if (includeSegmentDetails) {
                for (String dictPat : segment.getDictionaryPaths()) {
                    addRequired(dictPat);
                }
                for (String snapshotPath : segment.getSnapshotPaths()) {
                    addRequired(snapshotPath);
                }
            }

            if (includeJobs) {
                String lastJobId = segment.getLastBuildJobID();
                if (StringUtils.isEmpty(lastJobId)) {
                    throw new RuntimeException("No job exist for segment :" + segment);
                } else {
                    try {
                        if (onlyJobOutput) {
                            addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId);
                        } else {
                            ExecutablePO executablePO = executableDao.getJob(lastJobId);
                            addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + lastJobId);
                            addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId);
                            for (ExecutablePO task : executablePO.getTasks()) {
                                addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + task.getUuid());
                                addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
                            }
                        }
                    } catch (PersistentException e) {
                        throw new RuntimeException("PersistentException", e);
                    }
                }
            }
        }
    } else {
        if (includeJobs) {
            logger.warn("It's useless to set includeJobs to true when includeSegments is set to false");
        }
        cube.setStatus(RealizationStatusEnum.DISABLED);
        cubesToTrimAndSave.add(cube);
    }
}

Example #21

Source File: CubeMetaExtractor.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

private void addSegAndJob(CubeInstance cube) {
    if (includeSegments) {
        addRequired(CubeInstance.concatResourcePath(cube.getName()));
        for (CubeSegment segment : cube.getSegments(SegmentStatusEnum.READY)) {
            addRequired(CubeSegment.getStatisticsResourcePath(cube.getName(), segment.getUuid()));
            if (includeSegmentDetails) {
                for (String dictPat : segment.getDictionaryPaths()) {
                    addRequired(dictPat);
                }
                for (String snapshotPath : segment.getSnapshotPaths()) {
                    addRequired(snapshotPath);
                }
            }

            if (includeJobs) {
                String lastJobId = segment.getLastBuildJobID();
                if (StringUtils.isEmpty(lastJobId)) {
                    throw new RuntimeException("No job exist for segment :" + segment);
                } else {
                    try {
                        if (onlyJobOutput) {
                            addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId);
                        } else {
                            ExecutablePO executablePO = executableDao.getJob(lastJobId);
                            addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + lastJobId);
                            addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + lastJobId);
                            for (ExecutablePO task : executablePO.getTasks()) {
                                addRequired(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + task.getUuid());
                                addRequired(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
                            }
                        }
                    } catch (PersistentException e) {
                        throw new RuntimeException("PersistentException", e);
                    }
                }
            }
        }
    } else {
        if (includeJobs) {
            logger.warn("It's useless to set includeJobs to true when includeSegments is set to false");
        }
        cube.setStatus(RealizationStatusEnum.DISABLED);
        cubesToTrimAndSave.add(cube);
    }
}

Example #22

Source File: MetadataCleanupJob.java From kylin with Apache License 2.0

4 votes

public void cleanup() throws Exception {
    CubeManager cubeManager = CubeManager.getInstance(config);

    List<String> toDeleteResource = Lists.newArrayList();

    // two level resources, snapshot tables and cube statistics
    for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) {
        NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot);

        if (snapshotTables != null) {
            for (String snapshotTable : snapshotTables) {
                NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
                if (snapshotNames != null)
                    for (String snapshot : snapshotNames) {
                        if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
                            toDeleteResource.add(snapshot);
                    }
            }
        }
    }

    // three level resources, only dictionaries
    NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);

    if (dictTables != null) {
        for (String table : dictTables) {
            NavigableSet<String> tableColNames = getStore().listResources(table);
            if (tableColNames != null)
                for (String tableCol : tableColNames) {
                    NavigableSet<String> dictionaries = getStore().listResources(tableCol);
                    if (dictionaries != null)
                        for (String dict : dictionaries)
                            if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
                                toDeleteResource.add(dict);
                }
        }
    }

    Set<String> activeResourceList = Sets.newHashSet();
    for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
        for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
            activeResourceList.addAll(segment.getSnapshotPaths());
            activeResourceList.addAll(segment.getDictionaryPaths());
            activeResourceList.add(segment.getStatisticsResourcePath());
        }
    }

    toDeleteResource.removeAll(activeResourceList);

    // delete old and completed jobs
    ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid());
        if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) {
            toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid());
            toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());

            for (ExecutablePO task : executable.getTasks()) {
                toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
            }
        }
    }

    if (toDeleteResource.size() > 0) {
        logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n");

        for (String s : toDeleteResource) {
            logger.info(s);
            if (delete == true) {
                getStore().deleteResource(s);
            }
        }
    } else {
        logger.info("No resource to be cleaned up from metadata store;");
    }

}

Example #23

Source File: MetadataCleanupJob.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public void cleanup() throws Exception {
    CubeManager cubeManager = CubeManager.getInstance(config);

    List<String> toDeleteResource = Lists.newArrayList();

    // two level resources, snapshot tables and cube statistics
    for (String resourceRoot : new String[] { ResourceStore.SNAPSHOT_RESOURCE_ROOT, ResourceStore.CUBE_STATISTICS_ROOT }) {
        NavigableSet<String> snapshotTables = getStore().listResources(resourceRoot);

        if (snapshotTables != null) {
            for (String snapshotTable : snapshotTables) {
                NavigableSet<String> snapshotNames = getStore().listResources(snapshotTable);
                if (snapshotNames != null)
                    for (String snapshot : snapshotNames) {
                        if (isOlderThanThreshold(getStore().getResourceTimestamp(snapshot)))
                            toDeleteResource.add(snapshot);
                    }
            }
        }
    }

    // three level resources, only dictionaries
    NavigableSet<String> dictTables = getStore().listResources(ResourceStore.DICT_RESOURCE_ROOT);

    if (dictTables != null) {
        for (String table : dictTables) {
            NavigableSet<String> tableColNames = getStore().listResources(table);
            if (tableColNames != null)
                for (String tableCol : tableColNames) {
                    NavigableSet<String> dictionaries = getStore().listResources(tableCol);
                    if (dictionaries != null)
                        for (String dict : dictionaries)
                            if (isOlderThanThreshold(getStore().getResourceTimestamp(dict)))
                                toDeleteResource.add(dict);
                }
        }
    }

    Set<String> activeResourceList = Sets.newHashSet();
    for (org.apache.kylin.cube.CubeInstance cube : cubeManager.listAllCubes()) {
        for (org.apache.kylin.cube.CubeSegment segment : cube.getSegments()) {
            activeResourceList.addAll(segment.getSnapshotPaths());
            activeResourceList.addAll(segment.getDictionaryPaths());
            activeResourceList.add(segment.getStatisticsResourcePath());
        }
    }

    toDeleteResource.removeAll(activeResourceList);

    // delete old and completed jobs
    ExecutableDao executableDao = ExecutableDao.getInstance(KylinConfig.getInstanceFromEnv());
    List<ExecutablePO> allExecutable = executableDao.getJobs();
    for (ExecutablePO executable : allExecutable) {
        long lastModified = executable.getLastModified();
        ExecutableOutputPO output = executableDao.getJobOutput(executable.getUuid());
        if (System.currentTimeMillis() - lastModified > TIME_THREADSHOLD_FOR_JOB && (ExecutableState.SUCCEED.toString().equals(output.getStatus()) || ExecutableState.DISCARDED.toString().equals(output.getStatus()))) {
            toDeleteResource.add(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + executable.getUuid());
            toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + executable.getUuid());

            for (ExecutablePO task : executable.getTasks()) {
                toDeleteResource.add(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + task.getUuid());
            }
        }
    }

    if (toDeleteResource.size() > 0) {
        logger.info("The following resources have no reference or is too old, will be cleaned from metadata store: \n");

        for (String s : toDeleteResource) {
            logger.info(s);
            if (delete == true) {
                getStore().deleteResource(s);
            }
        }
    } else {
        logger.info("No resource to be cleaned up from metadata store;");
    }

}