org.apache.kylin.job.engine.JobEngineConfig Java Examples

The following examples show how to use org.apache.kylin.job.engine.JobEngineConfig. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: StreamingCubingJobBuilder.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private MapReduceExecutable createMergeDictStep(String streamingStoragePath, String jobId, DefaultChainedExecutable jobFlow) {
    MapReduceExecutable mergeDict = new MapReduceExecutable();
    mergeDict.setName(ExecutableConstants.STEP_NAME_STREAMING_CREATE_DICTIONARY);
    StringBuilder cmd = new StringBuilder();

    appendMapReduceParameters(cmd, JobEngineConfig.CUBE_MERGE_JOB_CONF_SUFFIX);
    appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME,
            ExecutableConstants.STEP_NAME_STREAMING_CREATE_DICTIONARY);
    appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, streamingStoragePath);
    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_NAME, seg.getName());
    //Instead of using mr job output, trySaveNewDict api is used, so output path is useless here
    appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getDictPath(jobId));

    final String cubeName = CubingExecutableUtil.getCubeName(jobFlow.getParams());
    mergeDict.setMapReduceParams(cmd.toString());
    mergeDict.setMapReduceJobClass(MergeDictJob.class);
    mergeDict.setLockPathName(cubeName);
    mergeDict.setIsNeedLock(true);
    mergeDict.setIsNeedReleaseLock(false);
    mergeDict.setJobFlowJobId(jobFlow.getId());

    return mergeDict;

}

Example #2

Source File: BuildCubeWithEngine.java From kylin with Apache License 2.0

6 votes

public void before() throws Exception {
    deployEnv();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        AbstractExecutable executable = jobService.getJob(jobId);
        if (executable instanceof CubingJob || executable instanceof CheckpointExecutable) {
            jobService.deleteJob(jobId);
        }
    }

    cubeDescManager = CubeDescManager.getInstance(kylinConfig);

    // update enginType
    updateCubeEngineType(Lists.newArrayList("ci_inner_join_cube", "ci_left_join_cube"));
}

Example #3

Source File: BatchCubingJobBuilder2.java From kylin with Apache License 2.0

6 votes

protected void addInMemCubingSteps(final CubingJob result, String jobId, String cuboidRootPath) {
    // base cuboid job
    MapReduceExecutable cubeStep = new MapReduceExecutable();

    StringBuilder cmd = new StringBuilder();
    appendMapReduceParameters(cmd, JobEngineConfig.IN_MEM_JOB_CONF_SUFFIX);

    cubeStep.setName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE);

    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
    appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidRootPath);
    appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Cube_Builder_" + seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
    if (seg.getCubeDesc().isShrunkenDictFromGlobalEnabled()) {
        appendExecCmdParameters(cmd, BatchConstants.ARG_SHRUNKEN_DICT_PATH, getShrunkenDictionaryPath(jobId));
    }

    cubeStep.setMapReduceParams(cmd.toString());
    cubeStep.setMapReduceJobClass(getInMemCuboidJob());
    result.addTask(cubeStep);
}

Example #4

Source File: JoinedFlatTable.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc) {
    CubeSegment segment = ((CubeSegment) flatDesc.getSegment());
    KylinConfig kylinConfig;
    if (null == segment) {
        kylinConfig = KylinConfig.getInstanceFromEnv();
    } else {
        kylinConfig = (flatDesc.getSegment()).getConfig();
    }

    if (kylinConfig.isAdvancedFlatTableUsed()) {
        try {
            Class advancedFlatTable = Class.forName(kylinConfig.getAdvancedFlatTableClass());
            Method method = advancedFlatTable.getMethod("generateInsertDataStatement", IJoinedFlatTableDesc.class,
                    JobEngineConfig.class);
            return (String) method.invoke(null, flatDesc);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    return "INSERT OVERWRITE TABLE " + quoteIdentifier(flatDesc.getTableName(), null) + " " + generateSelectDataStatement(flatDesc)
            + ";\n";
}

Example #5

Source File: BatchOptimizeJobBuilder2.java From kylin with Apache License 2.0

6 votes

private MapReduceExecutable createInMemCubingStep(String jobId, CuboidModeEnum cuboidMode, String cuboidRootPath) {
    MapReduceExecutable cubeStep = new MapReduceExecutable();

    StringBuilder cmd = new StringBuilder();
    appendMapReduceParameters(cmd, JobEngineConfig.IN_MEM_JOB_CONF_SUFFIX);

    cubeStep.setName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE);

    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
    appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, getBaseCuboidPath(cuboidRootPath));
    appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getInMemCuboidPath(cuboidRootPath));
    appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME,
            "Kylin_Cube_Builder_" + seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBOID_MODE, cuboidMode.toString());

    cubeStep.setMapReduceParams(cmd.toString());
    cubeStep.setMapReduceJobClass(InMemCuboidFromBaseCuboidJob.class);
    cubeStep.setCounterSaveAs(
            CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES + "," + CubingJob.CUBE_SIZE_BYTES);
    return cubeStep;
}

Example #6

Source File: StreamingCubingJobBuilder.java From kylin with Apache License 2.0

6 votes

private MapReduceExecutable createMergeDictStep(String streamingStoragePath, String jobId, DefaultChainedExecutable jobFlow) {
    MapReduceExecutable mergeDict = new MapReduceExecutable();
    mergeDict.setName(ExecutableConstants.STEP_NAME_STREAMING_CREATE_DICTIONARY);
    StringBuilder cmd = new StringBuilder();

    appendMapReduceParameters(cmd, JobEngineConfig.CUBE_MERGE_JOB_CONF_SUFFIX);
    appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME,
            ExecutableConstants.STEP_NAME_STREAMING_CREATE_DICTIONARY);
    appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, streamingStoragePath);
    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_NAME, seg.getName());
    //Instead of using mr job output, trySaveNewDict api is used, so output path is useless here
    appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getDictPath(jobId));

    final String cubeName = CubingExecutableUtil.getCubeName(jobFlow.getParams());
    mergeDict.setMapReduceParams(cmd.toString());
    mergeDict.setMapReduceJobClass(MergeDictJob.class);
    mergeDict.setLockPathName(cubeName);
    mergeDict.setIsNeedLock(true);
    mergeDict.setIsNeedReleaseLock(false);
    mergeDict.setJobFlowJobId(jobFlow.getId());

    return mergeDict;

}

Example #7

Source File: StreamingCubingJobBuilder.java From kylin with Apache License 2.0

6 votes

private MapReduceExecutable createInMemCubingStep(String jobId, CuboidModeEnum cuboidMode, String cuboidRootPath,
                                                  String tmpBaseCuboidPath) {
    MapReduceExecutable cubeStep = new MapReduceExecutable();

    StringBuilder cmd = new StringBuilder();
    appendMapReduceParameters(cmd, JobEngineConfig.IN_MEM_JOB_CONF_SUFFIX);

    cubeStep.setName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE);

    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
    appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, tmpBaseCuboidPath);
    appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getInMemCuboidPath(cuboidRootPath));
    appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Cube_Builder_"
            + seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBOID_MODE, cuboidMode.toString());
    appendExecCmdParameters(cmd, BatchConstants.ARG_UPDATE_SHARD, "true");

    cubeStep.setMapReduceParams(cmd.toString());
    cubeStep.setMapReduceJobClass(InMemCuboidFromBaseCuboidJob.class);
    cubeStep.setCounterSaveAs(",,"
            + CubingJob.CUBE_SIZE_BYTES);
    return cubeStep;
}

Example #8

Source File: StreamingCubingJobBuilder.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private MapReduceExecutable createInMemCubingStep(String jobId, CuboidModeEnum cuboidMode, String cuboidRootPath,
                                                  String tmpBaseCuboidPath) {
    MapReduceExecutable cubeStep = new MapReduceExecutable();

    StringBuilder cmd = new StringBuilder();
    appendMapReduceParameters(cmd, JobEngineConfig.IN_MEM_JOB_CONF_SUFFIX);

    cubeStep.setName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE);

    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
    appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, tmpBaseCuboidPath);
    appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getInMemCuboidPath(cuboidRootPath));
    appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Cube_Builder_"
            + seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBOID_MODE, cuboidMode.toString());
    appendExecCmdParameters(cmd, BatchConstants.ARG_UPDATE_SHARD, "true");

    cubeStep.setMapReduceParams(cmd.toString());
    cubeStep.setMapReduceJobClass(InMemCuboidFromBaseCuboidJob.class);
    cubeStep.setCounterSaveAs(",,"
            + CubingJob.CUBE_SIZE_BYTES);
    return cubeStep;
}

Example #9

Source File: BatchOptimizeJobBuilder2.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private MapReduceExecutable createInMemCubingStep(String jobId, CuboidModeEnum cuboidMode, String cuboidRootPath) {
    MapReduceExecutable cubeStep = new MapReduceExecutable();

    StringBuilder cmd = new StringBuilder();
    appendMapReduceParameters(cmd, JobEngineConfig.IN_MEM_JOB_CONF_SUFFIX);

    cubeStep.setName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE);

    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
    appendExecCmdParameters(cmd, BatchConstants.ARG_INPUT, getBaseCuboidPath(cuboidRootPath));
    appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, getInMemCuboidPath(cuboidRootPath));
    appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME,
            "Kylin_Cube_Builder_" + seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBOID_MODE, cuboidMode.toString());

    cubeStep.setMapReduceParams(cmd.toString());
    cubeStep.setMapReduceJobClass(InMemCuboidFromBaseCuboidJob.class);
    cubeStep.setCounterSaveAs(
            CubingJob.SOURCE_RECORD_COUNT + "," + CubingJob.SOURCE_SIZE_BYTES + "," + CubingJob.CUBE_SIZE_BYTES);
    return cubeStep;
}

Example #10

Source File: BuildCubeWithEngineTest.java From Kylin with Apache License 2.0

6 votes

@Before
public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();


    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(kylinConfig));
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    jobEngineConfig = new JobEngineConfig(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        if(jobService.getJob(jobId) instanceof CubingJob){
            jobService.deleteJob(jobId);
        }
    }

}

Example #11

Source File: JoinedFlatTable.java From kylin with Apache License 2.0

6 votes

public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc) {
    CubeSegment segment = ((CubeSegment) flatDesc.getSegment());
    KylinConfig kylinConfig;
    if (null == segment) {
        kylinConfig = KylinConfig.getInstanceFromEnv();
    } else {
        kylinConfig = (flatDesc.getSegment()).getConfig();
    }

    if (kylinConfig.isAdvancedFlatTableUsed()) {
        try {
            Class advancedFlatTable = Class.forName(kylinConfig.getAdvancedFlatTableClass());
            Method method = advancedFlatTable.getMethod("generateInsertDataStatement", IJoinedFlatTableDesc.class,
                    JobEngineConfig.class);
            return (String) method.invoke(null, flatDesc);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    return "INSERT OVERWRITE TABLE " + quoteIdentifier(flatDesc.getTableName(), null) + " " + generateSelectDataStatement(flatDesc)
            + ";\n";
}

Example #12

Source File: BatchCubingJobBuilder2.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

protected void addInMemCubingSteps(final CubingJob result, String jobId, String cuboidRootPath) {
    // base cuboid job
    MapReduceExecutable cubeStep = new MapReduceExecutable();

    StringBuilder cmd = new StringBuilder();
    appendMapReduceParameters(cmd, JobEngineConfig.IN_MEM_JOB_CONF_SUFFIX);

    cubeStep.setName(ExecutableConstants.STEP_NAME_BUILD_IN_MEM_CUBE);

    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
    appendExecCmdParameters(cmd, BatchConstants.ARG_OUTPUT, cuboidRootPath);
    appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Cube_Builder_" + seg.getRealization().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBING_JOB_ID, jobId);
    if (seg.getCubeDesc().isShrunkenDictFromGlobalEnabled()) {
        appendExecCmdParameters(cmd, BatchConstants.ARG_SHRUNKEN_DICT_PATH, getShrunkenDictionaryPath(jobId));
    }

    cubeStep.setMapReduceParams(cmd.toString());
    cubeStep.setMapReduceJobClass(getInMemCuboidJob());
    result.addTask(cubeStep);
}

Example #13

Source File: BatchMergeJobBuilder2.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public MapReduceExecutable createMergeDictionaryStep(CubeSegment seg, String jobID, List<String> mergingSegmentIds) {
    MapReduceExecutable mergeDictionaryStep = new MapReduceExecutable();
    mergeDictionaryStep.setName(ExecutableConstants.STEP_NAME_MERGE_DICTIONARY);
    StringBuilder cmd = new StringBuilder();
    appendMapReduceParameters(cmd, JobEngineConfig.CUBE_MERGE_JOB_CONF_SUFFIX);

    appendExecCmdParameters(cmd, BatchConstants.ARG_CUBE_NAME, seg.getCubeInstance().getName());
    appendExecCmdParameters(cmd, BatchConstants.ARG_SEGMENT_ID, seg.getUuid());
    appendExecCmdParameters(cmd, BatchConstants.ARG_META_URL, getSegmentMetadataUrl(seg.getConfig(), jobID));
    appendExecCmdParameters(cmd, MergeDictionaryJob.OPTION_MERGE_SEGMENT_IDS.getOpt(), StringUtil.join(mergingSegmentIds, ","));
    appendExecCmdParameters(cmd, MergeDictionaryJob.OPTION_OUTPUT_PATH_DICT.getOpt(), getDictInfoPath(jobID));
    appendExecCmdParameters(cmd, MergeDictionaryJob.OPTION_OUTPUT_PATH_STAT.getOpt(), getStatisticsPath(jobID));
    appendExecCmdParameters(cmd, BatchConstants.ARG_JOB_NAME, "Kylin_Merge_Dictionary_" + seg.getCubeInstance().getName() + "_Step");

    mergeDictionaryStep.setMapReduceParams(cmd.toString());
    mergeDictionaryStep.setMapReduceJobClass(MergeDictionaryJob.class);

    return mergeDictionaryStep;
}

Example #14

Source File: BuildCubeWithEngine.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public void before() throws Exception {
    deployEnv();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        AbstractExecutable executable = jobService.getJob(jobId);
        if (executable instanceof CubingJob || executable instanceof CheckpointExecutable) {
            jobService.deleteJob(jobId);
        }
    }

    cubeDescManager = CubeDescManager.getInstance(kylinConfig);

    // update enginType
    updateCubeEngineType(Lists.newArrayList("ci_inner_join_cube", "ci_left_join_cube"));
}

Example #15

Source File: CuratorLeaderSelector.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

CuratorLeaderSelector(CuratorFramework client, String path, String name, JobEngineConfig jobEngineConfig) {
    this.name = name;
    this.leaderSelector = new LeaderSelector(client, path, this);
    this.leaderSelector.setId(name);
    this.leaderSelector.autoRequeue();
    this.jobEngineConfig = jobEngineConfig;
    this.defaultScheduler = DefaultScheduler.getInstance();
}

Example #16

Source File: StorageCleanupJob.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private void deleteHiveTables(List<String> allHiveTablesNeedToBeDeleted, Map<String, String> segmentId2JobId)
        throws IOException {
    final JobEngineConfig engineConfig = new JobEngineConfig(config);
    final int uuidLength = 36;

    final String useDatabaseHql = "USE " + config.getHiveDatabaseForIntermediateTable() + ";";
    final HiveCmdBuilder hiveCmdBuilder = new HiveCmdBuilder();
    hiveCmdBuilder.addStatement(useDatabaseHql);
    for (String delHive : allHiveTablesNeedToBeDeleted) {
        hiveCmdBuilder.addStatement("drop table if exists " + delHive + "; ");
        logger.info("Deleting Hive table " + delHive);
    }
    getCliCommandExecutor().execute(hiveCmdBuilder.build());

    // If kylin.source.hive.keep-flat-table, some intermediate table might be kept.
    // Do delete external path.
    for (String tableToDelete : allHiveTablesNeedToBeDeleted) {
        String uuid = tableToDelete.substring(tableToDelete.length() - uuidLength, tableToDelete.length());
        String segmentId = uuid.replace("_", "-");

        if (segmentId2JobId.containsKey(segmentId)) {
            String path = JobBuilderSupport.getJobWorkingDir(engineConfig.getHdfsWorkingDirectory(),
                    segmentId2JobId.get(segmentId)) + "/" + tableToDelete;
            Path externalDataPath = new Path(path);
            if (defaultFs.exists(externalDataPath)) {
                defaultFs.delete(externalDataPath, true);
                logger.info("Hive table {}'s external path {} deleted", tableToDelete, path);
            } else {
                logger.info(
                        "Hive table {}'s external path {} not exist. It's normal if kylin.source.hive.keep-flat-table set false (By default)",
                        tableToDelete, path);
            }
        } else {
            logger.warn("Hive table {}'s job ID not found, segmentId2JobId: {}", tableToDelete,
                    segmentId2JobId.toString());
        }
    }
}

Example #17

Source File: JobService.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@SuppressWarnings("unchecked")
@Override
public void afterPropertiesSet() throws Exception {

    String timeZone = getConfig().getTimeZone();
    TimeZone tzone = TimeZone.getTimeZone(timeZone);
    TimeZone.setDefault(tzone);

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();

    // In case of that kylin.server.cluster-name is not set,
    // this method have to be called first to avoid the influence of the change of kylin.metadata.url
    String clusterName = kylinConfig.getClusterName();
    logger.info("starting to initialize an instance in cluster {}", clusterName);

    final Scheduler<AbstractExecutable> scheduler = (Scheduler<AbstractExecutable>) SchedulerFactory
            .scheduler(kylinConfig.getSchedulerType());

    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());

    Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
        @Override
        public void run() {
            try {
                scheduler.shutdown();
            } catch (SchedulerException e) {
                logger.error("error occurred to shutdown scheduler", e);
            }
        }
    }));
}

Example #18

Source File: JobService.java From kylin with Apache License 2.0

5 votes

@SuppressWarnings("unchecked")
@Override
public void afterPropertiesSet() throws Exception {

    String timeZone = getConfig().getTimeZone();
    TimeZone tzone = TimeZone.getTimeZone(timeZone);
    TimeZone.setDefault(tzone);

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();

    // In case of that kylin.server.cluster-name is not set,
    // this method have to be called first to avoid the influence of the change of kylin.metadata.url
    String clusterName = kylinConfig.getClusterName();
    logger.info("starting to initialize an instance in cluster {}", clusterName);

    final Scheduler<AbstractExecutable> scheduler = (Scheduler<AbstractExecutable>) SchedulerFactory
            .scheduler(kylinConfig.getSchedulerType());

    if (kylinConfig.getServerSelfDiscoveryEnabled()) {
        KylinServerDiscovery.getInstance();
    }
    logger.info("Cluster servers: {}", Lists.newArrayList(kylinConfig.getRestServers()));
    
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());

    Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
        @Override
        public void run() {
            try {
                scheduler.shutdown();
            } catch (SchedulerException e) {
                logger.error("error occurred to shutdown scheduler", e);
            }
        }
    }));
}

Example #19

Source File: BaseSchedulerTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

protected void startScheduler() throws SchedulerException {
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
}

Example #20

Source File: BuildCubeWithStream.java From kylin with Apache License 2.0

5 votes

public void before() throws Exception {
    deployEnv();
    simpleBuildMode = isSimpleBuildMode();
    if (simpleBuildMode) {
        logger.info("Will use simple build mode");
    } else {
        logger.info("Will not use simple build mode");
    }

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(kylinConfig), new ZookeeperJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);

    final CubeInstance cubeInstance = CubeManager.getInstance(kylinConfig).getCube(cubeName);
    final String factTable = cubeInstance.getRootFactTable();

    final StreamingManager streamingManager = StreamingManager.getInstance(kylinConfig);
    final StreamingConfig streamingConfig = streamingManager.getStreamingConfig(factTable);
    kafkaConfig = KafkaConfigManager.getInstance(kylinConfig).getKafkaConfig(streamingConfig.getName());

    String topicName = RandomUtil.randomUUID().toString();
    BrokerConfig brokerConfig = kafkaConfig.getKafkaClusterConfigs().get(0).getBrokerConfigs().get(0);
    kafkaConfig.setTopic(topicName);
    KafkaConfigManager.getInstance(kylinConfig).updateKafkaConfig(kafkaConfig);

    startEmbeddedKafka(topicName, brokerConfig);
}

Example #21

Source File: ExampleServer.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public ExampleServer(String address) throws Exception {
    this.address = address;

    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    KylinConfig kylinConfig1 = KylinConfig.createKylinConfig(kylinConfig);
    kylinConfig1.setProperty("kylin.server.host-address", address);

    CuratorFramework client = ZKUtil.newZookeeperClient(kylinConfig1);
    scheduler = new CuratorScheduler(client);
    scheduler.init(new JobEngineConfig(kylinConfig1), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
}

Example #22

Source File: LocalWithSparkSessionTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Before
public void setup() throws SchedulerException {
    overwriteSystemProp("kylin.job.scheduler.poll-interval-second", "1");
    overwriteSystemProp("calcite.keep-in-clause", "true");
    overwriteSystemProp("kylin.metadata.distributed-lock-impl", "org.apache.kylin.engine.spark.utils.MockedDistributedLock$MockedFactory");
    this.createTestMetadata();
    DefaultScheduler scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
}

Example #23

Source File: JoinedFlatTableTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testGenerateInsertSql() throws IOException {
    String sqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, fakeJobUUID, new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
    System.out.println(sqls);

    int length = sqls.length();
    assertEquals(1155, length);
}

Example #24

Source File: JobBuilderSupport.java From kylin with Apache License 2.0

5 votes

public JobBuilderSupport(CubeSegment seg, String submitter, Integer priorityOffset) {
    Preconditions.checkNotNull(seg, "segment cannot be null");
    this.config = new JobEngineConfig(seg.getConfig());
    this.seg = seg;
    this.submitter = submitter;
    this.priorityOffset = priorityOffset;
}

Example #25

Source File: BaseSchedulerTest.java From Kylin with Apache License 2.0

5 votes

@Before
public void setup() throws Exception {
    createTestMetadata();
    setFinalStatic(ExecutableConstants.class.getField("DEFAULT_SCHEDULER_INTERVAL_SECONDS"), 10);
    jobService = ExecutableManager.getInstance(KylinConfig.getInstanceFromEnv());
    scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }

}

Example #26

Source File: CubingJob.java From kylin with Apache License 2.0

5 votes

private static CubingJob initCubingJob(CubeSegment seg, String jobType, String submitter, JobEngineConfig config) {
    KylinConfig kylinConfig = config.getConfig();
    CubeInstance cube = seg.getCubeInstance();
    List<ProjectInstance> projList = ProjectManager.getInstance(kylinConfig).findProjects(cube.getType(),
            cube.getName());
    if (projList == null || projList.size() == 0) {
        throw new RuntimeException("Cannot find the project containing the cube " + cube.getName() + "!!!");
    } else if (projList.size() >= 2) {
        String msg = "Find more than one project containing the cube " + cube.getName()
                + ". It does't meet the uniqueness requirement!!! ";
        if (!config.getConfig().allowCubeAppearInMultipleProjects()) {
            throw new RuntimeException(msg);
        } else {
            logger.warn(msg);
        }
    }

    CubingJob result = new CubingJob();
    SimpleDateFormat format = new SimpleDateFormat("z yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    format.setTimeZone(TimeZone.getTimeZone(config.getTimeZone()));
    result.setDeployEnvName(kylinConfig.getDeployEnv());
    result.setProjectName(projList.get(0).getName());
    result.setJobType(jobType);
    CubingExecutableUtil.setCubeName(seg.getCubeInstance().getName(), result.getParams());
    CubingExecutableUtil.setSegmentId(seg.getUuid(), result.getParams());
    CubingExecutableUtil.setSegmentName(seg.getName(), result.getParams());
    result.setName(jobType + " CUBE - " + seg.getCubeInstance().getDisplayName() + " - " + seg.getName() + " - "
            + format.format(new Date(System.currentTimeMillis())));
    result.setSubmitter(submitter);
    result.setNotifyList(seg.getCubeInstance().getDescriptor().getNotifyList());
    return result;
}

Example #27

Source File: CuratorScheduler.java From kylin with Apache License 2.0

5 votes

@Override
public void init(JobEngineConfig jobEngineConfig, JobLock jobLock) throws SchedulerException {
    kylinConfig = jobEngineConfig.getConfig();

    synchronized (this) {
        if (started) {
            logger.info("CuratorScheduler already started, skipped.");
            return;
        }

        // curatorClient can be assigned before only for test cases
        // due to creating independent curator client rather than share a cached one to avoid influences
        if (curatorClient == null) {
            curatorClient = ZKUtil.getZookeeperClient(kylinConfig);
        }

        String restAddress = kylinConfig.getServerRestAddress();

        String jobEnginePath = JOB_ENGINE_LEADER_PATH;

        if (ServerMode.isJob(jobEngineConfig.getConfig())) {
            jobClient = new CuratorLeaderSelector(curatorClient, jobEnginePath, restAddress, jobEngineConfig);
            try {
                logger.info("start Job Engine, lock path is: " + jobEnginePath);
                jobClient.start();
                monitorJobEngine();
            } catch (IOException e) {
                throw new SchedulerException(e);
            }
        } else {
            logger.info("server mode: " + jobEngineConfig.getConfig().getServerMode()
                    + ", no need to run job scheduler");
        }
        started = true;
    }
}

Example #28

Source File: JobEngineConfigTest.java From kylin with Apache License 2.0

5 votes

@Test
public void testPropertiesHotLoad() throws IOException {
    KylinConfig baseConfig = KylinConfig.getInstanceFromEnv();
    JobEngineConfig jobEngineConfig = new JobEngineConfig(baseConfig);
    assertEquals(10, jobEngineConfig.getMaxConcurrentJobLimit());

    updateProperty("kylin.job.max-concurrent-jobs", "20");
    KylinConfig.getInstanceFromEnv().reloadFromSiteProperties();

    assertEquals(20, jobEngineConfig.getMaxConcurrentJobLimit());
}

Example #29

Source File: BaseSchedulerTest.java From kylin with Apache License 2.0

5 votes

protected void startScheduler() throws SchedulerException {
    scheduler = DefaultScheduler.createInstance();
    scheduler.init(new JobEngineConfig(KylinConfig.getInstanceFromEnv()), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
}

Example #30

Source File: KafkaInputBase.java From kylin with Apache License 2.0

5 votes

public BaseBatchCubingInputSide(CubeSegment seg, IJoinedFlatTableDesc flatDesc) {
    this.conf = new JobEngineConfig(KylinConfig.getInstanceFromEnv());
    this.config = seg.getConfig();
    this.flatDesc = flatDesc;
    this.hiveTableDatabase = config.getHiveDatabaseForIntermediateTable();
    this.seg = seg;
    this.cubeDesc = seg.getCubeDesc();
    this.cubeName = seg.getCubeInstance().getName();
}