Java Code Examples for org.apache.kylin.job.JoinedFlatTable#generateInsertDataStatement()

The following examples show how to use org.apache.kylin.job.JoinedFlatTable#generateInsertDataStatement() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: HiveInputBase.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

protected static AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
        String cubeName, IJoinedFlatTableDesc flatDesc) {
    //from hive to hive
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);

    CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();
    step.setInitStatement(hiveInitStatements);
    step.setCreateTableStatement(dropTableHql + createTableHql + insertDataHqls);
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
    return step;
}

Example 2

Source File: HiveInputBase.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

protected static AbstractExecutable createFlatHiveTableByLivyStep(String hiveInitStatements, String jobWorkingDir,
        String cubeName, IJoinedFlatTableDesc flatDesc) {
    //from hive to hive
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);

    CreateFlatHiveTableByLivyStep step = new CreateFlatHiveTableByLivyStep();
    step.setInitStatement(hiveInitStatements);
    step.setCreateTableStatement(dropTableHql + createTableHql + insertDataHqls);
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
    return step;
}

Example 3

Source File: KafkaInputBase.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

protected static AbstractExecutable createFlatTable(final String hiveTableDatabase,
                                                    final String baseLocation, final String cubeName,
                                                    final StreamCubeFactTableDesc streamFactDesc, final List<String> intermediateTables,
                                                    final List<String> intermediatePaths) {
    final IJoinedFlatTableDesc flatDesc = streamFactDesc.getFlatTableDesc();

    final String hiveInitStatements = JoinedFlatTable.generateHiveInitStatements(hiveTableDatabase);

    final String dropFactTableHql = JoinedFlatTable.generateDropTableStatement(streamFactDesc);
    // the table inputformat is sequence file
    final String createFactTableHql = JoinedFlatTable.generateCreateTableStatement(streamFactDesc, baseLocation,
            JoinedFlatTable.SEQUENCEFILE);

    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, baseLocation);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);
    insertDataHqls = insertDataHqls.replace(
            quoteTableIdentity(flatDesc.getDataModel().getRootFactTable(), null) + " ",
            quoteTableIdentity(hiveTableDatabase, streamFactDesc.getTableName(), null) + " ");

    CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setInitStatement(hiveInitStatements);
    step.setCreateTableStatement(
            dropFactTableHql + createFactTableHql + dropTableHql + createTableHql + insertDataHqls);
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);

    intermediateTables.add(flatDesc.getTableName());
    intermediateTables.add(streamFactDesc.getTableName());
    intermediatePaths.add(baseLocation + "/" + flatDesc.getTableName());
    intermediatePaths.add(baseLocation + "/" + streamFactDesc.getTableName());
    return step;
}

Example 4

Source File: HiveInputBase.java From kylin with Apache License 2.0

5 votes

protected static AbstractExecutable createFlatHiveTableStep(String hiveInitStatements, String jobWorkingDir,
                                                            String cubeName, IJoinedFlatTableDesc flatDesc) {
    //from hive to hive
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);

    CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();
    step.setInitStatement(hiveInitStatements);
    step.setCreateTableStatement(dropTableHql + createTableHql + insertDataHqls);
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
    return step;
}

Example 5

Source File: HiveInputBase.java From kylin with Apache License 2.0

5 votes

protected static AbstractExecutable createFlatHiveTableByLivyStep(String hiveInitStatements, String jobWorkingDir,
                                                                  String cubeName, IJoinedFlatTableDesc flatDesc) {
    //from hive to hive
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, jobWorkingDir);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);

    CreateFlatHiveTableByLivyStep step = new CreateFlatHiveTableByLivyStep();
    step.setInitStatement(hiveInitStatements);
    step.setCreateTableStatement(dropTableHql + createTableHql + insertDataHqls);
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);
    return step;
}

Example 6

Source File: KafkaInputBase.java From kylin with Apache License 2.0

5 votes

protected static AbstractExecutable createFlatTable(final String hiveTableDatabase,
                                                    final String baseLocation, final String cubeName,
                                                    final StreamCubeFactTableDesc streamFactDesc, final List<String> intermediateTables,
                                                    final List<String> intermediatePaths) {
    final IJoinedFlatTableDesc flatDesc = streamFactDesc.getFlatTableDesc();

    final String hiveInitStatements = JoinedFlatTable.generateHiveInitStatements(hiveTableDatabase);

    final String dropFactTableHql = JoinedFlatTable.generateDropTableStatement(streamFactDesc);
    // the table inputformat is sequence file
    final String createFactTableHql = JoinedFlatTable.generateCreateTableStatement(streamFactDesc, baseLocation,
            JoinedFlatTable.SEQUENCEFILE);

    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc, baseLocation);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);
    insertDataHqls = insertDataHqls.replace(
            quoteTableIdentity(flatDesc.getDataModel().getRootFactTable(), null) + " ",
            quoteTableIdentity(hiveTableDatabase, streamFactDesc.getTableName(), null) + " ");

    CreateFlatHiveTableStep step = new CreateFlatHiveTableStep();
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setInitStatement(hiveInitStatements);
    step.setCreateTableStatement(
            dropFactTableHql + createFactTableHql + dropTableHql + createTableHql + insertDataHqls);
    step.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_HIVE_TABLE);

    intermediateTables.add(flatDesc.getTableName());
    intermediateTables.add(streamFactDesc.getTableName());
    intermediatePaths.add(baseLocation + "/" + flatDesc.getTableName());
    intermediatePaths.add(baseLocation + "/" + streamFactDesc.getTableName());
    return step;
}

Example 7

Source File: JoinedFlatTableTest.java From Kylin with Apache License 2.0

5 votes

@Test
public void testGenerateInsertSql() throws IOException {
    String sqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, fakeJobUUID, new JobEngineConfig(KylinConfig.getInstanceFromEnv()));
    System.out.println(sqls);

    int length = sqls.length();
    assertEquals(1155, length);
}

Example 8

Source File: HiveInputBase.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

protected static AbstractExecutable createFlatHiveTableBySparkSql(String hiveInitStatements,
        String jobWorkingDir, String cubeName, IJoinedFlatTableDesc flatDesc) {
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc,
            jobWorkingDir);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);

    KylinConfig config = flatDesc.getSegment().getConfig();
    final SparkExecutable sparkExecutable = SparkExecutableFactory.instance(config);
    sparkExecutable.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_TABLE_WITH_SPARK);
    sparkExecutable.setClassName(SparkCreatingFlatTable.class.getName());

    sparkExecutable.setParam(SparkSqlBatch.OPTION_CUBE_NAME.getOpt(), cubeName);
    sparkExecutable.setParam(SparkSqlBatch.OPTION_STEP_NAME.getOpt(),
            base64EncodeStr(ExecutableConstants.STEP_NAME_CREATE_FLAT_TABLE_WITH_SPARK));
    sparkExecutable.setParam(SparkSqlBatch.OPTION_SEGMENT_ID.getOpt(),
            flatDesc.getSegment().getName());
    sparkExecutable.setParam(SparkSqlBatch.OPTION_SQL_COUNT.getOpt(),
            String.valueOf(SparkCreatingFlatTable.SQL_COUNT));

    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(0).getOpt(),
            base64EncodeStr(hiveInitStatements));
    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(1).getOpt(),
            base64EncodeStr(dropTableHql));

    // createTableHql include create table sql and alter table sql
    String[] sqlArr = createTableHql.trim().split(";");
    if (2 != sqlArr.length) {
        throw new RuntimeException("create table hql should combined by a create table sql " +
                "and a alter sql, but got: " + createTableHql);
    }
    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(2).getOpt(),
            base64EncodeStr(sqlArr[0]));
    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(3).getOpt(),
            base64EncodeStr(sqlArr[1]));

    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(4).getOpt(),
            base64EncodeStr(insertDataHqls));

    StringBuilder jars = new StringBuilder();
    StringUtil.appendWithSeparator(jars, config.getSparkAdditionalJars());
    sparkExecutable.setJars(jars.toString());

    return sparkExecutable;
}

Example 9

Source File: HiveInputBase.java From kylin with Apache License 2.0

4 votes

protected static AbstractExecutable createFlatHiveTableBySparkSql(String hiveInitStatements,
                                                                  String jobWorkingDir, String cubeName, IJoinedFlatTableDesc flatDesc) {
    final String dropTableHql = JoinedFlatTable.generateDropTableStatement(flatDesc);
    final String createTableHql = JoinedFlatTable.generateCreateTableStatement(flatDesc,
            jobWorkingDir);
    String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(flatDesc);

    KylinConfig config = flatDesc.getSegment().getConfig();
    final SparkExecutable sparkExecutable = SparkExecutableFactory.instance(config);
    sparkExecutable.setName(ExecutableConstants.STEP_NAME_CREATE_FLAT_TABLE_WITH_SPARK);
    sparkExecutable.setClassName(SparkCreatingFlatTable.class.getName());

    sparkExecutable.setParam(SparkSqlBatch.OPTION_CUBE_NAME.getOpt(), cubeName);
    sparkExecutable.setParam(SparkSqlBatch.OPTION_STEP_NAME.getOpt(),
            base64EncodeStr(ExecutableConstants.STEP_NAME_CREATE_FLAT_TABLE_WITH_SPARK));
    sparkExecutable.setParam(SparkSqlBatch.OPTION_SEGMENT_ID.getOpt(),
            flatDesc.getSegment().getName());
    sparkExecutable.setParam(SparkSqlBatch.OPTION_SQL_COUNT.getOpt(),
            String.valueOf(SparkCreatingFlatTable.SQL_COUNT));

    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(0).getOpt(),
            base64EncodeStr(hiveInitStatements));
    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(1).getOpt(),
            base64EncodeStr(dropTableHql));

    // createTableHql include create table sql and alter table sql
    String[] sqlArr = createTableHql.trim().split(";");
    if (2 != sqlArr.length) {
        throw new RuntimeException("create table hql should combined by a create table sql " +
                "and a alter sql, but got: " + createTableHql);
    }
    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(2).getOpt(),
            base64EncodeStr(sqlArr[0]));
    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(3).getOpt(),
            base64EncodeStr(sqlArr[1]));

    sparkExecutable.setParam(SparkCreatingFlatTable.getSqlOption(4).getOpt(),
            base64EncodeStr(insertDataHqls));

    StringBuilder jars = new StringBuilder();
    StringUtil.appendWithSeparator(jars, config.getSparkAdditionalJars());
    sparkExecutable.setJars(jars.toString());

    return sparkExecutable;
}

Example 10

Source File: IIFlattenHiveJob.java From Kylin with Apache License 2.0

4 votes

@Override
public int run(String[] args) throws Exception {
    Options options = new Options();
    try {
        options.addOption(OPTION_II_NAME);
        parseOptions(options, args);

        String iiname = getOptionValue(OPTION_II_NAME);
        KylinConfig config = KylinConfig.getInstanceFromEnv();

        IIInstance iiInstance = IIManager.getInstance(config).getII(iiname);
        IIDesc iidesc = IIDescManager.getInstance(config).getIIDesc(iiInstance.getDescName());

        String jobUUID = "00bf87b5-c7b5-4420-a12a-07f6b37b3187";
        JobEngineConfig engineConfig = new JobEngineConfig(config);
        IJoinedFlatTableDesc intermediateTableDesc = new IIJoinedFlatTableDesc(iidesc);
        String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc, jobUUID);
        String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, //
                JobInstance.getJobWorkingDir(jobUUID, engineConfig.getHdfsWorkingDirectory()), jobUUID);
        String insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, jobUUID, engineConfig);

        StringBuffer buf = new StringBuffer();
        buf.append("hive -e \"");
        buf.append(dropTableHql + "\n");
        buf.append(createTableHql + "\n");
        buf.append(insertDataHqls + "\n");
        buf.append("\"");
        
        System.out.println(buf.toString());
        System.out.println("========================");

        ShellCmd cmd = new ShellCmd(buf.toString(), null, null, null, false);
        ICommandOutput output = cmd.execute();
        System.out.println(output.getOutput());
        System.out.println(output.getExitCode());
        
        return 0;
    } catch (Exception e) {
        printUsage(options);
        throw e;
    }
}