Java Code Examples for org.apache.kylin.metadata.model.TblColRef#getTableAlias()

The following examples show how to use org.apache.kylin.metadata.model.TblColRef#getTableAlias() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FlatTableSqlQuoteUtils.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private static Map<String, Map<String, String>> buildTableToColumnsMap(IJoinedFlatTableDesc flatDesc) {
    Map<String, Map<String, String>> map = Maps.newHashMap();
    List<TblColRef> colRefs = flatDesc.getAllColumns();
    for (TblColRef colRef : colRefs) {
        String colName = colRef.getName();
        String tableName = colRef.getTableRef().getTableName();
        String colAlias = colRef.getTableAlias() + "_" + colRef.getName();
        if (map.containsKey(tableName)) {
            map.get(tableName).put(colName, colAlias);
        } else {
            Map<String, String> colToAliasMap = Maps.newHashMap();
            colToAliasMap.put(colName, colAlias);
            map.put(tableName, colToAliasMap);
        }
    }
    return map;
}
Example 2
Source File: FlatTableSqlQuoteUtils.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private static Map<String, Map<String, String>> buildTableToColumnsMap(IJoinedFlatTableDesc flatDesc) {
    Map<String, Map<String, String>> map = Maps.newHashMap();
    List<TblColRef> colRefs = flatDesc.getAllColumns();
    for (TblColRef colRef : colRefs) {
        String colName = colRef.getName();
        String tableName = colRef.getTableRef().getTableName();
        String colAlias = colRef.getTableAlias() + "_" + colRef.getName();
        if (map.containsKey(tableName)) {
            map.get(tableName).put(colName, colAlias);
        } else {
            Map<String, String> colToAliasMap = Maps.newHashMap();
            colToAliasMap.put(colName, colAlias);
            map.put(tableName, colToAliasMap);
        }
    }
    return map;
}
Example 3
Source File: FlatTableSqlQuoteUtils.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
private static Map<String, String> buildTableToTableAliasMap(IJoinedFlatTableDesc flatDesc) {
    Map<String, String> map = Maps.newHashMap();
    List<TblColRef> colRefs = flatDesc.getAllColumns();
    for (TblColRef colRef : colRefs) {
        String tableName = colRef.getTableRef().getTableName();
        String alias = colRef.getTableAlias();
        map.put(tableName, alias);
    }
    return map;
}
Example 4
Source File: FlatTableSqlQuoteUtils.java    From kylin with Apache License 2.0 5 votes vote down vote up 
private static Map<String, String> buildTableToTableAliasMap(IJoinedFlatTableDesc flatDesc) {
    Map<String, String> map = Maps.newHashMap();
    List<TblColRef> colRefs = flatDesc.getAllColumns();
    for (TblColRef colRef : colRefs) {
        String tableName = colRef.getTableRef().getTableName();
        String alias = colRef.getTableAlias();
        map.put(tableName, alias);
    }
    return map;
}
Example 5
Source File: JdbcHiveInputBase.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
protected AbstractExecutable createSqoopToFlatHiveStep(String jobWorkingDir, String cubeName) {
    KylinConfig config = getConfig();
    PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc();
    String partCol = null;

    if (partitionDesc.isPartitioned()) {
        partCol = partitionDesc.getPartitionDateColumn();//tablename.colname
    }

    String splitTableAlias;
    String splitColumn;
    String splitDatabase;
    TblColRef splitColRef = determineSplitColumn();
    splitTableAlias = splitColRef.getTableAlias();

    splitColumn = getColumnIdentityQuoted(splitColRef, jdbcMetadataDialect, metaMap, true);
    splitDatabase = splitColRef.getColumnDesc().getTable().getDatabase();

    String selectSql = generateSelectDataStatementRDBMS(flatDesc, true, new String[] { partCol },
            jdbcMetadataDialect, metaMap);
    selectSql = escapeQuotationInSql(selectSql);

    String hiveTable = flatDesc.getTableName();
    String connectionUrl = config.getJdbcSourceConnectionUrl();
    String driverClass = config.getJdbcSourceDriver();
    String jdbcUser = config.getJdbcSourceUser();
    String jdbcPass = config.getJdbcSourcePass();
    String sqoopHome = config.getSqoopHome();
    String sqoopNullString = config.getSqoopNullString();
    String sqoopNullNonString = config.getSqoopNullNonString();
    String filedDelimiter = config.getJdbcSourceFieldDelimiter();
    int mapperNum = config.getSqoopMapperNum();

    String bquery = String.format(Locale.ROOT, "SELECT min(%s), max(%s) FROM %s.%s ", splitColumn,
            splitColumn, getSchemaQuoted(metaMap, splitDatabase, jdbcMetadataDialect, true),
            getTableIdentityQuoted(splitColRef.getTableRef(), metaMap, jdbcMetadataDialect, true));
    if (partitionDesc.isPartitioned()) {
        SegmentRange segRange = flatDesc.getSegRange();
        if (segRange != null && !segRange.isInfinite()) {
            if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias)
                    && (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc
                            .getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {

                String quotedPartCond = partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(
                        partitionDesc, flatDesc.getSegment(), segRange,
                        col -> getTableColumnIdentityQuoted(col, jdbcMetadataDialect, metaMap, true));
                bquery += " WHERE " + quotedPartCond;
            }
        }
    }
    bquery = escapeQuotationInSql(bquery);

    // escape ` in cmd
    splitColumn = escapeQuotationInSql(splitColumn);

    String cmd = String.format(Locale.ROOT, "%s/bin/sqoop import" + generateSqoopConfigArgString()
            + "--connect \"%s\" --driver %s --username %s --password \"%s\" --query \"%s AND \\$CONDITIONS\" "
            + "--target-dir %s/%s --split-by %s --boundary-query \"%s\" --null-string '%s' "
            + "--null-non-string '%s' --fields-terminated-by '%s' --num-mappers %d", sqoopHome, connectionUrl,
            driverClass, jdbcUser, jdbcPass, selectSql, jobWorkingDir, hiveTable, splitColumn, bquery,
            sqoopNullString, sqoopNullNonString, filedDelimiter, mapperNum);
    logger.debug("sqoop cmd : {}", cmd);
    CmdStep step = new CmdStep();
    step.setCmd(cmd);
    step.setName(ExecutableConstants.STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE);
    return step;
}
Example 6
Source File: JdbcHiveInputBase.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Override
protected AbstractExecutable createSqoopToFlatHiveStep(String jobWorkingDir, String cubeName) {
    KylinConfig config = flatDesc.getDataModel().getConfig();
    PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc();
    String partCol = null;
    boolean enableQuote = dataSource.getSqlConverter().getConfigurer().enableQuote();
    SqlDialect sqlDialect = enableQuote ? dataSource.getSqlConverter().getConfigurer().getSqlDialect() : FlatTableSqlQuoteUtils.NON_QUOTE_DIALECT;
    SqlConverter.IConfigurer iconfigurer = dataSource.getSqlConverter().getConfigurer();

    if (partitionDesc.isPartitioned()) {
        partCol = partitionDesc.getPartitionDateColumn(); //tablename.colname
    }

    String splitTable;
    String splitTableAlias;
    String splitColumn;
    String splitDatabase;
    TblColRef splitColRef = determineSplitColumn();
    splitTable = splitColRef.getTableRef().getTableDesc().getName();
    splitTableAlias = splitColRef.getTableAlias();
    splitDatabase = splitColRef.getColumnDesc().getTable().getDatabase();

    if (enableQuote) {
        splitColumn = sqlDialect.quoteIdentifier(splitColRef.getTableAlias()) + "."
                + sqlDialect.quoteIdentifier(splitColRef.getName());
        splitDatabase = sqlDialect.quoteIdentifier(splitDatabase);
        splitTable = sqlDialect.quoteIdentifier(splitTable);
        splitTableAlias = sqlDialect.quoteIdentifier(splitTableAlias);
    } else {
        splitColumn = splitColRef.getTableAlias() + "." + splitColRef.getName();
    }

    String selectSql = JoinedFlatTable.generateSelectDataStatement(flatDesc, true, new String[]{partCol}, sqlDialect);
    selectSql = escapeQuotationInSql(dataSource.convertSql(selectSql));

    String hiveTable = flatDesc.getTableName();
    String filedDelimiter = config.getJdbcSourceFieldDelimiter();
    int mapperNum = config.getSqoopMapperNum();

    String bquery;
    bquery = String.format(Locale.ROOT, "SELECT min(%s), max(%s) FROM %s.%s as %s",
            splitColumn, splitColumn, splitDatabase, splitTable, splitTableAlias);
    bquery = dataSource.convertSql(bquery);

    if (partitionDesc.isPartitioned()) {
        SegmentRange segRange = flatDesc.getSegRange();
        if (segRange != null && !segRange.isInfinite()) {
            if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias)
                    && (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc
                            .getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
                String quotedPartCond = FlatTableSqlQuoteUtils.quoteIdentifierInSqlExpr(flatDesc,
                        partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(partitionDesc,
                                flatDesc.getSegment(), segRange, null), sqlDialect);
                bquery += " WHERE " + quotedPartCond;
            }
        }
    }

    bquery = escapeQuotationInSql(bquery);
    splitColumn = escapeQuotationInSql(splitColumn);
    String cmd = StringUtils.format(
            "--connect \"%s\" --driver \"%s\" --username \"%s\" --password \"%s\" --query \"%s AND \\$CONDITIONS\" "
                    + "--target-dir \"%s/%s\" --split-by \"%s\" --boundary-query \"%s\" --null-string '' "
                    + "--fields-terminated-by '%s' --num-mappers %d",
            dataSource.getJdbcUrl(), dataSource.getJdbcDriver(), dataSource.getJdbcUser(),
            dataSource.getJdbcPassword(), selectSql, jobWorkingDir, hiveTable, splitColumn, bquery,
            filedDelimiter, mapperNum);
    if (iconfigurer.getTransactionIsolationLevel() != null) {
        cmd = cmd + " --relaxed-isolation --metadata-transaction-isolation-level "
                + iconfigurer.getTransactionIsolationLevel();
    }
    logger.debug("sqoop cmd: {}", cmd);

    SqoopCmdStep step = new SqoopCmdStep();
    step.setCmd(cmd);
    step.setName(ExecutableConstants.STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE);
    return step;
}
Example 7
Source File: JoinedFlatTable.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
public static String colName(TblColRef col, boolean useAlias) {
    return useAlias ? col.getTableAlias() + "_" + col.getName() : col.getName();
}
Example 8
Source File: CubeDesc.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
/**
 * Get columns that have dictionary
 */
public Set<TblColRef> getAllColumnsHaveDictionary() {
    Set<TblColRef> result = Sets.newLinkedHashSet();

    // dictionaries in dimensions
    result.addAll(getAllDimsHaveDictionary());

    // dictionaries in measures
    for (MeasureDesc measure : measures) {
        MeasureType<?> aggrType = measure.getFunction().getMeasureType();
        result.addAll(aggrType.getColumnsNeedDictionary(measure.getFunction()));
    }

    // any additional dictionaries
    if (dictionaries != null) {
        for (DictionaryDesc dictDesc : dictionaries) {
            TblColRef col = dictDesc.getColumnRef();
            result.add(col);
        }
    }

    //mr - hive global dict
    if (overrideKylinProps.containsKey("kylin.dictionary.mr-hive.columns")) {
        String mrHiveDictColumns = overrideKylinProps.get("kylin.dictionary.mr-hive.columns");
        if (StringUtils.isNotEmpty(mrHiveDictColumns)) {
            String[] mrHiveDictColumnArr = mrHiveDictColumns.split(",");
            for (String dictColumn : mrHiveDictColumnArr) {
                Iterator<TblColRef> it = result.iterator();
                while (it.hasNext()) {
                    TblColRef colRef = it.next();
                    String aliasCol = colRef.getTableAlias() + "_" + colRef.getName();
                    if (aliasCol.equalsIgnoreCase(dictColumn)) {
                        logger.debug("Remove column {} because it has been built by MR", aliasCol);
                        it.remove();
                    }
                }
            }
        }
    }

    return result;
}
Example 9
Source File: JdbcHiveInputBase.java    From kylin with Apache License 2.0 4 votes vote down vote up 
protected AbstractExecutable createSqoopToFlatHiveStep(String jobWorkingDir, String cubeName) {
    KylinConfig config = getConfig();
    PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc();
    String partCol = null;

    if (partitionDesc.isPartitioned()) {
        partCol = partitionDesc.getPartitionDateColumn();//tablename.colname
    }

    String splitTableAlias;
    String splitColumn;
    String splitDatabase;
    TblColRef splitColRef = determineSplitColumn();
    splitTableAlias = splitColRef.getTableAlias();

    splitColumn = getColumnIdentityQuoted(splitColRef, jdbcMetadataDialect, metaMap, true);
    splitDatabase = splitColRef.getColumnDesc().getTable().getDatabase();

    String selectSql = generateSelectDataStatementRDBMS(flatDesc, true, new String[] { partCol },
            jdbcMetadataDialect, metaMap);
    selectSql = escapeQuotationInSql(selectSql);

    String hiveTable = flatDesc.getTableName();
    String connectionUrl = config.getJdbcSourceConnectionUrl();
    String driverClass = config.getJdbcSourceDriver();
    String jdbcUser = config.getJdbcSourceUser();
    String jdbcPass = config.getJdbcSourcePass();
    String sqoopHome = config.getSqoopHome();
    String sqoopNullString = config.getSqoopNullString();
    String sqoopNullNonString = config.getSqoopNullNonString();
    String filedDelimiter = config.getJdbcSourceFieldDelimiter();
    int mapperNum = config.getSqoopMapperNum();

    String bquery = String.format(Locale.ROOT, "SELECT min(%s), max(%s) FROM %s.%s ", splitColumn,
            splitColumn, getSchemaQuoted(metaMap, splitDatabase, jdbcMetadataDialect, true),
            getTableIdentityQuoted(splitColRef.getTableRef(), metaMap, jdbcMetadataDialect, true));
    if (partitionDesc.isPartitioned()) {
        SegmentRange segRange = flatDesc.getSegRange();
        if (segRange != null && !segRange.isInfinite()) {
            if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias)
                    && (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc
                            .getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {

                String quotedPartCond = partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(
                        partitionDesc, flatDesc.getSegment(), segRange,
                        col -> getTableColumnIdentityQuoted(col, jdbcMetadataDialect, metaMap, true));
                bquery += " WHERE " + quotedPartCond;
            }
        }
    }
    bquery = escapeQuotationInSql(bquery);

    // escape ` in cmd
    splitColumn = escapeQuotationInSql(splitColumn);

    String cmd = String.format(Locale.ROOT, "%s/bin/sqoop import" + generateSqoopConfigArgString()
            + "--connect \"%s\" --driver %s --username %s --password \"%s\" --query \"%s AND \\$CONDITIONS\" "
            + "--target-dir %s/%s --split-by %s --boundary-query \"%s\" --null-string '%s' "
            + "--null-non-string '%s' --fields-terminated-by '%s' --num-mappers %d", sqoopHome, connectionUrl,
            driverClass, jdbcUser, jdbcPass, selectSql, jobWorkingDir, hiveTable, splitColumn, bquery,
            sqoopNullString, sqoopNullNonString, filedDelimiter, mapperNum);
    logger.debug("sqoop cmd : {}", cmd);
    CmdStep step = new CmdStep();
    step.setCmd(cmd);
    step.setName(ExecutableConstants.STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE);
    return step;
}
Example 10
Source File: JdbcHiveInputBase.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Override
protected AbstractExecutable createSqoopToFlatHiveStep(String jobWorkingDir, String cubeName) {
    KylinConfig config = flatDesc.getDataModel().getConfig();
    PartitionDesc partitionDesc = flatDesc.getDataModel().getPartitionDesc();
    String partCol = null;
    boolean enableQuote = dataSource.getSqlConverter().getConfigurer().enableQuote();
    enableQuote = enableQuote && config.enableHiveDdlQuote();
    logger.debug("Quote switch is set to {}", enableQuote);
    SqlDialect sqlDialect = enableQuote ? dataSource.getSqlConverter().getConfigurer().getSqlDialect() : FlatTableSqlQuoteUtils.NON_QUOTE_DIALECT;
    SqlConverter.IConfigurer iconfigurer = dataSource.getSqlConverter().getConfigurer();

    if (partitionDesc.isPartitioned()) {
        partCol = partitionDesc.getPartitionDateColumn(); //tablename.colname
    }

    String splitTable;
    String splitTableAlias;
    String splitColumn;
    String splitDatabase;
    TblColRef splitColRef = determineSplitColumn();
    splitTable = splitColRef.getTableRef().getTableDesc().getName();
    splitTableAlias = splitColRef.getTableAlias();
    splitDatabase = splitColRef.getColumnDesc().getTable().getDatabase();

    if (enableQuote) {
        splitColumn = sqlDialect.quoteIdentifier(splitColRef.getTableAlias()) + "."
                + sqlDialect.quoteIdentifier(splitColRef.getName());
        splitDatabase = sqlDialect.quoteIdentifier(splitDatabase);
        splitTable = sqlDialect.quoteIdentifier(splitTable);
        splitTableAlias = sqlDialect.quoteIdentifier(splitTableAlias);
    } else {
        splitColumn = splitColRef.getTableAlias() + "." + splitColRef.getName();
    }

    String selectSql = JoinedFlatTable.generateSelectDataStatement(flatDesc, true, new String[]{partCol}, sqlDialect);
    selectSql = escapeQuotationInSql(dataSource.convertSql(selectSql));

    String hiveTable = flatDesc.getTableName();
    String filedDelimiter = config.getJdbcSourceFieldDelimiter();
    int mapperNum = config.getSqoopMapperNum();

    String bquery;
    bquery = String.format(Locale.ROOT, "SELECT min(%s), max(%s) FROM %s.%s as %s",
            splitColumn, splitColumn, splitDatabase, splitTable, splitTableAlias);
    bquery = dataSource.convertSql(bquery);

    if (partitionDesc.isPartitioned()) {
        SegmentRange segRange = flatDesc.getSegRange();
        if (segRange != null && !segRange.isInfinite()) {
            if (partitionDesc.getPartitionDateColumnRef().getTableAlias().equals(splitTableAlias)
                    && (partitionDesc.getPartitionTimeColumnRef() == null || partitionDesc
                    .getPartitionTimeColumnRef().getTableAlias().equals(splitTableAlias))) {
                String quotedPartCond = FlatTableSqlQuoteUtils.quoteIdentifierInSqlExpr(flatDesc,
                        partitionDesc.getPartitionConditionBuilder().buildDateRangeCondition(partitionDesc,
                                flatDesc.getSegment(), segRange, null), sqlDialect);
                bquery += " WHERE " + quotedPartCond;
            }
        }
    }

    bquery = escapeQuotationInSql(bquery);
    splitColumn = escapeQuotationInSql(splitColumn);
    String cmd = StringUtils.format(
            "--connect \"%s\" --driver \"%s\" --username \"%s\" --password \"%s\" --query \"%s AND \\$CONDITIONS\" "
                    + "--target-dir \"%s/%s\" --split-by \"%s\" --boundary-query \"%s\" --null-string '' "
                    + "--fields-terminated-by '%s' --num-mappers %d",
            dataSource.getJdbcUrl(), dataSource.getJdbcDriver(), dataSource.getJdbcUser(),
            dataSource.getJdbcPassword(), selectSql, jobWorkingDir, hiveTable, splitColumn, bquery,
            filedDelimiter, mapperNum);
    if (iconfigurer.getTransactionIsolationLevel() != null) {
        cmd = cmd + " --relaxed-isolation --metadata-transaction-isolation-level "
                + iconfigurer.getTransactionIsolationLevel();
    }
    logger.debug("sqoop cmd: {}", cmd);

    SqoopCmdStep step = new SqoopCmdStep();
    step.setCmd(cmd);
    step.setName(ExecutableConstants.STEP_NAME_SQOOP_TO_FLAT_HIVE_TABLE);
    return step;
}
Example 11
Source File: JoinedFlatTable.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static String colName(TblColRef col, boolean useAlias) {
    return useAlias ? col.getTableAlias() + "_" + col.getName() : col.getName();
}
Example 12
Source File: CubeDesc.java    From kylin with Apache License 2.0 4 votes vote down vote up 
/**
 * Get columns that have dictionary
 */
public Set<TblColRef> getAllColumnsHaveDictionary() {
    Set<TblColRef> result = Sets.newLinkedHashSet();

    // dictionaries in dimensions
    result.addAll(getAllDimsHaveDictionary());

    // dictionaries in measures
    for (MeasureDesc measure : measures) {
        MeasureType<?> aggrType = measure.getFunction().getMeasureType();
        result.addAll(aggrType.getColumnsNeedDictionary(measure.getFunction()));
    }

    // any additional dictionaries
    if (dictionaries != null) {
        for (DictionaryDesc dictDesc : dictionaries) {
            TblColRef col = dictDesc.getColumnRef();
            result.add(col);
        }
    }

    //mr - hive global dict
    if (overrideKylinProps.containsKey("kylin.dictionary.mr-hive.columns")) {
        String mrHiveDictColumns = overrideKylinProps.get("kylin.dictionary.mr-hive.columns");
        if (StringUtils.isNotEmpty(mrHiveDictColumns)) {
            String[] mrHiveDictColumnArr = mrHiveDictColumns.split(",");
            for (String dictColumn : mrHiveDictColumnArr) {
                Iterator<TblColRef> it = result.iterator();
                while (it.hasNext()) {
                    TblColRef colRef = it.next();
                    String aliasCol = colRef.getTableAlias() + "_" + colRef.getName();
                    if (aliasCol.equalsIgnoreCase(dictColumn)) {
                        logger.debug("Remove column {} because it has been built by MR", aliasCol);
                        it.remove();
                    }
                }
            }
        }
    }

    return result;
}