Java Code Examples for org.apache.kylin.metadata.model.TableDesc#setColumns()

The following examples show how to use org.apache.kylin.metadata.model.TableDesc#setColumns() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: KylinTableCreator.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public static TableDesc generateKylinTable(KylinConfig kylinConfig, MetricsSinkDesc sinkDesc, String subject,
                                           List<Pair<String, String>> columns) {
    TableDesc kylinTable = new TableDesc();

    Pair<String, String> tableNameSplits = ActiveReservoirReporter
            .getTableNameSplits(sinkDesc.getTableNameForMetrics(subject));
    kylinTable.setUuid(RandomUtil.randomUUID().toString());
    kylinTable.setDatabase(tableNameSplits.getFirst());
    kylinTable.setName(tableNameSplits.getSecond());
    kylinTable.setTableType(null);
    kylinTable.setLastModified(0L);
    kylinTable.setSourceType(sinkDesc.getSourceType());

    ColumnDesc[] columnDescs = new ColumnDesc[columns.size()];
    for (int i = 0; i < columns.size(); i++) {
        columnDescs[i] = new ColumnDesc();
        Pair<String, String> entry = columns.get(i);
        columnDescs[i].setId(Integer.toString(i + 1));
        columnDescs[i].setName(entry.getFirst());
        columnDescs[i].setDatatype(entry.getSecond());
    }
    kylinTable.setColumns(columnDescs);

    kylinTable.init(kylinConfig, MetricsManager.SYSTEM_PROJECT);

    return kylinTable;
}

Example 2

Source File: JdbcExplorer.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj)
        throws SQLException {
    TableDesc tableDesc = new TableDesc();
    tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
    tableDesc.setName(table.toUpperCase(Locale.ROOT));
    tableDesc.setUuid(RandomUtil.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setSourceType(ISourceAware.ID_JDBC);

    Connection con = SqlUtil.getConnection(dbconf);
    DatabaseMetaData dbmd = con.getMetaData();

    try (ResultSet rs = jdbcMetadataDialect.getTable(dbmd, database, table)) {
        String tableType = null;
        while (rs.next()) {
            tableType = rs.getString("TABLE_TYPE");
        }
        if (tableType != null) {
            tableDesc.setTableType(tableType);
        } else {
            throw new RuntimeException(
                    String.format(Locale.ROOT, "table %s not found in schema:%s", table, database));
        }
    }

    try (ResultSet rs = jdbcMetadataDialect.listColumns(dbmd, database, table)) {
        tableDesc.setColumns(extractColumnFromMeta(rs));
    } finally {
        DBUtils.closeQuietly(con);
    }

    TableExtDesc tableExtDesc = new TableExtDesc();
    tableExtDesc.setIdentity(tableDesc.getIdentity());
    tableExtDesc.setUuid(RandomUtil.randomUUID().toString());
    tableExtDesc.setLastModified(0);
    tableExtDesc.init(prj);

    return Pair.newPair(tableDesc, tableExtDesc);
}

Example 3

Source File: SnapshotManagerTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private TableDesc genTableDesc(String tableName) {
    TableDesc table = TableDesc.mockup(tableName);
    ColumnDesc desc1 = new ColumnDesc("1", "id", "string", null, null, null, null);
    desc1.setId("1");
    desc1.setDatatype("long");
    ColumnDesc desc2 = new ColumnDesc("2", "country", "string", null, null, null, null);
    desc2.setId("2");
    desc2.setDatatype("string");
    ColumnDesc[] columns = { desc1, desc2 };
    table.setColumns(columns);
    table.init(kylinConfig, "default");
    return table;
}

Example 4

Source File: TableService.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public TableDesc generateCsvTableDesc(String tableName, List<String> columnDescList) throws IOException {
    String[] strs = tableName.split("\\.");
    if (strs.length != 2) {
        throw new IllegalArgumentException("Invalid table name + '" + tableName + "'");
    }
    TableDesc tableDesc = new TableDesc();

    tableDesc.setDatabase(strs[0]);
    tableDesc.setName(strs[1]);
    tableDesc.setUuid(RandomUtil.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setSourceType(ISourceAware.ID_SPARK);
    List<ColumnDesc> columnDescs = new ArrayList<>();
    int index = 0;

    for (String csvColumnDescStr : columnDescList) {
        index++;
        ColumnDesc columnDesc = new ColumnDesc();
        CsvColumnDesc csvColumnDesc = JsonUtil.readValue(csvColumnDescStr, CsvColumnDesc.class);
        columnDesc.setId("" + index);
        columnDesc.setName((csvColumnDesc).getName());
        columnDesc.setDatatype((csvColumnDesc).getType());
        columnDescs.add(columnDesc);
    }

    tableDesc.setColumns(columnDescs.toArray(new ColumnDesc[columnDescs.size()]));

    return tableDesc;
}

Example 5

Source File: KylinTableCreator.java From kylin with Apache License 2.0

5 votes

public static TableDesc generateKylinTable(KylinConfig kylinConfig, MetricsSinkDesc sinkDesc, String subject,
                                           List<Pair<String, String>> columns) {
    TableDesc kylinTable = new TableDesc();

    Pair<String, String> tableNameSplits = ActiveReservoirReporter
            .getTableNameSplits(sinkDesc.getTableNameForMetrics(subject));
    kylinTable.setUuid(RandomUtil.randomUUID().toString());
    kylinTable.setDatabase(tableNameSplits.getFirst());
    kylinTable.setName(tableNameSplits.getSecond());
    kylinTable.setTableType(null);
    kylinTable.setLastModified(0L);
    kylinTable.setSourceType(sinkDesc.getSourceType());

    ColumnDesc[] columnDescs = new ColumnDesc[columns.size()];
    for (int i = 0; i < columns.size(); i++) {
        columnDescs[i] = new ColumnDesc();
        Pair<String, String> entry = columns.get(i);
        columnDescs[i].setId(Integer.toString(i + 1));
        columnDescs[i].setName(entry.getFirst());
        columnDescs[i].setDatatype(entry.getSecond());
    }
    kylinTable.setColumns(columnDescs);

    kylinTable.init(kylinConfig, MetricsManager.SYSTEM_PROJECT);

    return kylinTable;
}

Example 6

Source File: JdbcExplorer.java From kylin with Apache License 2.0

5 votes

@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj)
        throws SQLException {
    TableDesc tableDesc = new TableDesc();
    tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
    tableDesc.setName(table.toUpperCase(Locale.ROOT));
    tableDesc.setUuid(RandomUtil.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setSourceType(ISourceAware.ID_JDBC);

    Connection con = SqlUtil.getConnection(dbconf);
    DatabaseMetaData dbmd = con.getMetaData();

    try (ResultSet rs = jdbcMetadataDialect.getTable(dbmd, database, table)) {
        String tableType = null;
        while (rs.next()) {
            tableType = rs.getString("TABLE_TYPE");
        }
        if (tableType != null) {
            tableDesc.setTableType(tableType);
        } else {
            throw new RuntimeException(
                    String.format(Locale.ROOT, "table %s not found in schema:%s", table, database));
        }
    }

    try (ResultSet rs = jdbcMetadataDialect.listColumns(dbmd, database, table)) {
        tableDesc.setColumns(extractColumnFromMeta(rs));
    } finally {
        DBUtils.closeQuietly(con);
    }

    TableExtDesc tableExtDesc = new TableExtDesc();
    tableExtDesc.setIdentity(tableDesc.getIdentity());
    tableExtDesc.setUuid(RandomUtil.randomUUID().toString());
    tableExtDesc.setLastModified(0);
    tableExtDesc.init(prj);

    return Pair.newPair(tableDesc, tableExtDesc);
}

Example 7

Source File: SnapshotManagerTest.java From kylin with Apache License 2.0

5 votes

private TableDesc genTableDesc(String tableName) {
    TableDesc table = TableDesc.mockup(tableName);
    ColumnDesc desc1 = new ColumnDesc("1", "id", "string", null, null, null, null);
    desc1.setId("1");
    desc1.setDatatype("long");
    ColumnDesc desc2 = new ColumnDesc("2", "country", "string", null, null, null, null);
    desc2.setId("2");
    desc2.setDatatype("string");
    ColumnDesc[] columns = { desc1, desc2 };
    table.setColumns(columns);
    table.init(kylinConfig, "default");
    return table;
}

Example 8

Source File: JdbcExplorer.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj)
        throws SQLException {
    TableDesc tableDesc = new TableDesc();
    tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
    tableDesc.setName(table.toUpperCase(Locale.ROOT));
    tableDesc.setUuid(UUID.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setProject(prj);
    tableDesc.setSourceType(JdbcSource.SOURCE_ID);

    try (CachedRowSet tables = dataSource.getTable(database, table)) {
        String tableType = null;
        while (tables.next()) {
            tableType = tables.getString("TABLE_TYPE");
        }
        if (tableType != null) {
            tableDesc.setTableType(tableType);
        } else {
            throw new RuntimeException(String.format(Locale.ROOT, "table %s not found in schema:%s", table, database));
        }
    }

    try (CachedRowSet columns = dataSource.listColumns(database, table)) {
        List<ColumnDesc> columnDescs = new ArrayList<>();

        while (columns.next()) {
            String cname = columns.getString("COLUMN_NAME");
            int type = columns.getInt("DATA_TYPE");
            int csize = columns.getInt("COLUMN_SIZE");
            int digits = columns.getInt("DECIMAL_DIGITS");
            int pos = columns.getInt("ORDINAL_POSITION");
            String remarks = columns.getString("REMARKS");

            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(cname.toUpperCase(Locale.ROOT));

            String kylinType = dataSource.toKylinTypeName(type);
            if ("any".equals(kylinType)) {
                String typeName = columns.getString("TYPE_NAME");
                int kylinTypeId = dataSource.toKylinTypeId(typeName, type);
                kylinType = dataSource.toKylinTypeName(kylinTypeId);
            }
            int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1;
            precision = Math.min(precision, KylinConfig.getInstanceFromEnv().getDefaultVarcharPrecision());
            int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1;

            cdesc.setDatatype(new DataType(kylinType, precision, scale).toString());
            cdesc.setId(String.valueOf(pos));
            cdesc.setComment(remarks);
            columnDescs.add(cdesc);
        }

        tableDesc.setColumns(columnDescs.toArray(new ColumnDesc[columnDescs.size()]));

        TableExtDesc tableExtDesc = new TableExtDesc();
        tableExtDesc.setIdentity(tableDesc.getIdentity());
        tableExtDesc.setUuid(UUID.randomUUID().toString());
        tableExtDesc.setLastModified(0);
        tableExtDesc.init(prj);

        return Pair.newPair(tableDesc, tableExtDesc);
    }
}

Example 9

Source File: JdbcExplorer.java From kylin with Apache License 2.0

4 votes

@Override
public Pair<TableDesc, TableExtDesc> loadTableMetadata(String database, String table, String prj)
        throws SQLException {
    TableDesc tableDesc = new TableDesc();
    tableDesc.setDatabase(database.toUpperCase(Locale.ROOT));
    tableDesc.setName(table.toUpperCase(Locale.ROOT));
    tableDesc.setUuid(UUID.randomUUID().toString());
    tableDesc.setLastModified(0);
    tableDesc.setProject(prj);
    tableDesc.setSourceType(JdbcSource.SOURCE_ID);

    try (CachedRowSet tables = dataSource.getTable(database, table)) {
        String tableType = null;
        while (tables.next()) {
            tableType = tables.getString("TABLE_TYPE");
        }
        if (tableType != null) {
            tableDesc.setTableType(tableType);
        } else {
            throw new RuntimeException(String.format(Locale.ROOT, "table %s not found in schema:%s", table, database));
        }
    }

    try (CachedRowSet columns = dataSource.listColumns(database, table)) {
        List<ColumnDesc> columnDescs = new ArrayList<>();

        while (columns.next()) {
            String cname = columns.getString("COLUMN_NAME");
            int type = columns.getInt("DATA_TYPE");
            int csize = columns.getInt("COLUMN_SIZE");
            int digits = columns.getInt("DECIMAL_DIGITS");
            int pos = columns.getInt("ORDINAL_POSITION");
            String remarks = columns.getString("REMARKS");

            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(cname.toUpperCase(Locale.ROOT));

            String kylinType = dataSource.toKylinTypeName(type);
            if ("any".equals(kylinType)) {
                String typeName = columns.getString("TYPE_NAME");
                int kylinTypeId = dataSource.toKylinTypeId(typeName, type);
                kylinType = dataSource.toKylinTypeName(kylinTypeId);
            }
            int precision = (SqlUtil.isPrecisionApplicable(kylinType) && csize > 0) ? csize : -1;
            precision = Math.min(precision, KylinConfig.getInstanceFromEnv().getDefaultVarcharPrecision());
            int scale = (SqlUtil.isScaleApplicable(kylinType) && digits > 0) ? digits : -1;

            cdesc.setDatatype(new DataType(kylinType, precision, scale).toString());
            cdesc.setId(String.valueOf(pos));
            cdesc.setComment(remarks);
            columnDescs.add(cdesc);
        }

        tableDesc.setColumns(columnDescs.toArray(new ColumnDesc[columnDescs.size()]));

        TableExtDesc tableExtDesc = new TableExtDesc();
        tableExtDesc.setIdentity(tableDesc.getIdentity());
        tableExtDesc.setUuid(UUID.randomUUID().toString());
        tableExtDesc.setLastModified(0);
        tableExtDesc.init(prj);

        return Pair.newPair(tableDesc, tableExtDesc);
    }
}