org.apache.kylin.metadata.model.ColumnDesc Java Examples

@Override
public boolean moveNext() {
    boolean hasNext = iterator.hasNext();
    if (hasNext) {
        String[] row = iterator.next();
        for (int i = 0, n = colDescs.size(); i < n; i++) {
            ColumnDesc colDesc = colDescs.get(i);
            int colIdx = colDesc.getZeroBasedIndex();
            if (colIdx >= 0) {
                current[i] = Tuple.convertOptiqCellValue(row[colIdx], colDesc.getType().getName());
            } else {
                current[i] = null; // fake column
            }
        }
    }
    return hasNext;
}
 

private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) {
    for (TableDesc t : tables) {
        if (t.isView())
            continue;

        out.print("DROP TABLE IF EXISTS " + normHiveIdentifier(t.getIdentity()) + ";\n");

        out.print("CREATE TABLE " + normHiveIdentifier(t.getIdentity()) + "(" + "\n");

        for (int i = 0; i < t.getColumns().length; i++) {
            ColumnDesc col = t.getColumns()[i];
            out.print("    ");
            if (i > 0) {
                out.print(",");
            }
            out.print(normHiveIdentifier(col.getName()) + " " + hiveType(col.getType()) + "\n");
        }

        out.print(")" + "\n");
        out.print("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
        out.print("STORED AS TEXTFILE" + ";\n");
        out.print("\n");
    }
}
 

private boolean isColumnCompatible(ColumnDesc column, ColumnDesc newCol) {
    if (!column.getName().equalsIgnoreCase(newCol.getName())) {
        return false;
    }

    if (column.getType().isIntegerFamily()) {
        // OLAPTable.listSourceColumns converts some integer columns to bigint,
        // therefore strict type comparison won't work.
        // changing from one integer type to another should be fine.
        return newCol.getType().isIntegerFamily();
    } else if (column.getType().isNumberFamily()) {
        // Both are float/double should be fine.
        return newCol.getType().isNumberFamily();
    } else {
        // only compare base type name, changing precision or scale should be fine
        return column.getTypeName().equals(newCol.getTypeName());
    }
}
 

private Set<ColumnDesc> findUsedColumnsInFactTable(DataModelDesc usedModel, TableDesc factTable) {
    Set<ColumnDesc> usedColumns = Sets.newHashSet();
    // column in dimension
    for (ModelDimensionDesc dim : usedModel.getDimensions()) {
        if (dim.getTable().equalsIgnoreCase(factTable.getName())) {
            for (String col : dim.getColumns()) {
                usedColumns.add(mustGetColumnDesc(factTable, col));
            }
        }
    }

    // column in measure
    for (String columnInMeasure : usedModel.getMetrics()) {
        if (factTable.getName().equalsIgnoreCase(getTableName(columnInMeasure))) {
            usedColumns.add(mustGetColumnDesc(factTable, columnInMeasure));
        }
    }

    return usedColumns;
}
 

private String generateCreateH2TableSql(TableDesc tableDesc, String csvFilePath) {
    StringBuilder ddl = new StringBuilder();
    StringBuilder csvColumns = new StringBuilder();

    ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n");
    ddl.append("(" + "\n");

    for (int i = 0; i < tableDesc.getColumns().length; i++) {
        ColumnDesc col = tableDesc.getColumns()[i];
        if (col.isComputedColumn()) {
            continue;
        }
        if (i > 0) {
            ddl.append(",");
            csvColumns.append(",");
        }
        ddl.append(col.getName() + " " + getH2DataType((col.getDatatype())) + "\n");
        csvColumns.append(col.getName());
    }
    ddl.append(")" + "\n");
    ddl.append("AS SELECT * FROM CSVREAD('" + csvFilePath + "', '" + csvColumns
            + "', 'charset=UTF-8 fieldSeparator=,');");

    return ddl.toString();
}
 

@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    int bodyOffset = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        String datatype = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        col.setDatatype(datatype);
        col.init(table);
        cols[i] = col.getRef();

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes, bodyOffset);
}
 

/**
 * Used to quote identifiers for JDBC ext job when quoting cc expr
 * @param tableDesc
 * @param sqlExpr
 * @return
 */
public static String quoteIdentifierInSqlExpr(TableDesc tableDesc, String sqlExpr, SqlDialect sqlDialect) {
    String table = tableDesc.getName();
    boolean tableMatched = false;
    List<String> tabPatterns = getTableNameOrAliasPatterns(table);
    if (isIdentifierNeedToQuote(sqlExpr, table, tabPatterns)) {
        sqlExpr = quoteIdentifier(sqlExpr, table, tabPatterns, sqlDialect);
        tableMatched = true;
    }

    if (tableMatched) {
        for (ColumnDesc columnDesc : tableDesc.getColumns()) {
            String column = columnDesc.getName();
            List<String> colPatterns = getColumnNameOrAliasPatterns(column);
            if (isIdentifierNeedToQuote(sqlExpr, column, colPatterns)) {
                sqlExpr = quoteIdentifier(sqlExpr, column, colPatterns, sqlDialect);
            }
        }
    }

    return sqlExpr;
}
 

@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        cols[i] = new TblColRef(col);

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes);
}
 

protected List<TblColRef> buildGroups() {
    List<TblColRef> groups = new ArrayList<TblColRef>();

    TableDesc t1 = new TableDesc();
    t1.setName("TEST_KYLIN_FACT");
    t1.setDatabase("DEFAULT");
    ColumnDesc c1 = new ColumnDesc();
    c1.setName("CAL_DT");
    c1.setDatatype("String");
    c1.setTable(t1);
    TblColRef cf1 = new TblColRef(c1);
    groups.add(cf1);

    TableDesc t2 = new TableDesc();
    t2.setName("TEST_CATEGORY_GROUPINGS");
    t2.setDatabase("DEFAULT");
    ColumnDesc c2 = new ColumnDesc();
    c2.setName("META_CATEG_NAME");
    c1.setDatatype("String");
    c2.setTable(t2);
    TblColRef cf2 = new TblColRef(c2);
    groups.add(cf2);

    return groups;
}
 

@Test
public void testSerialize07() {
    TableDesc table = new TableDesc();
    table.setName("TEST_KYLIN_FACT");
    table.setDatabase("DEFAULT");

    ColumnDesc column = new ColumnDesc();
    column.setTable(table);
    TblColRef colRef = new TblColRef(column);
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);
    TupleFilter filter = buildCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
 

private List<String> generateCreateH2IndexSql(TableDesc tableDesc) {
    List<String> result = Lists.newArrayList();
    int x = 0;
    for (ColumnDesc col : tableDesc.getColumns()) {
        if ("T".equalsIgnoreCase(col.getIndex())) {
            StringBuilder ddl = new StringBuilder();
            ddl.append("CREATE INDEX IDX_" + tableDesc.getName() + "_" + x + " ON " + tableDesc.getIdentity() + "("
                    + col.getName() + ")");
            ddl.append("\n");
            result.add(ddl.toString());
            x++;
        }
    }

    return result;
}
 

private List<TblColRef> buildGroups() {
    List<TblColRef> groups = new ArrayList<TblColRef>();

    TableDesc t1 = new TableDesc();
    t1.setName("TEST_KYLIN_FACT");
    t1.setDatabase("DEFAULT");
    ColumnDesc c1 = new ColumnDesc();
    c1.setName("CAL_DT");
    c1.setTable(t1);
    c1.setDatatype("string");
    TblColRef cf1 = new TblColRef(c1);
    groups.add(cf1);

    TableDesc t2 = new TableDesc();
    t2.setName("TEST_CATEGORY_GROUPINGS");
    t2.setDatabase("DEFAULT");
    ColumnDesc c2 = new ColumnDesc();
    c2.setName("META_CATEG_NAME");
    c2.setTable(t2);
    c2.setDatatype("string");
    TblColRef cf2 = new TblColRef(c2);
    groups.add(cf2);

    return groups;
}
 

private String[] generateCreateTableSql(TableDesc tableDesc) {
    logger.info("Generate create table sql: {}", tableDesc);
    String tableIdentity = String
            .format(Locale.ROOT, "%s.%s", tableDesc.getDatabase().toUpperCase(Locale.ROOT), tableDesc.getName())
            .toUpperCase(Locale.ROOT);
    String dropsql = "DROP TABLE IF EXISTS " + tableIdentity;
    String dropsql2 = "DROP VIEW IF EXISTS " + tableIdentity;

    StringBuilder ddl = new StringBuilder();
    ddl.append("CREATE TABLE " + tableIdentity + "\n");
    ddl.append("(" + "\n");

    for (int i = 0; i < tableDesc.getColumns().length; i++) {
        ColumnDesc col = tableDesc.getColumns()[i];
        if (i > 0) {
            ddl.append(",");
        }
        ddl.append(col.getName() + " " + getSqlDataType((col.getDatatype())) + "\n");
    }

    ddl.append(")");

    return new String[] { dropsql, dropsql2, ddl.toString() };
}
 

private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) {
    for (TableDesc t : tables) {
        if (t.isView())
            continue;

        out.print("DROP TABLE IF EXISTS " + normHiveIdentifier(t.getIdentity()) + ";\n");

        out.print("CREATE TABLE " + normHiveIdentifier(t.getIdentity()) + "(" + "\n");

        for (int i = 0; i < t.getColumns().length; i++) {
            ColumnDesc col = t.getColumns()[i];
            out.print("    ");
            if (i > 0) {
                out.print(",");
            }
            out.print(normHiveIdentifier(col.getName()) + " " + hiveType(col.getType()) + "\n");
        }

        out.print(")" + "\n");
        out.print("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
        out.print("STORED AS TEXTFILE" + ";\n");
        out.print("\n");
    }
}
 

@Override
public boolean moveNext() {
    boolean hasNext = iterator.hasNext();
    if (hasNext) {
        String[] row = iterator.next();
        for (int i = 0, n = colDescs.size(); i < n; i++) {
            ColumnDesc colDesc = colDescs.get(i);
            int colIdx = colDesc.getZeroBasedIndex();
            if (colIdx >= 0) {
                current[i] = Tuple.convertOptiqCellValue(row[colIdx], colDesc.getUpgradedType().getName());
            } else {
                current[i] = null; // fake column
            }
        }
    }
    return hasNext;
}
 

@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    int bodyOffset = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        String datatype = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        col.setDatatype(datatype);
        col.init(table);
        cols[i] = col.getRef();

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes, bodyOffset);
}
 

@Override
public void doMap(T key, Object value, Context context) throws IOException, InterruptedException {
    ColumnDesc[] columns = tableDesc.getColumns();
    Collection<String[]> valuesCollection = tableInputFormat.parseMapperInput(value);

    for (String[] values: valuesCollection) {
        for (int m = 0; m < columns.length; m++) {
            String field = columns[m].getName();
            String fieldValue = values[m];
            if (fieldValue == null)
                fieldValue = "NULL";

            if (counter < 5 && m < 10) {
                System.out.println("Get row " + counter + " column '" + field + "'  value: " + fieldValue);
            }

            getHllc(m).add(Bytes.toBytes(fieldValue.toString()));
        }

        counter++;
    }
}
 

private ColumnDesc[] extractColumnFromMeta(HiveTableMeta hiveTableMeta) {
    int columnNumber = hiveTableMeta.allColumns.size();
    List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber);

    for (int i = 0; i < columnNumber; i++) {
        HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);

        // skip unsupported fields, e.g. map<string, int>
        if (DataType.isKylinSupported(field.dataType)) {
            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(field.name.toUpperCase(Locale.ROOT));

            // use "double" in kylin for "float"
            if ("float".equalsIgnoreCase(field.dataType)) {
                cdesc.setDatatype("double");
            } else {
                cdesc.setDatatype(field.dataType);
            }

            cdesc.setId(String.valueOf(i + 1));
            cdesc.setComment(field.comment);
            columns.add(cdesc);
        } else {
            logger.warn("Unsupported data type {}, excluding the field '{}'.", field.dataType, field.name);
        }
    }

    return  columns.toArray(new ColumnDesc[0]);
}
 

@Test
public void testEvalQueryMetadata() {
    ColumnDesc[] columnDescs = explorer
            .evalQueryMetadata("select cal_dt, count(*) as cnt from DEFAULT.test_kylin_fact group by cal_dt");
    Assert.assertNotNull(columnDescs);
    Assert.assertEquals(2, columnDescs.length);
    Assert.assertEquals("date", columnDescs[0].getDatatype());
    Assert.assertEquals("CAL_DT", columnDescs[0].getName());
    Assert.assertEquals("bigint", columnDescs[1].getDatatype());
    Assert.assertEquals("CNT", columnDescs[1].getName());
}
 

private static boolean containsAll(Set<ColumnDesc> allColumnDescs, Set<TblColRef> allColumns) {
    for (TblColRef col : allColumns) {
        if (allColumnDescs.contains(col.getColumnDesc()) == false)
            return false;
    }
    return true;
}
 

private ColumnDesc[] extractColumnFromMeta(HiveTableMeta hiveTableMeta) {
    int columnNumber = hiveTableMeta.allColumns.size();
    List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber);

    for (int i = 0; i < columnNumber; i++) {
        HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);

        // skip unsupported fields, e.g. map<string, int>
        if (DataType.isKylinSupported(field.dataType)) {
            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(field.name.toUpperCase(Locale.ROOT));

            // use "double" in kylin for "float"
            if ("float".equalsIgnoreCase(field.dataType)) {
                cdesc.setDatatype("double");
            } else {
                cdesc.setDatatype(field.dataType);
            }

            cdesc.setId(String.valueOf(i + 1));
            cdesc.setComment(field.comment);
            columns.add(cdesc);
        } else {
            logger.warn("Unsupported data type {}, excluding the field '{}'.", field.dataType, field.name);
        }
    }

    return  columns.toArray(new ColumnDesc[0]);
}
 

@Test
public void testEvalQueryMetadata() {
    ColumnDesc[] columnDescs = explorer
            .evalQueryMetadata("select cal_dt, count(*) as cnt from DEFAULT.test_kylin_fact group by cal_dt");
    Assert.assertNotNull(columnDescs);
    Assert.assertEquals(2, columnDescs.length);
    Assert.assertEquals("date", columnDescs[0].getDatatype());
    Assert.assertEquals("CAL_DT", columnDescs[0].getName());
    Assert.assertEquals("bigint", columnDescs[1].getDatatype());
    Assert.assertEquals("CNT", columnDescs[1].getName());
}
 

private void initMeasureColumns(Map<String, TableDesc> tables) {
    if (measures == null || measures.isEmpty()) {
        return;
    }

    TableDesc factTable = tables.get(getFactTable());
    for (MeasureDesc m : measures) {
        m.setName(m.getName().toUpperCase());

        if (m.getDependentMeasureRef() != null) {
            m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase());
        }
        
        FunctionDesc f = m.getFunction();
        f.setExpression(f.getExpression().toUpperCase());
        f.setReturnDataType(DataType.getInstance(f.getReturnType()));

        ParameterDesc p = f.getParameter();
        p.normalizeColumnValue();

        if (p.isColumnType()) {
            ArrayList<TblColRef> colRefs = Lists.newArrayList();
            for (String cName : p.getValue().split("\\s*,\\s*")) {
                ColumnDesc sourceColumn = factTable.findColumnByName(cName);
                TblColRef colRef = new TblColRef(sourceColumn);
                colRefs.add(colRef);
                allColumns.add(colRef);
            }
            if (colRefs.isEmpty() == false)
                p.setColRefs(colRefs);
        }
        
        // verify holistic count distinct as a dependent measure
        if (m.isHolisticCountDistinct() && StringUtils.isBlank(m.getDependentMeasureRef())) {
            throw new IllegalStateException(m + " is a holistic count distinct but it has no DependentMeasureRef defined!");
        }
    }
}
 

@Test
public void testSerialize07() {
    TableDesc table = new TableDesc();
    table.setName("TEST_KYLIN_FACT");
    table.setDatabase("DEFAULT");

    ColumnDesc column = new ColumnDesc();
    column.setTable(table);
    TblColRef colRef = column.getRef();
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);

    assertFilterSerDe(buildEQCompareFilter(groups, 0));
}
 

@Test
public void testSerialize05() {
    ColumnDesc column = new ColumnDesc();
    TblColRef colRef = column.getRef();
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);

    assertFilterSerDe(buildEQCompareFilter(groups, 0));
}
 

@Test
public void testLoadTableMetadata() throws SQLException {
    String tableName = "tb1";
    String databaseName = "testdb";
    ResultSet rs1 = mock(ResultSet.class);
    when(rs1.next()).thenReturn(true).thenReturn(false);
    when(rs1.getString("TABLE_TYPE")).thenReturn("TABLE");

    ResultSet rs2 = mock(ResultSet.class);
    when(rs2.next()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false);
    when(rs2.getString("COLUMN_NAME")).thenReturn("COL1").thenReturn("COL2").thenReturn("COL3");
    when(rs2.getInt("DATA_TYPE")).thenReturn(Types.VARCHAR).thenReturn(Types.INTEGER).thenReturn(Types.DECIMAL);
    when(rs2.getInt("COLUMN_SIZE")).thenReturn(128).thenReturn(10).thenReturn(19);
    when(rs2.getInt("DECIMAL_DIGITS")).thenReturn(0).thenReturn(0).thenReturn(4);
    when(rs2.getInt("ORDINAL_POSITION")).thenReturn(1).thenReturn(3).thenReturn(2);
    when(rs2.getString("REMARKS")).thenReturn("comment1").thenReturn("comment2").thenReturn("comment3");

    when(jdbcMetadata.getTable(dbmd, databaseName, tableName)).thenReturn(rs1);
    when(jdbcMetadata.listColumns(dbmd, databaseName, tableName)).thenReturn(rs2);

    Pair<TableDesc, TableExtDesc> result = jdbcExplorer.loadTableMetadata(databaseName, tableName, "proj");
    TableDesc tableDesc = result.getFirst();
    ColumnDesc columnDesc = tableDesc.getColumns()[1];

    Assert.assertEquals(databaseName.toUpperCase(Locale.ROOT), tableDesc.getDatabase());
    Assert.assertEquals(3, tableDesc.getColumnCount());
    Assert.assertEquals("TABLE", tableDesc.getTableType());
    Assert.assertEquals("COL2", columnDesc.getName());
    Assert.assertEquals("integer", columnDesc.getTypeName());
    Assert.assertEquals("comment2", columnDesc.getComment());
    Assert.assertEquals(databaseName.toUpperCase(Locale.ROOT) + "." + tableName.toUpperCase(Locale.ROOT),
            result.getSecond().getIdentity());
}
 

private TblColRef initDimensionColRef(DimensionDesc dim, String colName) {
    TableDesc table = dim.getTableDesc();
    ColumnDesc col = table.findColumnByName(colName);
    if (col == null)
        throw new IllegalArgumentException("No column '" + colName + "' found in table " + table);
    
    TblColRef ref = new TblColRef(col);
    return initDimensionColRef(ref);
}
 

private void init(ColumnDesc col, ModelDataGenerator modelGen) throws IOException {
    
    Map<String, String> config = Util.parseEqualCommaPairs(col.getDataGen(), "values");

    values = Arrays.asList(Util.parseString(config, "values", "").split("[|]"));
    
    List<String> pkValues = modelGen.getPkValuesIfIsFk(col);
    
    if (FK.equals(values.get(0)) || (values.get(0).isEmpty() && pkValues != null)) {
        isFK = true;
        values = getPkValues(modelGen, config, pkValues);
    } else if (ID.equals(values.get(0))) {
        isID = true;
        idStart = (values.size() > 1) ? Integer.parseInt(values.get(1)) : 0;
    } else if (RAND.equals(values.get(0)) || values.get(0).isEmpty()) {
        isRandom = true;
        randFormat = (values.size() > 1) ? values.get(1) : "";
        randStart = (values.size() > 2) ? Integer.parseInt(values.get(2)) : 0;
        randEnd = (values.size() > 3) ? Integer.parseInt(values.get(3)) : 0;
    } else {
        isDiscrete = true;
    }
    
    cardinality = Util.parseInt(config, "card", guessCardinality(col.getName()));
    genNull = Util.parseBoolean(config, "null", guessGenNull(col.getName()));
    genNullPct = Util.parseDouble(config, "nullpct", 0.01);
    genNullStr = Util.parseString(config, "nullstr", "\\N"); // '\N' is null in hive
    order = Util.parseBoolean(config, "order", false);
    unique = Util.parseBoolean(config, "uniq", modelGen.isPK(col));
}
 

private Set<ColumnDesc> asColumnDescs(Set<TblColRef> columns) {
    LinkedHashSet<ColumnDesc> result = new LinkedHashSet<>();
    for (TblColRef col : columns) {
        result.add(col.getColumnDesc());
    }
    return result;
}
 

private TableDesc genTableDesc(String tableName) {
    TableDesc table = TableDesc.mockup(tableName);
    ColumnDesc desc1 = new ColumnDesc("1", "id", "string", null, null, null, null);
    desc1.setId("1");
    desc1.setDatatype("long");
    ColumnDesc desc2 = new ColumnDesc("2", "country", "string", null, null, null, null);
    desc2.setId("2");
    desc2.setDatatype("string");
    ColumnDesc[] columns = { desc1, desc2 };
    table.setColumns(columns);
    table.init(kylinConfig, "default");
    return table;
}