org.apache.kylin.metadata.model.ColumnDesc Java Examples

The following examples show how to use org.apache.kylin.metadata.model.ColumnDesc. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LookupTableEnumerator.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Override
public boolean moveNext() {
    boolean hasNext = iterator.hasNext();
    if (hasNext) {
        String[] row = iterator.next();
        for (int i = 0, n = colDescs.size(); i < n; i++) {
            ColumnDesc colDesc = colDescs.get(i);
            int colIdx = colDesc.getZeroBasedIndex();
            if (colIdx >= 0) {
                current[i] = Tuple.convertOptiqCellValue(row[colIdx], colDesc.getType().getName());
            } else {
                current[i] = null; // fake column
            }
        }
    }
    return hasNext;
}
 
Example #2
Source File: ModelDataGenerator.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) {
    for (TableDesc t : tables) {
        if (t.isView())
            continue;

        out.print("DROP TABLE IF EXISTS " + normHiveIdentifier(t.getIdentity()) + ";\n");

        out.print("CREATE TABLE " + normHiveIdentifier(t.getIdentity()) + "(" + "\n");

        for (int i = 0; i < t.getColumns().length; i++) {
            ColumnDesc col = t.getColumns()[i];
            out.print("    ");
            if (i > 0) {
                out.print(",");
            }
            out.print(normHiveIdentifier(col.getName()) + " " + hiveType(col.getType()) + "\n");
        }

        out.print(")" + "\n");
        out.print("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
        out.print("STORED AS TEXTFILE" + ";\n");
        out.print("\n");
    }
}
 
Example #3
Source File: TableSchemaUpdateChecker.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private boolean isColumnCompatible(ColumnDesc column, ColumnDesc newCol) {
    if (!column.getName().equalsIgnoreCase(newCol.getName())) {
        return false;
    }

    if (column.getType().isIntegerFamily()) {
        // OLAPTable.listSourceColumns converts some integer columns to bigint,
        // therefore strict type comparison won't work.
        // changing from one integer type to another should be fine.
        return newCol.getType().isIntegerFamily();
    } else if (column.getType().isNumberFamily()) {
        // Both are float/double should be fine.
        return newCol.getType().isNumberFamily();
    } else {
        // only compare base type name, changing precision or scale should be fine
        return column.getTypeName().equals(newCol.getTypeName());
    }
}
 
Example #4
Source File: TableSchemaUpdateChecker.java    From kylin with Apache License 2.0 6 votes vote down vote up
private Set<ColumnDesc> findUsedColumnsInFactTable(DataModelDesc usedModel, TableDesc factTable) {
    Set<ColumnDesc> usedColumns = Sets.newHashSet();
    // column in dimension
    for (ModelDimensionDesc dim : usedModel.getDimensions()) {
        if (dim.getTable().equalsIgnoreCase(factTable.getName())) {
            for (String col : dim.getColumns()) {
                usedColumns.add(mustGetColumnDesc(factTable, col));
            }
        }
    }

    // column in measure
    for (String columnInMeasure : usedModel.getMetrics()) {
        if (factTable.getName().equalsIgnoreCase(getTableName(columnInMeasure))) {
            usedColumns.add(mustGetColumnDesc(factTable, columnInMeasure));
        }
    }

    return usedColumns;
}
 
Example #5
Source File: H2Database.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private String generateCreateH2TableSql(TableDesc tableDesc, String csvFilePath) {
    StringBuilder ddl = new StringBuilder();
    StringBuilder csvColumns = new StringBuilder();

    ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n");
    ddl.append("(" + "\n");

    for (int i = 0; i < tableDesc.getColumns().length; i++) {
        ColumnDesc col = tableDesc.getColumns()[i];
        if (col.isComputedColumn()) {
            continue;
        }
        if (i > 0) {
            ddl.append(",");
            csvColumns.append(",");
        }
        ddl.append(col.getName() + " " + getH2DataType((col.getDatatype())) + "\n");
        csvColumns.append(col.getName());
    }
    ddl.append(")" + "\n");
    ddl.append("AS SELECT * FROM CSVREAD('" + csvFilePath + "', '" + csvColumns
            + "', 'charset=UTF-8 fieldSeparator=,');");

    return ddl.toString();
}
 
Example #6
Source File: CoprocessorRowType.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    int bodyOffset = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        String datatype = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        col.setDatatype(datatype);
        col.init(table);
        cols[i] = col.getRef();

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes, bodyOffset);
}
 
Example #7
Source File: FlatTableSqlQuoteUtils.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
/**
 * Used to quote identifiers for JDBC ext job when quoting cc expr
 * @param tableDesc
 * @param sqlExpr
 * @return
 */
public static String quoteIdentifierInSqlExpr(TableDesc tableDesc, String sqlExpr, SqlDialect sqlDialect) {
    String table = tableDesc.getName();
    boolean tableMatched = false;
    List<String> tabPatterns = getTableNameOrAliasPatterns(table);
    if (isIdentifierNeedToQuote(sqlExpr, table, tabPatterns)) {
        sqlExpr = quoteIdentifier(sqlExpr, table, tabPatterns, sqlDialect);
        tableMatched = true;
    }

    if (tableMatched) {
        for (ColumnDesc columnDesc : tableDesc.getColumns()) {
            String column = columnDesc.getName();
            List<String> colPatterns = getColumnNameOrAliasPatterns(column);
            if (isIdentifierNeedToQuote(sqlExpr, column, colPatterns)) {
                sqlExpr = quoteIdentifier(sqlExpr, column, colPatterns, sqlDialect);
            }
        }
    }

    return sqlExpr;
}
 
Example #8
Source File: CoprocessorRowType.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        cols[i] = new TblColRef(col);

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes);
}
 
Example #9
Source File: FilterBaseTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up
protected List<TblColRef> buildGroups() {
    List<TblColRef> groups = new ArrayList<TblColRef>();

    TableDesc t1 = new TableDesc();
    t1.setName("TEST_KYLIN_FACT");
    t1.setDatabase("DEFAULT");
    ColumnDesc c1 = new ColumnDesc();
    c1.setName("CAL_DT");
    c1.setDatatype("String");
    c1.setTable(t1);
    TblColRef cf1 = new TblColRef(c1);
    groups.add(cf1);

    TableDesc t2 = new TableDesc();
    t2.setName("TEST_CATEGORY_GROUPINGS");
    t2.setDatabase("DEFAULT");
    ColumnDesc c2 = new ColumnDesc();
    c2.setName("META_CATEG_NAME");
    c1.setDatatype("String");
    c2.setTable(t2);
    TblColRef cf2 = new TblColRef(c2);
    groups.add(cf2);

    return groups;
}
 
Example #10
Source File: FilterSerializeTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void testSerialize07() {
    TableDesc table = new TableDesc();
    table.setName("TEST_KYLIN_FACT");
    table.setDatabase("DEFAULT");

    ColumnDesc column = new ColumnDesc();
    column.setTable(table);
    TblColRef colRef = new TblColRef(column);
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);
    TupleFilter filter = buildCompareFilter(groups, 0);

    byte[] bytes = TupleFilterSerializer.serialize(filter);
    TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes);

    compareFilter(filter, newFilter);
}
 
Example #11
Source File: H2Database.java    From kylin with Apache License 2.0 6 votes vote down vote up
private List<String> generateCreateH2IndexSql(TableDesc tableDesc) {
    List<String> result = Lists.newArrayList();
    int x = 0;
    for (ColumnDesc col : tableDesc.getColumns()) {
        if ("T".equalsIgnoreCase(col.getIndex())) {
            StringBuilder ddl = new StringBuilder();
            ddl.append("CREATE INDEX IDX_" + tableDesc.getName() + "_" + x + " ON " + tableDesc.getIdentity() + "("
                    + col.getName() + ")");
            ddl.append("\n");
            result.add(ddl.toString());
            x++;
        }
    }

    return result;
}
 
Example #12
Source File: StorageTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private List<TblColRef> buildGroups() {
    List<TblColRef> groups = new ArrayList<TblColRef>();

    TableDesc t1 = new TableDesc();
    t1.setName("TEST_KYLIN_FACT");
    t1.setDatabase("DEFAULT");
    ColumnDesc c1 = new ColumnDesc();
    c1.setName("CAL_DT");
    c1.setTable(t1);
    c1.setDatatype("string");
    TblColRef cf1 = new TblColRef(c1);
    groups.add(cf1);

    TableDesc t2 = new TableDesc();
    t2.setName("TEST_CATEGORY_GROUPINGS");
    t2.setDatabase("DEFAULT");
    ColumnDesc c2 = new ColumnDesc();
    c2.setName("META_CATEG_NAME");
    c2.setTable(t2);
    c2.setDatatype("string");
    TblColRef cf2 = new TblColRef(c2);
    groups.add(cf2);

    return groups;
}
 
Example #13
Source File: JdbcExplorer.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private String[] generateCreateTableSql(TableDesc tableDesc) {
    logger.info("Generate create table sql: {}", tableDesc);
    String tableIdentity = String
            .format(Locale.ROOT, "%s.%s", tableDesc.getDatabase().toUpperCase(Locale.ROOT), tableDesc.getName())
            .toUpperCase(Locale.ROOT);
    String dropsql = "DROP TABLE IF EXISTS " + tableIdentity;
    String dropsql2 = "DROP VIEW IF EXISTS " + tableIdentity;

    StringBuilder ddl = new StringBuilder();
    ddl.append("CREATE TABLE " + tableIdentity + "\n");
    ddl.append("(" + "\n");

    for (int i = 0; i < tableDesc.getColumns().length; i++) {
        ColumnDesc col = tableDesc.getColumns()[i];
        if (i > 0) {
            ddl.append(",");
        }
        ddl.append(col.getName() + " " + getSqlDataType((col.getDatatype())) + "\n");
    }

    ddl.append(")");

    return new String[] { dropsql, dropsql2, ddl.toString() };
}
 
Example #14
Source File: ModelDataGenerator.java    From kylin with Apache License 2.0 6 votes vote down vote up
private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) {
    for (TableDesc t : tables) {
        if (t.isView())
            continue;

        out.print("DROP TABLE IF EXISTS " + normHiveIdentifier(t.getIdentity()) + ";\n");

        out.print("CREATE TABLE " + normHiveIdentifier(t.getIdentity()) + "(" + "\n");

        for (int i = 0; i < t.getColumns().length; i++) {
            ColumnDesc col = t.getColumns()[i];
            out.print("    ");
            if (i > 0) {
                out.print(",");
            }
            out.print(normHiveIdentifier(col.getName()) + " " + hiveType(col.getType()) + "\n");
        }

        out.print(")" + "\n");
        out.print("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n");
        out.print("STORED AS TEXTFILE" + ";\n");
        out.print("\n");
    }
}
 
Example #15
Source File: LookupTableEnumerator.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
public boolean moveNext() {
    boolean hasNext = iterator.hasNext();
    if (hasNext) {
        String[] row = iterator.next();
        for (int i = 0, n = colDescs.size(); i < n; i++) {
            ColumnDesc colDesc = colDescs.get(i);
            int colIdx = colDesc.getZeroBasedIndex();
            if (colIdx >= 0) {
                current[i] = Tuple.convertOptiqCellValue(row[colIdx], colDesc.getUpgradedType().getName());
            } else {
                current[i] = null; // fake column
            }
        }
    }
    return hasNext;
}
 
Example #16
Source File: CoprocessorRowType.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
public CoprocessorRowType deserialize(ByteBuffer in) {
    int n = BytesUtil.readVInt(in);
    int bodyOffset = BytesUtil.readVInt(in);
    TblColRef[] cols = new TblColRef[n];
    int[] colSizes = new int[n];
    for (int i = 0; i < n; i++) {
        String tableName = BytesUtil.readAsciiString(in);
        String colName = BytesUtil.readAsciiString(in);
        String datatype = BytesUtil.readAsciiString(in);
        TableDesc table = new TableDesc();
        table.setName(tableName);
        ColumnDesc col = new ColumnDesc();
        col.setTable(table);
        col.setName(colName);
        col.setDatatype(datatype);
        col.init(table);
        cols[i] = col.getRef();

        int colSize = BytesUtil.readVInt(in);
        colSizes[i] = colSize;
    }
    return new CoprocessorRowType(cols, colSizes, bodyOffset);
}
 
Example #17
Source File: ColumnCardinalityMapper.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Override
public void doMap(T key, Object value, Context context) throws IOException, InterruptedException {
    ColumnDesc[] columns = tableDesc.getColumns();
    Collection<String[]> valuesCollection = tableInputFormat.parseMapperInput(value);

    for (String[] values: valuesCollection) {
        for (int m = 0; m < columns.length; m++) {
            String field = columns[m].getName();
            String fieldValue = values[m];
            if (fieldValue == null)
                fieldValue = "NULL";

            if (counter < 5 && m < 10) {
                System.out.println("Get row " + counter + " column '" + field + "'  value: " + fieldValue);
            }

            getHllc(m).add(Bytes.toBytes(fieldValue.toString()));
        }

        counter++;
    }
}
 
Example #18
Source File: HiveMetadataExplorer.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private ColumnDesc[] extractColumnFromMeta(HiveTableMeta hiveTableMeta) {
    int columnNumber = hiveTableMeta.allColumns.size();
    List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber);

    for (int i = 0; i < columnNumber; i++) {
        HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);

        // skip unsupported fields, e.g. map<string, int>
        if (DataType.isKylinSupported(field.dataType)) {
            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(field.name.toUpperCase(Locale.ROOT));

            // use "double" in kylin for "float"
            if ("float".equalsIgnoreCase(field.dataType)) {
                cdesc.setDatatype("double");
            } else {
                cdesc.setDatatype(field.dataType);
            }

            cdesc.setId(String.valueOf(i + 1));
            cdesc.setComment(field.comment);
            columns.add(cdesc);
        } else {
            logger.warn("Unsupported data type {}, excluding the field '{}'.", field.dataType, field.name);
        }
    }

    return  columns.toArray(new ColumnDesc[0]);
}
 
Example #19
Source File: JdbcExplorerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvalQueryMetadata() {
    ColumnDesc[] columnDescs = explorer
            .evalQueryMetadata("select cal_dt, count(*) as cnt from DEFAULT.test_kylin_fact group by cal_dt");
    Assert.assertNotNull(columnDescs);
    Assert.assertEquals(2, columnDescs.length);
    Assert.assertEquals("date", columnDescs[0].getDatatype());
    Assert.assertEquals("CAL_DT", columnDescs[0].getName());
    Assert.assertEquals("bigint", columnDescs[1].getDatatype());
    Assert.assertEquals("CNT", columnDescs[1].getName());
}
 
Example #20
Source File: RealizationChooser.java    From kylin with Apache License 2.0 5 votes vote down vote up
private static boolean containsAll(Set<ColumnDesc> allColumnDescs, Set<TblColRef> allColumns) {
    for (TblColRef col : allColumns) {
        if (allColumnDescs.contains(col.getColumnDesc()) == false)
            return false;
    }
    return true;
}
 
Example #21
Source File: HiveMetadataExplorer.java    From kylin with Apache License 2.0 5 votes vote down vote up
private ColumnDesc[] extractColumnFromMeta(HiveTableMeta hiveTableMeta) {
    int columnNumber = hiveTableMeta.allColumns.size();
    List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber);

    for (int i = 0; i < columnNumber; i++) {
        HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i);

        // skip unsupported fields, e.g. map<string, int>
        if (DataType.isKylinSupported(field.dataType)) {
            ColumnDesc cdesc = new ColumnDesc();
            cdesc.setName(field.name.toUpperCase(Locale.ROOT));

            // use "double" in kylin for "float"
            if ("float".equalsIgnoreCase(field.dataType)) {
                cdesc.setDatatype("double");
            } else {
                cdesc.setDatatype(field.dataType);
            }

            cdesc.setId(String.valueOf(i + 1));
            cdesc.setComment(field.comment);
            columns.add(cdesc);
        } else {
            logger.warn("Unsupported data type {}, excluding the field '{}'.", field.dataType, field.name);
        }
    }

    return  columns.toArray(new ColumnDesc[0]);
}
 
Example #22
Source File: JdbcExplorerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvalQueryMetadata() {
    ColumnDesc[] columnDescs = explorer
            .evalQueryMetadata("select cal_dt, count(*) as cnt from DEFAULT.test_kylin_fact group by cal_dt");
    Assert.assertNotNull(columnDescs);
    Assert.assertEquals(2, columnDescs.length);
    Assert.assertEquals("date", columnDescs[0].getDatatype());
    Assert.assertEquals("CAL_DT", columnDescs[0].getName());
    Assert.assertEquals("bigint", columnDescs[1].getDatatype());
    Assert.assertEquals("CNT", columnDescs[1].getName());
}
 
Example #23
Source File: CubeDesc.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private void initMeasureColumns(Map<String, TableDesc> tables) {
    if (measures == null || measures.isEmpty()) {
        return;
    }

    TableDesc factTable = tables.get(getFactTable());
    for (MeasureDesc m : measures) {
        m.setName(m.getName().toUpperCase());

        if (m.getDependentMeasureRef() != null) {
            m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase());
        }
        
        FunctionDesc f = m.getFunction();
        f.setExpression(f.getExpression().toUpperCase());
        f.setReturnDataType(DataType.getInstance(f.getReturnType()));

        ParameterDesc p = f.getParameter();
        p.normalizeColumnValue();

        if (p.isColumnType()) {
            ArrayList<TblColRef> colRefs = Lists.newArrayList();
            for (String cName : p.getValue().split("\\s*,\\s*")) {
                ColumnDesc sourceColumn = factTable.findColumnByName(cName);
                TblColRef colRef = new TblColRef(sourceColumn);
                colRefs.add(colRef);
                allColumns.add(colRef);
            }
            if (colRefs.isEmpty() == false)
                p.setColRefs(colRefs);
        }
        
        // verify holistic count distinct as a dependent measure
        if (m.isHolisticCountDistinct() && StringUtils.isBlank(m.getDependentMeasureRef())) {
            throw new IllegalStateException(m + " is a holistic count distinct but it has no DependentMeasureRef defined!");
        }
    }
}
 
Example #24
Source File: FilterSerializeTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialize07() {
    TableDesc table = new TableDesc();
    table.setName("TEST_KYLIN_FACT");
    table.setDatabase("DEFAULT");

    ColumnDesc column = new ColumnDesc();
    column.setTable(table);
    TblColRef colRef = column.getRef();
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);

    assertFilterSerDe(buildEQCompareFilter(groups, 0));
}
 
Example #25
Source File: FilterSerializeTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialize05() {
    ColumnDesc column = new ColumnDesc();
    TblColRef colRef = column.getRef();
    List<TblColRef> groups = new ArrayList<TblColRef>();
    groups.add(colRef);

    assertFilterSerDe(buildEQCompareFilter(groups, 0));
}
 
Example #26
Source File: JdbcExplorerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testLoadTableMetadata() throws SQLException {
    String tableName = "tb1";
    String databaseName = "testdb";
    ResultSet rs1 = mock(ResultSet.class);
    when(rs1.next()).thenReturn(true).thenReturn(false);
    when(rs1.getString("TABLE_TYPE")).thenReturn("TABLE");

    ResultSet rs2 = mock(ResultSet.class);
    when(rs2.next()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false);
    when(rs2.getString("COLUMN_NAME")).thenReturn("COL1").thenReturn("COL2").thenReturn("COL3");
    when(rs2.getInt("DATA_TYPE")).thenReturn(Types.VARCHAR).thenReturn(Types.INTEGER).thenReturn(Types.DECIMAL);
    when(rs2.getInt("COLUMN_SIZE")).thenReturn(128).thenReturn(10).thenReturn(19);
    when(rs2.getInt("DECIMAL_DIGITS")).thenReturn(0).thenReturn(0).thenReturn(4);
    when(rs2.getInt("ORDINAL_POSITION")).thenReturn(1).thenReturn(3).thenReturn(2);
    when(rs2.getString("REMARKS")).thenReturn("comment1").thenReturn("comment2").thenReturn("comment3");

    when(jdbcMetadata.getTable(dbmd, databaseName, tableName)).thenReturn(rs1);
    when(jdbcMetadata.listColumns(dbmd, databaseName, tableName)).thenReturn(rs2);

    Pair<TableDesc, TableExtDesc> result = jdbcExplorer.loadTableMetadata(databaseName, tableName, "proj");
    TableDesc tableDesc = result.getFirst();
    ColumnDesc columnDesc = tableDesc.getColumns()[1];

    Assert.assertEquals(databaseName.toUpperCase(Locale.ROOT), tableDesc.getDatabase());
    Assert.assertEquals(3, tableDesc.getColumnCount());
    Assert.assertEquals("TABLE", tableDesc.getTableType());
    Assert.assertEquals("COL2", columnDesc.getName());
    Assert.assertEquals("integer", columnDesc.getTypeName());
    Assert.assertEquals("comment2", columnDesc.getComment());
    Assert.assertEquals(databaseName.toUpperCase(Locale.ROOT) + "." + tableName.toUpperCase(Locale.ROOT),
            result.getSecond().getIdentity());
}
 
Example #27
Source File: CubeDesc.java    From Kylin with Apache License 2.0 5 votes vote down vote up
private TblColRef initDimensionColRef(DimensionDesc dim, String colName) {
    TableDesc table = dim.getTableDesc();
    ColumnDesc col = table.findColumnByName(colName);
    if (col == null)
        throw new IllegalArgumentException("No column '" + colName + "' found in table " + table);
    
    TblColRef ref = new TblColRef(col);
    return initDimensionColRef(ref);
}
 
Example #28
Source File: ColumnGenConfig.java    From kylin with Apache License 2.0 5 votes vote down vote up
private void init(ColumnDesc col, ModelDataGenerator modelGen) throws IOException {
    
    Map<String, String> config = Util.parseEqualCommaPairs(col.getDataGen(), "values");

    values = Arrays.asList(Util.parseString(config, "values", "").split("[|]"));
    
    List<String> pkValues = modelGen.getPkValuesIfIsFk(col);
    
    if (FK.equals(values.get(0)) || (values.get(0).isEmpty() && pkValues != null)) {
        isFK = true;
        values = getPkValues(modelGen, config, pkValues);
    } else if (ID.equals(values.get(0))) {
        isID = true;
        idStart = (values.size() > 1) ? Integer.parseInt(values.get(1)) : 0;
    } else if (RAND.equals(values.get(0)) || values.get(0).isEmpty()) {
        isRandom = true;
        randFormat = (values.size() > 1) ? values.get(1) : "";
        randStart = (values.size() > 2) ? Integer.parseInt(values.get(2)) : 0;
        randEnd = (values.size() > 3) ? Integer.parseInt(values.get(3)) : 0;
    } else {
        isDiscrete = true;
    }
    
    cardinality = Util.parseInt(config, "card", guessCardinality(col.getName()));
    genNull = Util.parseBoolean(config, "null", guessGenNull(col.getName()));
    genNullPct = Util.parseDouble(config, "nullpct", 0.01);
    genNullStr = Util.parseString(config, "nullstr", "\\N"); // '\N' is null in hive
    order = Util.parseBoolean(config, "order", false);
    unique = Util.parseBoolean(config, "uniq", modelGen.isPK(col));
}
 
Example #29
Source File: HybridInstance.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private Set<ColumnDesc> asColumnDescs(Set<TblColRef> columns) {
    LinkedHashSet<ColumnDesc> result = new LinkedHashSet<>();
    for (TblColRef col : columns) {
        result.add(col.getColumnDesc());
    }
    return result;
}
 
Example #30
Source File: SnapshotManagerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private TableDesc genTableDesc(String tableName) {
    TableDesc table = TableDesc.mockup(tableName);
    ColumnDesc desc1 = new ColumnDesc("1", "id", "string", null, null, null, null);
    desc1.setId("1");
    desc1.setDatatype("long");
    ColumnDesc desc2 = new ColumnDesc("2", "country", "string", null, null, null, null);
    desc2.setId("2");
    desc2.setDatatype("string");
    ColumnDesc[] columns = { desc1, desc2 };
    table.setColumns(columns);
    table.init(kylinConfig, "default");
    return table;
}