org.apache.kylin.metadata.model.ColumnDesc Java Examples
The following examples show how to use
org.apache.kylin.metadata.model.ColumnDesc.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LookupTableEnumerator.java From Kylin with Apache License 2.0 | 6 votes |
@Override public boolean moveNext() { boolean hasNext = iterator.hasNext(); if (hasNext) { String[] row = iterator.next(); for (int i = 0, n = colDescs.size(); i < n; i++) { ColumnDesc colDesc = colDescs.get(i); int colIdx = colDesc.getZeroBasedIndex(); if (colIdx >= 0) { current[i] = Tuple.convertOptiqCellValue(row[colIdx], colDesc.getType().getName()); } else { current[i] = null; // fake column } } } return hasNext; }
Example #2
Source File: ModelDataGenerator.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) { for (TableDesc t : tables) { if (t.isView()) continue; out.print("DROP TABLE IF EXISTS " + normHiveIdentifier(t.getIdentity()) + ";\n"); out.print("CREATE TABLE " + normHiveIdentifier(t.getIdentity()) + "(" + "\n"); for (int i = 0; i < t.getColumns().length; i++) { ColumnDesc col = t.getColumns()[i]; out.print(" "); if (i > 0) { out.print(","); } out.print(normHiveIdentifier(col.getName()) + " " + hiveType(col.getType()) + "\n"); } out.print(")" + "\n"); out.print("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n"); out.print("STORED AS TEXTFILE" + ";\n"); out.print("\n"); } }
Example #3
Source File: TableSchemaUpdateChecker.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private boolean isColumnCompatible(ColumnDesc column, ColumnDesc newCol) { if (!column.getName().equalsIgnoreCase(newCol.getName())) { return false; } if (column.getType().isIntegerFamily()) { // OLAPTable.listSourceColumns converts some integer columns to bigint, // therefore strict type comparison won't work. // changing from one integer type to another should be fine. return newCol.getType().isIntegerFamily(); } else if (column.getType().isNumberFamily()) { // Both are float/double should be fine. return newCol.getType().isNumberFamily(); } else { // only compare base type name, changing precision or scale should be fine return column.getTypeName().equals(newCol.getTypeName()); } }
Example #4
Source File: TableSchemaUpdateChecker.java From kylin with Apache License 2.0 | 6 votes |
private Set<ColumnDesc> findUsedColumnsInFactTable(DataModelDesc usedModel, TableDesc factTable) { Set<ColumnDesc> usedColumns = Sets.newHashSet(); // column in dimension for (ModelDimensionDesc dim : usedModel.getDimensions()) { if (dim.getTable().equalsIgnoreCase(factTable.getName())) { for (String col : dim.getColumns()) { usedColumns.add(mustGetColumnDesc(factTable, col)); } } } // column in measure for (String columnInMeasure : usedModel.getMetrics()) { if (factTable.getName().equalsIgnoreCase(getTableName(columnInMeasure))) { usedColumns.add(mustGetColumnDesc(factTable, columnInMeasure)); } } return usedColumns; }
Example #5
Source File: H2Database.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private String generateCreateH2TableSql(TableDesc tableDesc, String csvFilePath) { StringBuilder ddl = new StringBuilder(); StringBuilder csvColumns = new StringBuilder(); ddl.append("CREATE TABLE " + tableDesc.getIdentity() + "\n"); ddl.append("(" + "\n"); for (int i = 0; i < tableDesc.getColumns().length; i++) { ColumnDesc col = tableDesc.getColumns()[i]; if (col.isComputedColumn()) { continue; } if (i > 0) { ddl.append(","); csvColumns.append(","); } ddl.append(col.getName() + " " + getH2DataType((col.getDatatype())) + "\n"); csvColumns.append(col.getName()); } ddl.append(")" + "\n"); ddl.append("AS SELECT * FROM CSVREAD('" + csvFilePath + "', '" + csvColumns + "', 'charset=UTF-8 fieldSeparator=,');"); return ddl.toString(); }
Example #6
Source File: CoprocessorRowType.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
@Override public CoprocessorRowType deserialize(ByteBuffer in) { int n = BytesUtil.readVInt(in); int bodyOffset = BytesUtil.readVInt(in); TblColRef[] cols = new TblColRef[n]; int[] colSizes = new int[n]; for (int i = 0; i < n; i++) { String tableName = BytesUtil.readAsciiString(in); String colName = BytesUtil.readAsciiString(in); String datatype = BytesUtil.readAsciiString(in); TableDesc table = new TableDesc(); table.setName(tableName); ColumnDesc col = new ColumnDesc(); col.setTable(table); col.setName(colName); col.setDatatype(datatype); col.init(table); cols[i] = col.getRef(); int colSize = BytesUtil.readVInt(in); colSizes[i] = colSize; } return new CoprocessorRowType(cols, colSizes, bodyOffset); }
Example #7
Source File: FlatTableSqlQuoteUtils.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
/** * Used to quote identifiers for JDBC ext job when quoting cc expr * @param tableDesc * @param sqlExpr * @return */ public static String quoteIdentifierInSqlExpr(TableDesc tableDesc, String sqlExpr, SqlDialect sqlDialect) { String table = tableDesc.getName(); boolean tableMatched = false; List<String> tabPatterns = getTableNameOrAliasPatterns(table); if (isIdentifierNeedToQuote(sqlExpr, table, tabPatterns)) { sqlExpr = quoteIdentifier(sqlExpr, table, tabPatterns, sqlDialect); tableMatched = true; } if (tableMatched) { for (ColumnDesc columnDesc : tableDesc.getColumns()) { String column = columnDesc.getName(); List<String> colPatterns = getColumnNameOrAliasPatterns(column); if (isIdentifierNeedToQuote(sqlExpr, column, colPatterns)) { sqlExpr = quoteIdentifier(sqlExpr, column, colPatterns, sqlDialect); } } } return sqlExpr; }
Example #8
Source File: CoprocessorRowType.java From Kylin with Apache License 2.0 | 6 votes |
@Override public CoprocessorRowType deserialize(ByteBuffer in) { int n = BytesUtil.readVInt(in); TblColRef[] cols = new TblColRef[n]; int[] colSizes = new int[n]; for (int i = 0; i < n; i++) { String tableName = BytesUtil.readAsciiString(in); String colName = BytesUtil.readAsciiString(in); TableDesc table = new TableDesc(); table.setName(tableName); ColumnDesc col = new ColumnDesc(); col.setTable(table); col.setName(colName); cols[i] = new TblColRef(col); int colSize = BytesUtil.readVInt(in); colSizes[i] = colSize; } return new CoprocessorRowType(cols, colSizes); }
Example #9
Source File: FilterBaseTest.java From Kylin with Apache License 2.0 | 6 votes |
protected List<TblColRef> buildGroups() { List<TblColRef> groups = new ArrayList<TblColRef>(); TableDesc t1 = new TableDesc(); t1.setName("TEST_KYLIN_FACT"); t1.setDatabase("DEFAULT"); ColumnDesc c1 = new ColumnDesc(); c1.setName("CAL_DT"); c1.setDatatype("String"); c1.setTable(t1); TblColRef cf1 = new TblColRef(c1); groups.add(cf1); TableDesc t2 = new TableDesc(); t2.setName("TEST_CATEGORY_GROUPINGS"); t2.setDatabase("DEFAULT"); ColumnDesc c2 = new ColumnDesc(); c2.setName("META_CATEG_NAME"); c1.setDatatype("String"); c2.setTable(t2); TblColRef cf2 = new TblColRef(c2); groups.add(cf2); return groups; }
Example #10
Source File: FilterSerializeTest.java From Kylin with Apache License 2.0 | 6 votes |
@Test public void testSerialize07() { TableDesc table = new TableDesc(); table.setName("TEST_KYLIN_FACT"); table.setDatabase("DEFAULT"); ColumnDesc column = new ColumnDesc(); column.setTable(table); TblColRef colRef = new TblColRef(column); List<TblColRef> groups = new ArrayList<TblColRef>(); groups.add(colRef); TupleFilter filter = buildCompareFilter(groups, 0); byte[] bytes = TupleFilterSerializer.serialize(filter); TupleFilter newFilter = TupleFilterSerializer.deserialize(bytes); compareFilter(filter, newFilter); }
Example #11
Source File: H2Database.java From kylin with Apache License 2.0 | 6 votes |
private List<String> generateCreateH2IndexSql(TableDesc tableDesc) { List<String> result = Lists.newArrayList(); int x = 0; for (ColumnDesc col : tableDesc.getColumns()) { if ("T".equalsIgnoreCase(col.getIndex())) { StringBuilder ddl = new StringBuilder(); ddl.append("CREATE INDEX IDX_" + tableDesc.getName() + "_" + x + " ON " + tableDesc.getIdentity() + "(" + col.getName() + ")"); ddl.append("\n"); result.add(ddl.toString()); x++; } } return result; }
Example #12
Source File: StorageTest.java From Kylin with Apache License 2.0 | 6 votes |
private List<TblColRef> buildGroups() { List<TblColRef> groups = new ArrayList<TblColRef>(); TableDesc t1 = new TableDesc(); t1.setName("TEST_KYLIN_FACT"); t1.setDatabase("DEFAULT"); ColumnDesc c1 = new ColumnDesc(); c1.setName("CAL_DT"); c1.setTable(t1); c1.setDatatype("string"); TblColRef cf1 = new TblColRef(c1); groups.add(cf1); TableDesc t2 = new TableDesc(); t2.setName("TEST_CATEGORY_GROUPINGS"); t2.setDatabase("DEFAULT"); ColumnDesc c2 = new ColumnDesc(); c2.setName("META_CATEG_NAME"); c2.setTable(t2); c2.setDatatype("string"); TblColRef cf2 = new TblColRef(c2); groups.add(cf2); return groups; }
Example #13
Source File: JdbcExplorer.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private String[] generateCreateTableSql(TableDesc tableDesc) { logger.info("Generate create table sql: {}", tableDesc); String tableIdentity = String .format(Locale.ROOT, "%s.%s", tableDesc.getDatabase().toUpperCase(Locale.ROOT), tableDesc.getName()) .toUpperCase(Locale.ROOT); String dropsql = "DROP TABLE IF EXISTS " + tableIdentity; String dropsql2 = "DROP VIEW IF EXISTS " + tableIdentity; StringBuilder ddl = new StringBuilder(); ddl.append("CREATE TABLE " + tableIdentity + "\n"); ddl.append("(" + "\n"); for (int i = 0; i < tableDesc.getColumns().length; i++) { ColumnDesc col = tableDesc.getColumns()[i]; if (i > 0) { ddl.append(","); } ddl.append(col.getName() + " " + getSqlDataType((col.getDatatype())) + "\n"); } ddl.append(")"); return new String[] { dropsql, dropsql2, ddl.toString() }; }
Example #14
Source File: ModelDataGenerator.java From kylin with Apache License 2.0 | 6 votes |
private void generateCreateTableDDL(Set<TableDesc> tables, PrintWriter out) { for (TableDesc t : tables) { if (t.isView()) continue; out.print("DROP TABLE IF EXISTS " + normHiveIdentifier(t.getIdentity()) + ";\n"); out.print("CREATE TABLE " + normHiveIdentifier(t.getIdentity()) + "(" + "\n"); for (int i = 0; i < t.getColumns().length; i++) { ColumnDesc col = t.getColumns()[i]; out.print(" "); if (i > 0) { out.print(","); } out.print(normHiveIdentifier(col.getName()) + " " + hiveType(col.getType()) + "\n"); } out.print(")" + "\n"); out.print("ROW FORMAT DELIMITED FIELDS TERMINATED BY ','" + "\n"); out.print("STORED AS TEXTFILE" + ";\n"); out.print("\n"); } }
Example #15
Source File: LookupTableEnumerator.java From kylin with Apache License 2.0 | 6 votes |
@Override public boolean moveNext() { boolean hasNext = iterator.hasNext(); if (hasNext) { String[] row = iterator.next(); for (int i = 0, n = colDescs.size(); i < n; i++) { ColumnDesc colDesc = colDescs.get(i); int colIdx = colDesc.getZeroBasedIndex(); if (colIdx >= 0) { current[i] = Tuple.convertOptiqCellValue(row[colIdx], colDesc.getUpgradedType().getName()); } else { current[i] = null; // fake column } } } return hasNext; }
Example #16
Source File: CoprocessorRowType.java From kylin with Apache License 2.0 | 6 votes |
@Override public CoprocessorRowType deserialize(ByteBuffer in) { int n = BytesUtil.readVInt(in); int bodyOffset = BytesUtil.readVInt(in); TblColRef[] cols = new TblColRef[n]; int[] colSizes = new int[n]; for (int i = 0; i < n; i++) { String tableName = BytesUtil.readAsciiString(in); String colName = BytesUtil.readAsciiString(in); String datatype = BytesUtil.readAsciiString(in); TableDesc table = new TableDesc(); table.setName(tableName); ColumnDesc col = new ColumnDesc(); col.setTable(table); col.setName(colName); col.setDatatype(datatype); col.init(table); cols[i] = col.getRef(); int colSize = BytesUtil.readVInt(in); colSizes[i] = colSize; } return new CoprocessorRowType(cols, colSizes, bodyOffset); }
Example #17
Source File: ColumnCardinalityMapper.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
@Override public void doMap(T key, Object value, Context context) throws IOException, InterruptedException { ColumnDesc[] columns = tableDesc.getColumns(); Collection<String[]> valuesCollection = tableInputFormat.parseMapperInput(value); for (String[] values: valuesCollection) { for (int m = 0; m < columns.length; m++) { String field = columns[m].getName(); String fieldValue = values[m]; if (fieldValue == null) fieldValue = "NULL"; if (counter < 5 && m < 10) { System.out.println("Get row " + counter + " column '" + field + "' value: " + fieldValue); } getHllc(m).add(Bytes.toBytes(fieldValue.toString())); } counter++; } }
Example #18
Source File: HiveMetadataExplorer.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
private ColumnDesc[] extractColumnFromMeta(HiveTableMeta hiveTableMeta) { int columnNumber = hiveTableMeta.allColumns.size(); List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber); for (int i = 0; i < columnNumber; i++) { HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i); // skip unsupported fields, e.g. map<string, int> if (DataType.isKylinSupported(field.dataType)) { ColumnDesc cdesc = new ColumnDesc(); cdesc.setName(field.name.toUpperCase(Locale.ROOT)); // use "double" in kylin for "float" if ("float".equalsIgnoreCase(field.dataType)) { cdesc.setDatatype("double"); } else { cdesc.setDatatype(field.dataType); } cdesc.setId(String.valueOf(i + 1)); cdesc.setComment(field.comment); columns.add(cdesc); } else { logger.warn("Unsupported data type {}, excluding the field '{}'.", field.dataType, field.name); } } return columns.toArray(new ColumnDesc[0]); }
Example #19
Source File: JdbcExplorerTest.java From kylin with Apache License 2.0 | 5 votes |
@Test public void testEvalQueryMetadata() { ColumnDesc[] columnDescs = explorer .evalQueryMetadata("select cal_dt, count(*) as cnt from DEFAULT.test_kylin_fact group by cal_dt"); Assert.assertNotNull(columnDescs); Assert.assertEquals(2, columnDescs.length); Assert.assertEquals("date", columnDescs[0].getDatatype()); Assert.assertEquals("CAL_DT", columnDescs[0].getName()); Assert.assertEquals("bigint", columnDescs[1].getDatatype()); Assert.assertEquals("CNT", columnDescs[1].getName()); }
Example #20
Source File: RealizationChooser.java From kylin with Apache License 2.0 | 5 votes |
private static boolean containsAll(Set<ColumnDesc> allColumnDescs, Set<TblColRef> allColumns) { for (TblColRef col : allColumns) { if (allColumnDescs.contains(col.getColumnDesc()) == false) return false; } return true; }
Example #21
Source File: HiveMetadataExplorer.java From kylin with Apache License 2.0 | 5 votes |
private ColumnDesc[] extractColumnFromMeta(HiveTableMeta hiveTableMeta) { int columnNumber = hiveTableMeta.allColumns.size(); List<ColumnDesc> columns = new ArrayList<ColumnDesc>(columnNumber); for (int i = 0; i < columnNumber; i++) { HiveTableMeta.HiveTableColumnMeta field = hiveTableMeta.allColumns.get(i); // skip unsupported fields, e.g. map<string, int> if (DataType.isKylinSupported(field.dataType)) { ColumnDesc cdesc = new ColumnDesc(); cdesc.setName(field.name.toUpperCase(Locale.ROOT)); // use "double" in kylin for "float" if ("float".equalsIgnoreCase(field.dataType)) { cdesc.setDatatype("double"); } else { cdesc.setDatatype(field.dataType); } cdesc.setId(String.valueOf(i + 1)); cdesc.setComment(field.comment); columns.add(cdesc); } else { logger.warn("Unsupported data type {}, excluding the field '{}'.", field.dataType, field.name); } } return columns.toArray(new ColumnDesc[0]); }
Example #22
Source File: JdbcExplorerTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Test public void testEvalQueryMetadata() { ColumnDesc[] columnDescs = explorer .evalQueryMetadata("select cal_dt, count(*) as cnt from DEFAULT.test_kylin_fact group by cal_dt"); Assert.assertNotNull(columnDescs); Assert.assertEquals(2, columnDescs.length); Assert.assertEquals("date", columnDescs[0].getDatatype()); Assert.assertEquals("CAL_DT", columnDescs[0].getName()); Assert.assertEquals("bigint", columnDescs[1].getDatatype()); Assert.assertEquals("CNT", columnDescs[1].getName()); }
Example #23
Source File: CubeDesc.java From Kylin with Apache License 2.0 | 5 votes |
private void initMeasureColumns(Map<String, TableDesc> tables) { if (measures == null || measures.isEmpty()) { return; } TableDesc factTable = tables.get(getFactTable()); for (MeasureDesc m : measures) { m.setName(m.getName().toUpperCase()); if (m.getDependentMeasureRef() != null) { m.setDependentMeasureRef(m.getDependentMeasureRef().toUpperCase()); } FunctionDesc f = m.getFunction(); f.setExpression(f.getExpression().toUpperCase()); f.setReturnDataType(DataType.getInstance(f.getReturnType())); ParameterDesc p = f.getParameter(); p.normalizeColumnValue(); if (p.isColumnType()) { ArrayList<TblColRef> colRefs = Lists.newArrayList(); for (String cName : p.getValue().split("\\s*,\\s*")) { ColumnDesc sourceColumn = factTable.findColumnByName(cName); TblColRef colRef = new TblColRef(sourceColumn); colRefs.add(colRef); allColumns.add(colRef); } if (colRefs.isEmpty() == false) p.setColRefs(colRefs); } // verify holistic count distinct as a dependent measure if (m.isHolisticCountDistinct() && StringUtils.isBlank(m.getDependentMeasureRef())) { throw new IllegalStateException(m + " is a holistic count distinct but it has no DependentMeasureRef defined!"); } } }
Example #24
Source File: FilterSerializeTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Test public void testSerialize07() { TableDesc table = new TableDesc(); table.setName("TEST_KYLIN_FACT"); table.setDatabase("DEFAULT"); ColumnDesc column = new ColumnDesc(); column.setTable(table); TblColRef colRef = column.getRef(); List<TblColRef> groups = new ArrayList<TblColRef>(); groups.add(colRef); assertFilterSerDe(buildEQCompareFilter(groups, 0)); }
Example #25
Source File: FilterSerializeTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Test public void testSerialize05() { ColumnDesc column = new ColumnDesc(); TblColRef colRef = column.getRef(); List<TblColRef> groups = new ArrayList<TblColRef>(); groups.add(colRef); assertFilterSerDe(buildEQCompareFilter(groups, 0)); }
Example #26
Source File: JdbcExplorerTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Test public void testLoadTableMetadata() throws SQLException { String tableName = "tb1"; String databaseName = "testdb"; ResultSet rs1 = mock(ResultSet.class); when(rs1.next()).thenReturn(true).thenReturn(false); when(rs1.getString("TABLE_TYPE")).thenReturn("TABLE"); ResultSet rs2 = mock(ResultSet.class); when(rs2.next()).thenReturn(true).thenReturn(true).thenReturn(true).thenReturn(false); when(rs2.getString("COLUMN_NAME")).thenReturn("COL1").thenReturn("COL2").thenReturn("COL3"); when(rs2.getInt("DATA_TYPE")).thenReturn(Types.VARCHAR).thenReturn(Types.INTEGER).thenReturn(Types.DECIMAL); when(rs2.getInt("COLUMN_SIZE")).thenReturn(128).thenReturn(10).thenReturn(19); when(rs2.getInt("DECIMAL_DIGITS")).thenReturn(0).thenReturn(0).thenReturn(4); when(rs2.getInt("ORDINAL_POSITION")).thenReturn(1).thenReturn(3).thenReturn(2); when(rs2.getString("REMARKS")).thenReturn("comment1").thenReturn("comment2").thenReturn("comment3"); when(jdbcMetadata.getTable(dbmd, databaseName, tableName)).thenReturn(rs1); when(jdbcMetadata.listColumns(dbmd, databaseName, tableName)).thenReturn(rs2); Pair<TableDesc, TableExtDesc> result = jdbcExplorer.loadTableMetadata(databaseName, tableName, "proj"); TableDesc tableDesc = result.getFirst(); ColumnDesc columnDesc = tableDesc.getColumns()[1]; Assert.assertEquals(databaseName.toUpperCase(Locale.ROOT), tableDesc.getDatabase()); Assert.assertEquals(3, tableDesc.getColumnCount()); Assert.assertEquals("TABLE", tableDesc.getTableType()); Assert.assertEquals("COL2", columnDesc.getName()); Assert.assertEquals("integer", columnDesc.getTypeName()); Assert.assertEquals("comment2", columnDesc.getComment()); Assert.assertEquals(databaseName.toUpperCase(Locale.ROOT) + "." + tableName.toUpperCase(Locale.ROOT), result.getSecond().getIdentity()); }
Example #27
Source File: CubeDesc.java From Kylin with Apache License 2.0 | 5 votes |
private TblColRef initDimensionColRef(DimensionDesc dim, String colName) { TableDesc table = dim.getTableDesc(); ColumnDesc col = table.findColumnByName(colName); if (col == null) throw new IllegalArgumentException("No column '" + colName + "' found in table " + table); TblColRef ref = new TblColRef(col); return initDimensionColRef(ref); }
Example #28
Source File: ColumnGenConfig.java From kylin with Apache License 2.0 | 5 votes |
private void init(ColumnDesc col, ModelDataGenerator modelGen) throws IOException { Map<String, String> config = Util.parseEqualCommaPairs(col.getDataGen(), "values"); values = Arrays.asList(Util.parseString(config, "values", "").split("[|]")); List<String> pkValues = modelGen.getPkValuesIfIsFk(col); if (FK.equals(values.get(0)) || (values.get(0).isEmpty() && pkValues != null)) { isFK = true; values = getPkValues(modelGen, config, pkValues); } else if (ID.equals(values.get(0))) { isID = true; idStart = (values.size() > 1) ? Integer.parseInt(values.get(1)) : 0; } else if (RAND.equals(values.get(0)) || values.get(0).isEmpty()) { isRandom = true; randFormat = (values.size() > 1) ? values.get(1) : ""; randStart = (values.size() > 2) ? Integer.parseInt(values.get(2)) : 0; randEnd = (values.size() > 3) ? Integer.parseInt(values.get(3)) : 0; } else { isDiscrete = true; } cardinality = Util.parseInt(config, "card", guessCardinality(col.getName())); genNull = Util.parseBoolean(config, "null", guessGenNull(col.getName())); genNullPct = Util.parseDouble(config, "nullpct", 0.01); genNullStr = Util.parseString(config, "nullstr", "\\N"); // '\N' is null in hive order = Util.parseBoolean(config, "order", false); unique = Util.parseBoolean(config, "uniq", modelGen.isPK(col)); }
Example #29
Source File: HybridInstance.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
private Set<ColumnDesc> asColumnDescs(Set<TblColRef> columns) { LinkedHashSet<ColumnDesc> result = new LinkedHashSet<>(); for (TblColRef col : columns) { result.add(col.getColumnDesc()); } return result; }
Example #30
Source File: SnapshotManagerTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
private TableDesc genTableDesc(String tableName) { TableDesc table = TableDesc.mockup(tableName); ColumnDesc desc1 = new ColumnDesc("1", "id", "string", null, null, null, null); desc1.setId("1"); desc1.setDatatype("long"); ColumnDesc desc2 = new ColumnDesc("2", "country", "string", null, null, null, null); desc2.setId("2"); desc2.setDatatype("string"); ColumnDesc[] columns = { desc1, desc2 }; table.setColumns(columns); table.init(kylinConfig, "default"); return table; }