Java Code Examples for org.apache.kylin.metadata.model.DataModelDesc#getJoinTables()

The following examples show how to use org.apache.kylin.metadata.model.DataModelDesc#getJoinTables() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CubeInstance.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
@Override
public int getCost() {
    if (getDescriptor() == null) {
        //in case not initialized
        return 0;
    }
    int countedDimensionNum = getRowKeyColumnCount();
    int c = countedDimensionNum * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;
    DataModelDesc model = getModel();
    if (model == null) {
        //in case broken cube
        return 0;
    }
    for (JoinTableDesc join : model.getJoinTables()) {
        if (join.getJoin().isInnerJoin())
            c += CubeInstance.COST_WEIGHT_INNER_JOIN;
    }
    return c;
}
Example 2
Source File: CubeInstance.java    From kylin with Apache License 2.0 6 votes vote down vote up 
@Override
public int getCost() {
    if (getDescriptor() == null) {
        //in case not initialized
        return 0;
    }
    int countedDimensionNum = getRowKeyColumnCount();
    int c = countedDimensionNum * COST_WEIGHT_DIMENSION + getMeasures().size() * COST_WEIGHT_MEASURE;
    DataModelDesc model = getModel();
    if (model == null) {
        //in case broken cube
        return 0;
    }
    for (JoinTableDesc join : model.getJoinTables()) {
        if (join.getJoin().isInnerJoin())
            c += CubeInstance.COST_WEIGHT_INNER_JOIN;
    }
    return c;
}
Example 3
Source File: JoinedFlatTable.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine, SqlDialect sqlDialect) {
    final String sep = singleLine ? " " : "\n";
    Set<TableRef> dimTableCache = new HashSet<>();

    DataModelDesc model = flatDesc.getDataModel();
    TableRef rootTable = model.getRootFactTable();
    sql.append(" FROM ").append(quoteTableIdentity(flatDesc.getDataModel().getRootFactTable(), sqlDialect))
            .append(" as ").append(quoteIdentifier(rootTable.getAlias(), sqlDialect)).append(sep);

    for (JoinTableDesc lookupDesc : model.getJoinTables()) {
        JoinDesc join = lookupDesc.getJoin();
        if (join != null && join.getType().equals("") == false) {
            TableRef dimTable = lookupDesc.getTableRef();
            if (!dimTableCache.contains(dimTable)) {
                TblColRef[] pk = join.getPrimaryKeyColumns();
                TblColRef[] fk = join.getForeignKeyColumns();
                if (pk.length != fk.length) {
                    throw new RuntimeException("Invalid join condition of lookup table:" + lookupDesc);
                }
                String joinType = join.getType().toUpperCase(Locale.ROOT);
                sql.append(joinType).append(" JOIN ").append(quoteTableIdentity(dimTable, sqlDialect))
                        .append(" as ").append(quoteIdentifier(dimTable.getAlias(), sqlDialect)).append(sep);
                sql.append("ON ");
                for (int i = 0; i < pk.length; i++) {
                    if (i > 0) {
                        sql.append(" AND ");
                    }
                    sql.append(getQuotedColExpressionInSourceDB(flatDesc, fk[i], sqlDialect)).append(" = ")
                            .append(getQuotedColExpressionInSourceDB(flatDesc, pk[i], sqlDialect));
                }
                sql.append(sep);

                dimTableCache.add(dimTable);
            }
        }
    }
}
Example 4
Source File: DimensionDesc.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up 
public void init(CubeDesc cubeDesc) {
    DataModelDesc model = cubeDesc.getModel();

    if (name != null)
        name = name.toUpperCase(Locale.ROOT);

    tableRef = model.findTable(table);
    table = tableRef.getAlias();

    join = null;
    for (JoinTableDesc joinTable : model.getJoinTables()) {
        if (joinTable.getTableRef().equals(this.tableRef)) {
            join = joinTable.getJoin();
            break;
        }
    }

    if (column != null && !"{FK}".equals(column)) {
        column = model.findColumn(table, column).getName();
    }
    if (derived != null && derived.length == 0) {
        derived = null;
    }
    if (derived != null) {
        for (int i = 0; i < derived.length; i++) {
            derived[i] = model.findColumn(table, derived[i]).getName();
        }
    }
    if (derived != null && join == null) {
        throw new IllegalStateException(
                "Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this);
    }

}
Example 5
Source File: JoinedFlatTable.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
        SqlDialect sqlDialect) {
    final String sep = singleLine ? " " : "\n";
    Set<TableRef> dimTableCache = new HashSet<>();

    DataModelDesc model = flatDesc.getDataModel();
    TableRef rootTable = model.getRootFactTable();
    sql.append(" FROM ").append(quoteTableIdentity(flatDesc.getDataModel().getRootFactTable(), sqlDialect))
            .append(" as ").append(quoteIdentifier(rootTable.getAlias(), sqlDialect)).append(sep);

    for (JoinTableDesc lookupDesc : model.getJoinTables()) {
        JoinDesc join = lookupDesc.getJoin();
        if (join != null && join.getType().equals("") == false) {
            TableRef dimTable = lookupDesc.getTableRef();
            if (!dimTableCache.contains(dimTable)) {
                TblColRef[] pk = join.getPrimaryKeyColumns();
                TblColRef[] fk = join.getForeignKeyColumns();
                if (pk.length != fk.length) {
                    throw new RuntimeException("Invalid join condition of lookup table:" + lookupDesc);
                }
                String joinType = join.getType().toUpperCase(Locale.ROOT);
                sql.append(joinType).append(" JOIN ").append(quoteTableIdentity(dimTable, sqlDialect))
                        .append(" as ").append(quoteIdentifier(dimTable.getAlias(), sqlDialect)).append(sep);
                sql.append("ON ");
                for (int i = 0; i < pk.length; i++) {
                    if (i > 0) {
                        sql.append(" AND ");
                    }
                    sql.append(getQuotedColExpressionInSourceDB(flatDesc, fk[i], sqlDialect)).append(" = ")
                            .append(getQuotedColExpressionInSourceDB(flatDesc, pk[i], sqlDialect));
                }
                sql.append(sep);

                dimTableCache.add(dimTable);
            }
        }
    }
}
Example 6
Source File: DimensionDesc.java    From kylin with Apache License 2.0 5 votes vote down vote up 
public void init(CubeDesc cubeDesc) {
    DataModelDesc model = cubeDesc.getModel();

    if (name != null)
        name = name.toUpperCase(Locale.ROOT);

    tableRef = model.findTable(table);
    table = tableRef.getAlias();

    join = null;
    for (JoinTableDesc joinTable : model.getJoinTables()) {
        if (joinTable.getTableRef().equals(this.tableRef)) {
            join = joinTable.getJoin();
            break;
        }
    }

    if (column != null && !"{FK}".equals(column)) {
        column = model.findColumn(table, column).getName();
    }
    if (derived != null && derived.length == 0) {
        derived = null;
    }
    if (derived != null) {
        for (int i = 0; i < derived.length; i++) {
            derived[i] = model.findColumn(table, derived[i]).getName();
        }
    }
    if (derived != null && join == null) {
        throw new IllegalStateException(
                "Derived can only be defined on lookup table, cube " + cubeDesc + ", " + this);
    }

}
Example 7
Source File: JdbcHiveInputBase.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
private static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
        IJdbcMetadata metadata, Map<String, String> metaMap) {
    final String sep = singleLine ? " " : "\n";
    Set<TableRef> dimTableCache = new HashSet<>();

    DataModelDesc model = flatDesc.getDataModel();
    sql.append(" FROM ")
            .append(getSchemaQuoted(metaMap,
                    flatDesc.getDataModel().getRootFactTable().getTableDesc().getDatabase(), metadata, true))
            .append(".")
            .append(getTableIdentityQuoted(flatDesc.getDataModel().getRootFactTable(), metaMap, metadata, true));

    sql.append(" ");
    sql.append((getTableIdentityQuoted(flatDesc.getDataModel().getRootFactTable(), metaMap, metadata, true)))
            .append(sep);

    for (JoinTableDesc lookupDesc : model.getJoinTables()) {
        JoinDesc join = lookupDesc.getJoin();
        if (join != null && !join.getType().equals("")) {
            TableRef dimTable = lookupDesc.getTableRef();
            if (!dimTableCache.contains(dimTable)) {
                TblColRef[] pk = join.getPrimaryKeyColumns();
                TblColRef[] fk = join.getForeignKeyColumns();
                if (pk.length != fk.length) {
                    throw new RuntimeException("Invalid join condition of lookup table:" + lookupDesc);
                }
                String joinType = join.getType().toUpperCase(Locale.ROOT);

                sql.append(joinType).append(" JOIN ")
                        .append(getSchemaQuoted(metaMap, dimTable.getTableDesc().getDatabase(), metadata, true))
                        .append(".").append(getTableIdentityQuoted(dimTable, metaMap, metadata, true));

                sql.append(" ");
                sql.append(getTableIdentityQuoted(dimTable, metaMap, metadata, true)).append(sep);
                sql.append("ON ");
                for (int i = 0; i < pk.length; i++) {
                    if (i > 0) {
                        sql.append(" AND ");
                    }
                    sql.append(getTableColumnIdentityQuoted(fk[i], metadata, metaMap, true)).append(" = ")
                            .append(getTableColumnIdentityQuoted(pk[i], metadata, metaMap, true));
                }
                sql.append(sep);
                dimTableCache.add(dimTable);
            }
        }
    }
}
Example 8
Source File: CacheServiceTest.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@Test
public void testMetaCRUD() throws Exception {
    final TableMetadataManager tableMgr = TableMetadataManager.getInstance(configA);
    final TableMetadataManager tableMgrB = TableMetadataManager.getInstance(configB);
    final DataModelManager modelMgr = DataModelManager.getInstance(configA);
    final DataModelManager modelMgrB = DataModelManager.getInstance(configB);
    final Broadcaster broadcaster = Broadcaster.getInstance(configA);
    broadcaster.getCounterAndClear();

    TableDesc tableDesc = createTestTableDesc();
    assertTrue(tableMgr.getTableDesc(tableDesc.getIdentity(), "default") == null);
    assertTrue(tableMgrB.getTableDesc(tableDesc.getIdentity(), "default") == null);
    tableMgr.saveSourceTable(tableDesc, "default");
    //only one for table insert
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertNotNull(tableMgr.getTableDesc(tableDesc.getIdentity(), "default"));
    assertNotNull(tableMgrB.getTableDesc(tableDesc.getIdentity(), "default"));

    final String dataModelName = "test_data_model";
    DataModelDesc dataModelDesc = modelMgr.getDataModelDesc("test_kylin_left_join_model_desc");
    dataModelDesc.setName(dataModelName);
    dataModelDesc.setLastModified(0);
    assertTrue(modelMgr.getDataModelDesc(dataModelName) == null);
    assertTrue(modelMgrB.getDataModelDesc(dataModelName) == null);

    dataModelDesc.setName(dataModelName);
    modelMgr.createDataModelDesc(dataModelDesc, "default", "ADMIN");
    //one for data model creation, one for project meta update
    assertEquals(2, broadcaster.getCounterAndClear());
    waitForCounterAndClear(2);
    assertEquals(dataModelDesc.getName(), modelMgrB.getDataModelDesc(dataModelName).getName());

    final JoinTableDesc[] lookups = dataModelDesc.getJoinTables();
    assertTrue(lookups.length > 0);
    modelMgr.updateDataModelDesc(dataModelDesc);
    //only one for data model update
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertEquals(dataModelDesc.getJoinTables().length,
            modelMgrB.getDataModelDesc(dataModelName).getJoinTables().length);

}
Example 9
Source File: JdbcHiveInputBase.java    From kylin with Apache License 2.0 4 votes vote down vote up 
private static void appendJoinStatement(IJoinedFlatTableDesc flatDesc, StringBuilder sql, boolean singleLine,
        IJdbcMetadata metadata, Map<String, String> metaMap) {
    final String sep = singleLine ? " " : "\n";
    Set<TableRef> dimTableCache = new HashSet<>();

    DataModelDesc model = flatDesc.getDataModel();
    sql.append(" FROM ")
            .append(getSchemaQuoted(metaMap,
                    flatDesc.getDataModel().getRootFactTable().getTableDesc().getDatabase(), metadata, true))
            .append(".")
            .append(getTableIdentityQuoted(flatDesc.getDataModel().getRootFactTable(), metaMap, metadata, true));

    sql.append(" ");
    sql.append((getTableIdentityQuoted(flatDesc.getDataModel().getRootFactTable(), metaMap, metadata, true)))
            .append(sep);

    for (JoinTableDesc lookupDesc : model.getJoinTables()) {
        JoinDesc join = lookupDesc.getJoin();
        if (join != null && !join.getType().equals("")) {
            TableRef dimTable = lookupDesc.getTableRef();
            if (!dimTableCache.contains(dimTable)) {
                TblColRef[] pk = join.getPrimaryKeyColumns();
                TblColRef[] fk = join.getForeignKeyColumns();
                if (pk.length != fk.length) {
                    throw new RuntimeException("Invalid join condition of lookup table:" + lookupDesc);
                }
                String joinType = join.getType().toUpperCase(Locale.ROOT);

                sql.append(joinType).append(" JOIN ")
                        .append(getSchemaQuoted(metaMap, dimTable.getTableDesc().getDatabase(), metadata, true))
                        .append(".").append(getTableIdentityQuoted(dimTable, metaMap, metadata, true));

                sql.append(" ");
                sql.append(getTableIdentityQuoted(dimTable, metaMap, metadata, true)).append(sep);
                sql.append("ON ");
                for (int i = 0; i < pk.length; i++) {
                    if (i > 0) {
                        sql.append(" AND ");
                    }
                    sql.append(getTableColumnIdentityQuoted(fk[i], metadata, metaMap, true)).append(" = ")
                            .append(getTableColumnIdentityQuoted(pk[i], metadata, metaMap, true));
                }
                sql.append(sep);
                dimTableCache.add(dimTable);
            }
        }
    }
}
Example 10
Source File: CacheServiceTest.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@Test
public void testMetaCRUD() throws Exception {
    final TableMetadataManager tableMgr = TableMetadataManager.getInstance(configA);
    final TableMetadataManager tableMgrB = TableMetadataManager.getInstance(configB);
    final DataModelManager modelMgr = DataModelManager.getInstance(configA);
    final DataModelManager modelMgrB = DataModelManager.getInstance(configB);
    final Broadcaster broadcaster = Broadcaster.getInstance(configA);
    broadcaster.getCounterAndClear();

    TableDesc tableDesc = createTestTableDesc();
    assertTrue(tableMgr.getTableDesc(tableDesc.getIdentity(), "default") == null);
    assertTrue(tableMgrB.getTableDesc(tableDesc.getIdentity(), "default") == null);
    tableMgr.saveSourceTable(tableDesc, "default");
    //only one for table insert
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertNotNull(tableMgr.getTableDesc(tableDesc.getIdentity(), "default"));
    assertNotNull(tableMgrB.getTableDesc(tableDesc.getIdentity(), "default"));

    final String dataModelName = "test_data_model";
    DataModelDesc dataModelDesc = modelMgr.getDataModelDesc("test_kylin_left_join_model_desc");
    dataModelDesc.setName(dataModelName);
    dataModelDesc.setLastModified(0);
    assertTrue(modelMgr.getDataModelDesc(dataModelName) == null);
    assertTrue(modelMgrB.getDataModelDesc(dataModelName) == null);

    dataModelDesc.setName(dataModelName);
    modelMgr.createDataModelDesc(dataModelDesc, "default", "ADMIN");
    //one for data model creation, one for project meta update
    assertEquals(2, broadcaster.getCounterAndClear());
    waitForCounterAndClear(2);
    assertEquals(dataModelDesc.getName(), modelMgrB.getDataModelDesc(dataModelName).getName());

    final JoinTableDesc[] lookups = dataModelDesc.getJoinTables();
    assertTrue(lookups.length > 0);
    modelMgr.updateDataModelDesc(dataModelDesc);
    //only one for data model update
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertEquals(dataModelDesc.getJoinTables().length,
            modelMgrB.getDataModelDesc(dataModelName).getJoinTables().length);

}
Example 11
Source File: TableSchemaUpdater.java    From kylin with Apache License 2.0 4 votes vote down vote up 
public static DataModelDesc dealWithMappingForModel(DataModelDesc other,
        Map<String, TableSchemaUpdateMapping> mappings) {
    // For filter condition, not support
    if (!Strings.isNullOrEmpty(other.getFilterCondition())) {
        throw new UnsupportedOperationException("Cannot deal with filter condition " + other.getFilterCondition());
    }

    DataModelDesc copy = DataModelDesc.getCopyOf(other);
    copy.setLastModified(other.getLastModified());

    // mapping for root fact table identity
    TableSchemaUpdateMapping rootMapping = getTableSchemaUpdateMapping(mappings, other.getRootFactTableName());
    if (rootMapping != null) {
        TableDesc rootFactTable = other.getRootFactTable().getTableDesc();
        copy.setRootFactTableName(
                rootMapping.getTableIdentity(rootFactTable.getDatabase(), rootFactTable.getName()));
    }

    // mapping for joins
    JoinTableDesc[] joinTables = other.getJoinTables();
    JoinTableDesc[] joinTablesCopy = new JoinTableDesc[joinTables.length];
    for (int i = 0; i < joinTables.length; i++) {
        JoinTableDesc joinTable = joinTables[i];
        joinTablesCopy[i] = JoinTableDesc.getCopyOf(joinTable);
        String tableIdentity = joinTable.getTable();
        TableSchemaUpdateMapping mapping = getTableSchemaUpdateMapping(mappings, tableIdentity);
        if (mapping != null && mapping.isTableIdentityChanged()) {
            joinTablesCopy[i].setTable(mapping.getTableIdentity(tableIdentity));
        }
    }
    copy.setJoinTables(joinTablesCopy);

    // mapping for partition columns
    PartitionDesc partDesc = other.getPartitionDesc();
    PartitionDesc partCopy = PartitionDesc.getCopyOf(partDesc);
    if (partDesc.getPartitionDateColumnRef() != null) {
        partCopy.setPartitionDateColumn(
                replacePartitionCol(partDesc.getPartitionDateColumnRef().getCanonicalName(), mappings));
    }
    if (partDesc.getPartitionTimeColumnRef() != null) {
        partCopy.setPartitionTimeColumn(
                replacePartitionCol(partDesc.getPartitionTimeColumnRef().getCanonicalName(), mappings));
    }
    copy.setPartitionDesc(partCopy);

    return copy;
}