Java Code Examples for org.apache.kylin.cube.CubeManager#getLookupTable()

The following examples show how to use org.apache.kylin.cube.CubeManager#getLookupTable() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SparkBuildDictionary.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up 
private void checkSnapshot(CubeManager cubeManager, CubeSegment cubeSegment) {
    List<DimensionDesc> dimensionDescs = cubeSegment.getCubeDesc().getDimensions();
    for (DimensionDesc dimensionDesc : dimensionDescs) {
        TableRef lookup = dimensionDesc.getTableRef();
        String tableIdentity = lookup.getTableIdentity();
        if (cubeSegment.getModel().isLookupTable(tableIdentity) && !cubeSegment.getCubeDesc().isExtSnapshotTable(tableIdentity)) {
            logger.info("Checking snapshot of {}", lookup);
            try {
                JoinDesc join = cubeSegment.getModel().getJoinsTree().getJoinByPKSide(lookup);
                ILookupTable table = cubeManager.getLookupTable(cubeSegment, join);
                if (table != null) {
                    IOUtils.closeStream(table);
                }
            } catch (Throwable th) {
                throw new RuntimeException(String.format(Locale.ROOT, "Checking snapshot of %s failed.", lookup), th);
            }
        }
    }
}
Example 2
Source File: SparkBuildDictionary.java    From kylin with Apache License 2.0 6 votes vote down vote up 
private void checkSnapshot(CubeManager cubeManager, CubeSegment cubeSegment) {
    List<DimensionDesc> dimensionDescs = cubeSegment.getCubeDesc().getDimensions();
    for (DimensionDesc dimensionDesc : dimensionDescs) {
        TableRef lookup = dimensionDesc.getTableRef();
        String tableIdentity = lookup.getTableIdentity();
        if (cubeSegment.getModel().isLookupTable(tableIdentity) && !cubeSegment.getCubeDesc().isExtSnapshotTable(tableIdentity)) {
            logger.info("Checking snapshot of {}", lookup);
            try {
                JoinDesc join = cubeSegment.getModel().getJoinsTree().getJoinByPKSide(lookup);
                ILookupTable table = cubeManager.getLookupTable(cubeSegment, join);
                if (table != null) {
                    IOUtils.closeStream(table);
                }
            } catch (Throwable th) {
                throw new RuntimeException(String.format(Locale.ROOT, "Checking snapshot of %s failed.", lookup), th);
            }
        }
    }
}
Example 3
Source File: LookupTableEnumerator.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
public LookupTableEnumerator(OLAPContext olapContext) {

        //TODO: assuming LookupTableEnumerator is handled by a cube
        CubeInstance cube = (CubeInstance) olapContext.realization;

        String lookupTableName = olapContext.firstTableScan.getTableName();
        DimensionDesc dim = cube.getDescriptor().findDimensionByTable(lookupTableName);
        if (dim == null)
            throw new IllegalStateException("No dimension with derived columns found for lookup table " + lookupTableName + ", cube desc " + cube.getDescriptor());

        CubeManager cubeMgr = CubeManager.getInstance(cube.getConfig());
        LookupStringTable table = cubeMgr.getLookupTable(cube.getLatestReadySegment(), dim);
        this.allRows = table.getAllRows();

        OLAPTable olapTable = (OLAPTable) olapContext.firstTableScan.getOlapTable();
        this.colDescs = olapTable.getExposedColumns();
        this.current = new Object[colDescs.size()];

        reset();
    }
Example 4
Source File: DictionaryGeneratorCLI.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private static void processSegment(KylinConfig config, CubeSegment cubeSeg, String factColumnsPath) throws IOException {
    CubeManager cubeMgr = CubeManager.getInstance(config);

    for (DimensionDesc dim : cubeSeg.getCubeDesc().getDimensions()) {
        // dictionary
        for (TblColRef col : dim.getColumnRefs()) {
            if (cubeSeg.getCubeDesc().getRowkey().isUseDictionary(col)) {
                logger.info("Building dictionary for " + col);
                cubeMgr.buildDictionary(cubeSeg, col, factColumnsPath);
            }
        }

        // build snapshot
        if (dim.getTable() != null && !dim.getTable().equalsIgnoreCase(cubeSeg.getCubeDesc().getFactTable())) {
            // CubeSegment seg = cube.getTheOnlySegment();
            logger.info("Building snapshot of " + dim.getTable());
            cubeMgr.buildSnapshotTable(cubeSeg, dim.getTable());
            logger.info("Checking snapshot of " + dim.getTable());
            cubeMgr.getLookupTable(cubeSeg, dim); // load the table for
                                                  // sanity check
        }
    }
}
Example 5
Source File: Tuple.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
public static IDerivedColumnFiller newDerivedColumnFiller(List<TblColRef> rowColumns, TblColRef[] hostCols, DeriveInfo deriveInfo, TupleInfo tupleInfo, CubeManager cubeMgr, CubeSegment cubeSegment) {

        int[] hostIndex = new int[hostCols.length];
        for (int i = 0; i < hostCols.length; i++) {
            hostIndex[i] = rowColumns.indexOf(hostCols[i]);
        }
        String[] derivedFieldNames = new String[deriveInfo.columns.length];
        for (int i = 0; i < deriveInfo.columns.length; i++) {
            derivedFieldNames[i] = tupleInfo.getFieldName(deriveInfo.columns[i]);
        }

        switch (deriveInfo.type) {
        case LOOKUP:
            LookupStringTable lookupTable = cubeMgr.getLookupTable(cubeSegment, deriveInfo.dimension);
            return new LookupFiller(hostIndex, lookupTable, deriveInfo, derivedFieldNames);
        case PK_FK:
            // composite key are split, see CubeDesc.initDimensionColumns()
            return new PKFKFiller(hostIndex[0], derivedFieldNames[0]);
        default:
            throw new IllegalArgumentException();
        }
    }
Example 6
Source File: CubeStorageEngine.java    From Kylin with Apache License 2.0 6 votes vote down vote up 
private TupleFilter translateDerivedInCompare(CompareTupleFilter compf, Set<TblColRef> collector) {
    if (compf.getColumn() == null || compf.getValues().isEmpty())
        return compf;

    TblColRef derived = compf.getColumn();
    if (cubeDesc.isDerived(derived) == false)
        return compf;

    DeriveInfo hostInfo = cubeDesc.getHostInfo(derived);
    CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
    CubeSegment seg = cubeInstance.getLatestReadySegment();
    LookupStringTable lookup = cubeMgr.getLookupTable(seg, hostInfo.dimension);
    Pair<TupleFilter, Boolean> translated = DerivedFilterTranslator.translate(lookup, hostInfo, compf);
    TupleFilter translatedFilter = translated.getFirst();
    boolean loosened = translated.getSecond();
    if (loosened) {
        collectColumnsRecursively(compf, collector);
    }
    return translatedFilter;
}
Example 7
Source File: GTCubeStorageQueryBase.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
protected ILookupTable getLookupStringTableForDerived(TblColRef derived, DeriveInfo hostInfo) {
    CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
    CubeSegment seg = cubeInstance.getLatestReadySegment();
    return cubeMgr.getLookupTable(seg, hostInfo.join);
}
Example 8
Source File: GTCubeStorageQueryBase.java    From kylin with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
protected ILookupTable getLookupStringTableForDerived(TblColRef derived, DeriveInfo hostInfo) {
    CubeManager cubeMgr = CubeManager.getInstance(this.cubeInstance.getConfig());
    CubeSegment seg = cubeInstance.getLatestReadySegment();
    return cubeMgr.getLookupTable(seg, hostInfo.join);
}