Java Code Examples for org.apache.kylin.cube.model.CubeDesc#isGlobalSnapshotTable()

The following examples show how to use org.apache.kylin.cube.model.CubeDesc#isGlobalSnapshotTable() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LookupSnapshotToMetaStoreStep.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig kylinConfig = context.getConfig();
    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    TableMetadataManager metaMgr = TableMetadataManager.getInstance(kylinConfig);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(kylinConfig);
    CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams()));
    List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams());
    String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams());
    CubeDesc cubeDesc = cube.getDescriptor();
    try {
        TableDesc tableDesc = metaMgr.getTableDesc(lookupTableName, cube.getProject());
        IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null);
        logger.info("take snapshot for table:" + lookupTableName);
        SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cube.getConfig());

        logger.info("update snapshot path to cube metadata");
        if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
            LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
                    snapshot.getResourcePath());
        } else {
            LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
                    snapshot.getResourcePath());
        }
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to build snapshot for:" + lookupTableName, e);
        return ExecuteResult.createError(e);
    }
}
 
Example 2
Source File: UpdateCubeAfterSnapshotStep.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig kylinConfig = context.getConfig();
    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);

    CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams()));
    List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams());
    String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams());

    String extLookupSnapshotStr = this.getParam(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO);
    if (extLookupSnapshotStr == null || extLookupSnapshotStr.isEmpty()) {
        return new ExecuteResult();
    }

    Map<String, String> extLookupSnapshotMap = LookupMaterializeContext.parseLookupSnapshots(extLookupSnapshotStr);
    String snapshotResPath = extLookupSnapshotMap.get(lookupTableName);
    if (snapshotResPath == null) {
        logger.info("no snapshot path exist in the context, so no need to update snapshot path");
        return new ExecuteResult();
    }
    CubeDesc cubeDesc = cube.getDescriptor();
    try {
        logger.info("update snapshot path:{} to cube:{}", snapshotResPath, cube.getName());
        if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
            if (!snapshotResPath.equals(cube.getSnapshotResPath(lookupTableName))) {
                LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
                        snapshotResPath);
            }
        } else {
            LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
                    snapshotResPath);
        }
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to save cuboid statistics", e);
        return ExecuteResult.createError(e);
    }
}
 
Example 3
Source File: CubeManager.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable, String uuid)
        throws IOException {
    // work on copy instead of cached objects
    CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy
    CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid());

    TableMetadataManager metaMgr = getTableManager();
    SnapshotManager snapshotMgr = getSnapshotManager();

    TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());

    CubeDesc cubeDesc = cubeSeg.getCubeDesc();
    if (!cubeDesc.isGlobalSnapshotTable(lookupTable)) {
        segCopy.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
        CubeUpdate update = new CubeUpdate(cubeCopy);
        update.setToUpdateSegs(segCopy);
        updateCube(update);

        // Update the input cubeSeg after the resource store updated
        cubeSeg.putSnapshotResPath(lookupTable, segCopy.getSnapshotResPath(lookupTable));
    } else {
        CubeUpdate cubeUpdate = new CubeUpdate(cubeCopy);
        Map<String, String> map = Maps.newHashMap();
        map.put(lookupTable, snapshot.getResourcePath());
        cubeUpdate.setUpdateTableSnapshotPath(map);
        updateCube(cubeUpdate);

        cubeSeg.getCubeInstance().putSnapshotResPath(lookupTable, snapshot.getResourcePath());
    }
    return snapshot;
}
 
Example 4
Source File: LookupSnapshotToMetaStoreStep.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig kylinConfig = context.getConfig();
    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);
    TableMetadataManager metaMgr = TableMetadataManager.getInstance(kylinConfig);
    SnapshotManager snapshotMgr = SnapshotManager.getInstance(kylinConfig);
    CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams()));
    List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams());
    String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams());
    CubeDesc cubeDesc = cube.getDescriptor();
    try {
        TableDesc tableDesc = metaMgr.getTableDesc(lookupTableName, cube.getProject());
        IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, null);
        logger.info("take snapshot for table:" + lookupTableName);
        SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cube.getConfig());

        logger.info("update snapshot path to cube metadata");
        if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
            LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
                    snapshot.getResourcePath());
        } else {
            LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
                    snapshot.getResourcePath());
        }
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to build snapshot for:" + lookupTableName, e);
        return ExecuteResult.createError(e);
    }
}
 
Example 5
Source File: UpdateCubeAfterSnapshotStep.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig kylinConfig = context.getConfig();
    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);

    CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams()));
    List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams());
    String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams());

    String extLookupSnapshotStr = this.getParam(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO);
    if (extLookupSnapshotStr == null || extLookupSnapshotStr.isEmpty()) {
        return new ExecuteResult();
    }

    Map<String, String> extLookupSnapshotMap = LookupMaterializeContext.parseLookupSnapshots(extLookupSnapshotStr);
    String snapshotResPath = extLookupSnapshotMap.get(lookupTableName);
    if (snapshotResPath == null) {
        logger.info("no snapshot path exist in the context, so no need to update snapshot path");
        return new ExecuteResult();
    }
    CubeDesc cubeDesc = cube.getDescriptor();
    try {
        logger.info("update snapshot path:{} to cube:{}", snapshotResPath, cube.getName());
        if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
            if (!snapshotResPath.equals(cube.getSnapshotResPath(lookupTableName))) {
                LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
                        snapshotResPath);
            }
        } else {
            LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
                    snapshotResPath);
        }
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to save cuboid statistics", e);
        return ExecuteResult.createError(e);
    }
}
 
Example 6
Source File: CubeManager.java    From kylin with Apache License 2.0 5 votes vote down vote up
public SnapshotTable buildSnapshotTable(CubeSegment cubeSeg, String lookupTable, String uuid)
        throws IOException {
    // work on copy instead of cached objects
    CubeInstance cubeCopy = cubeSeg.getCubeInstance().latestCopyForWrite(); // get a latest copy
    CubeSegment segCopy = cubeCopy.getSegmentById(cubeSeg.getUuid());

    TableMetadataManager metaMgr = getTableManager();
    SnapshotManager snapshotMgr = getSnapshotManager();

    TableDesc tableDesc = new TableDesc(metaMgr.getTableDesc(lookupTable, segCopy.getProject()));
    IReadableTable hiveTable = SourceManager.createReadableTable(tableDesc, uuid);
    SnapshotTable snapshot = snapshotMgr.buildSnapshot(hiveTable, tableDesc, cubeSeg.getConfig());

    CubeDesc cubeDesc = cubeSeg.getCubeDesc();
    if (!cubeDesc.isGlobalSnapshotTable(lookupTable)) {
        segCopy.putSnapshotResPath(lookupTable, snapshot.getResourcePath());
        CubeUpdate update = new CubeUpdate(cubeCopy);
        update.setToUpdateSegs(segCopy);
        updateCube(update);

        // Update the input cubeSeg after the resource store updated
        cubeSeg.putSnapshotResPath(lookupTable, segCopy.getSnapshotResPath(lookupTable));
    } else {
        CubeUpdate cubeUpdate = new CubeUpdate(cubeCopy);
        Map<String, String> map = Maps.newHashMap();
        map.put(lookupTable, snapshot.getResourcePath());
        cubeUpdate.setUpdateTableSnapshotPath(map);
        updateCube(cubeUpdate);

        cubeSeg.getCubeInstance().putSnapshotResPath(lookupTable, snapshot.getResourcePath());
    }
    return snapshot;
}