Java Code Examples for org.apache.kylin.cube.CubeSegment#putSnapshotResPath()

The following examples show how to use org.apache.kylin.cube.CubeSegment#putSnapshotResPath() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: UpdateCubeInfoAfterBuildStep.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private void saveExtSnapshotIfNeeded(CubeManager cubeManager, CubeInstance cube, CubeSegment segment)
        throws IOException {
    String extLookupSnapshotStr = this.getParam(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO);
    if (extLookupSnapshotStr == null || extLookupSnapshotStr.isEmpty()) {
        return;
    }
    Map<String, String> extLookupSnapshotMap = LookupMaterializeContext.parseLookupSnapshots(extLookupSnapshotStr);
    logger.info("update ext lookup snapshots:{}", extLookupSnapshotMap);
    List<SnapshotTableDesc> snapshotTableDescList = cube.getDescriptor().getSnapshotTableDescList();
    for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescList) {
        String tableName = snapshotTableDesc.getTableName();
        if (snapshotTableDesc.isExtSnapshotTable()) {
            String newSnapshotResPath = extLookupSnapshotMap.get(tableName);
            if (newSnapshotResPath == null || newSnapshotResPath.isEmpty()) {
                continue;
            }

            if (snapshotTableDesc.isGlobal()) {
                if (!newSnapshotResPath.equals(cube.getSnapshotResPath(tableName))) {
                    cubeManager.updateCubeLookupSnapshot(cube, tableName, newSnapshotResPath);
                }
            } else {
                segment.putSnapshotResPath(tableName, newSnapshotResPath);
            }
        }
    }
}

Example 2

Source File: UpdateCubeInfoAfterBuildStep.java From kylin with Apache License 2.0

5 votes

private void saveExtSnapshotIfNeeded(CubeManager cubeManager, CubeInstance cube, CubeSegment segment)
        throws IOException {
    String extLookupSnapshotStr = this.getParam(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO);
    if (extLookupSnapshotStr == null || extLookupSnapshotStr.isEmpty()) {
        return;
    }
    Map<String, String> extLookupSnapshotMap = LookupMaterializeContext.parseLookupSnapshots(extLookupSnapshotStr);
    logger.info("update ext lookup snapshots:{}", extLookupSnapshotMap);
    List<SnapshotTableDesc> snapshotTableDescList = cube.getDescriptor().getSnapshotTableDescList();
    for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescList) {
        String tableName = snapshotTableDesc.getTableName();
        if (snapshotTableDesc.isExtSnapshotTable()) {
            String newSnapshotResPath = extLookupSnapshotMap.get(tableName);
            if (newSnapshotResPath == null || newSnapshotResPath.isEmpty()) {
                continue;
            }

            if (snapshotTableDesc.isGlobal()) {
                if (!newSnapshotResPath.equals(cube.getSnapshotResPath(tableName))) {
                    cubeManager.updateCubeLookupSnapshot(cube, tableName, newSnapshotResPath);
                }
            } else {
                segment.putSnapshotResPath(tableName, newSnapshotResPath);
            }
        }
    }
}

Example 3

Source File: UpdateDictionaryStep.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeMgr = CubeManager.getInstance(context.getConfig());
    final DictionaryManager dictMgrHdfs;
    final DictionaryManager dictMgrHbase;
    final CubeInstance cube = cubeMgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
    final List<CubeSegment> mergingSegments = getMergingSegments(cube);
    final String dictInfoPath = this.getParams().get(BatchConstants.ARG_DICT_PATH);
    final String metadataUrl = this.getParams().get(BatchConstants.ARG_META_URL);

    final KylinConfig kylinConfHbase = cube.getConfig();
    final KylinConfig kylinConfHdfs = AbstractHadoopJob.loadKylinConfigFromHdfs(metadataUrl);

    Collections.sort(mergingSegments);

    try {
        Configuration conf = HadoopUtil.getCurrentConfiguration();
        FileSystem fs = HadoopUtil.getWorkingFileSystem();
        ResourceStore hbaseRS = ResourceStore.getStore(kylinConfHbase);
        ResourceStore hdfsRS = ResourceStore.getStore(kylinConfHdfs);
        dictMgrHdfs = DictionaryManager.getInstance(kylinConfHdfs);
        dictMgrHbase = DictionaryManager.getInstance(kylinConfHbase);

        // work on copy instead of cached objects
        CubeInstance cubeCopy = cube.latestCopyForWrite();
        CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid());

        // update cube segment dictionary

        FileStatus[] fileStatuss = fs.listStatus(new Path(dictInfoPath), new PathFilter() {
            @Override
            public boolean accept(Path path) {
                return path.getName().startsWith("part") || path.getName().startsWith("tmp");
            }
        });

        for (FileStatus fileStatus : fileStatuss) {
            Path filePath = fileStatus.getPath();

            SequenceFile.Reader reader = new SequenceFile.Reader(fs, filePath, conf);
            Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
            Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf);

            while (reader.next(key, value)) {
                String tblCol = key.toString();
                String dictInfoResource = value.toString();

                if (StringUtils.isNotEmpty(dictInfoResource)) {
                    logger.info(dictInfoResource);
                    // put dictionary file to metadata store
                    DictionaryInfo dictInfoHdfs = dictMgrHdfs.getDictionaryInfo(dictInfoResource);
                    DictionaryInfo dicInfoHbase = dictMgrHbase.trySaveNewDict(dictInfoHdfs.getDictionaryObject(), dictInfoHdfs);

                    if (dicInfoHbase != null){
                        TblColRef tblColRef = cube.getDescriptor().findColumnRef(tblCol.split(":")[0], tblCol.split(":")[1]);
                        newSegCopy.putDictResPath(tblColRef, dicInfoHbase.getResourcePath());
                    }
                }
            }

            IOUtils.closeStream(reader);
        }

        CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
        for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
            newSegCopy.putSnapshotResPath(entry.getKey(), entry.getValue());
        }

        // update statistics
        // put the statistics to metadata store
        String statisticsFileName = newSegment.getStatisticsResourcePath();
        hbaseRS.putResource(statisticsFileName, hdfsRS.getResource(newSegment.getStatisticsResourcePath()).content(), System.currentTimeMillis());

        CubeUpdate update = new CubeUpdate(cubeCopy);
        update.setToUpdateSegs(newSegCopy);
        cubeMgr.updateCube(update);

        return ExecuteResult.createSucceed();
    } catch (IOException e) {
        logger.error("fail to merge dictionary", e);
        return ExecuteResult.createError(e);
    }
}

Example 4

Source File: AfterMergeOrRefreshResourceMerger.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

private void makeSnapshotForNewSegment(CubeSegment newSeg, List<CubeSegment> mergingSegments) {
    CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
    for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
        newSeg.putSnapshotResPath(entry.getKey(), entry.getValue());
    }
}

Example 5

Source File: UpdateDictionaryStep.java From kylin with Apache License 2.0

4 votes

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    final CubeManager cubeMgr = CubeManager.getInstance(context.getConfig());
    final DictionaryManager dictMgrHdfs;
    final DictionaryManager dictMgrHbase;
    final CubeInstance cube = cubeMgr.getCube(CubingExecutableUtil.getCubeName(this.getParams()));
    final CubeSegment newSegment = cube.getSegmentById(CubingExecutableUtil.getSegmentId(this.getParams()));
    final List<CubeSegment> mergingSegments = getMergingSegments(cube);
    final String dictInfoPath = this.getParams().get(BatchConstants.ARG_DICT_PATH);
    final String metadataUrl = this.getParams().get(BatchConstants.ARG_META_URL);

    final KylinConfig kylinConfHbase = cube.getConfig();
    final KylinConfig kylinConfHdfs = AbstractHadoopJob.loadKylinConfigFromHdfs(metadataUrl);

    Collections.sort(mergingSegments);

    try {
        Configuration conf = HadoopUtil.getCurrentConfiguration();
        FileSystem fs = HadoopUtil.getWorkingFileSystem();
        ResourceStore hbaseRS = ResourceStore.getStore(kylinConfHbase);
        ResourceStore hdfsRS = ResourceStore.getStore(kylinConfHdfs);
        dictMgrHdfs = DictionaryManager.getInstance(kylinConfHdfs);
        dictMgrHbase = DictionaryManager.getInstance(kylinConfHbase);

        // work on copy instead of cached objects
        CubeInstance cubeCopy = cube.latestCopyForWrite();
        CubeSegment newSegCopy = cubeCopy.getSegmentById(newSegment.getUuid());

        // update cube segment dictionary

        FileStatus[] fileStatuss = fs.listStatus(new Path(dictInfoPath), new PathFilter() {
            @Override
            public boolean accept(Path path) {
                return path.getName().startsWith("part") || path.getName().startsWith("tmp");
            }
        });

        for (FileStatus fileStatus : fileStatuss) {
            Path filePath = fileStatus.getPath();

            SequenceFile.Reader reader = new SequenceFile.Reader(fs, filePath, conf);
            Text key = (Text) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
            Text value = (Text) ReflectionUtils.newInstance(reader.getValueClass(), conf);

            while (reader.next(key, value)) {
                String tblCol = key.toString();
                String dictInfoResource = value.toString();

                if (StringUtils.isNotEmpty(dictInfoResource)) {
                    logger.info(dictInfoResource);
                    // put dictionary file to metadata store
                    DictionaryInfo dictInfoHdfs = dictMgrHdfs.getDictionaryInfo(dictInfoResource);
                    DictionaryInfo dicInfoHbase = dictMgrHbase.trySaveNewDict(dictInfoHdfs.getDictionaryObject(), dictInfoHdfs);

                    if (dicInfoHbase != null){
                        TblColRef tblColRef = cube.getDescriptor().findColumnRef(tblCol.split(":")[0], tblCol.split(":")[1]);
                        newSegCopy.putDictResPath(tblColRef, dicInfoHbase.getResourcePath());
                    }
                }
            }

            IOUtils.closeStream(reader);
        }

        CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
        for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
            newSegCopy.putSnapshotResPath(entry.getKey(), entry.getValue());
        }

        // update statistics
        // put the statistics to metadata store
        String statisticsFileName = newSegment.getStatisticsResourcePath();
        hbaseRS.putResource(statisticsFileName, hdfsRS.getResource(newSegment.getStatisticsResourcePath()).content(), System.currentTimeMillis());

        CubeUpdate update = new CubeUpdate(cubeCopy);
        update.setToUpdateSegs(newSegCopy);
        cubeMgr.updateCube(update);

        return ExecuteResult.createSucceed();
    } catch (IOException e) {
        logger.error("fail to merge dictionary", e);
        return ExecuteResult.createError(e);
    }
}

Example 6

Source File: MergeDictionaryStep.java From kylin-on-parquet-v2 with Apache License 2.0

3 votes

/**
 * make snapshots for the new segment by copying from the latest one of the underlying
 * merging segments. It's guaranteed to be consistent under the assumption that lookup tables
 * would be either static or incremental.
 *
 * @param cube
 * @param newSeg
 */
private void makeSnapshotForNewSegment(CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) {
    CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
    for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
        newSeg.putSnapshotResPath(entry.getKey(), entry.getValue());
    }
}

Example 7

Source File: MergeDictionaryStep.java From kylin with Apache License 2.0

3 votes

/**
 * make snapshots for the new segment by copying from the latest one of the underlying
 * merging segments. It's guaranteed to be consistent under the assumption that lookup tables
 * would be either static or incremental.
 *
 * @param cube
 * @param newSeg
 */
private void makeSnapshotForNewSegment(CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) {
    CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
    for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
        newSeg.putSnapshotResPath(entry.getKey(), entry.getValue());
    }
}

Example 8

Source File: MergeDictionaryStep.java From Kylin with Apache License 2.0

3 votes

/**
 * make snapshots for the new segment by copying from one of the underlying
 * merging segments. it's guaranteed to be consistent(checked in
 * CubeSegmentValidator)
 *
 * @param cube
 * @param newSeg
 */
private void makeSnapshotForNewSegment(CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) {
    CubeSegment lastSeg = mergingSegments.get(mergingSegments.size() - 1);
    for (Map.Entry<String, String> entry : lastSeg.getSnapshots().entrySet()) {
        newSeg.putSnapshotResPath(entry.getKey(), entry.getValue());
    }
}