Java Code Examples for org.apache.kylin.common.KylinConfig#getHdfsWorkingDirectory()

The following examples show how to use org.apache.kylin.common.KylinConfig#getHdfsWorkingDirectory() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: HDFSResourceStore.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public HDFSResourceStore(KylinConfig kylinConfig, StorageURL metadataUrl) throws Exception {
    super(kylinConfig);
    Preconditions.checkState(HDFS_SCHEME.equals(metadataUrl.getScheme()));

    String path = metadataUrl.getParameter("path");
    if (path == null) {
        // missing path is not expected, but don't fail it
        path = kylinConfig.getHdfsWorkingDirectory(null) + "tmp_metadata";
        logger.warn("Missing path, fall back to {}. ", path);
    }

    fs = HadoopUtil.getFileSystem(path);
    Path metadataPath = new Path(path);
    if (!fs.exists(metadataPath)) {
        logger.warn("Path not exist in HDFS, create it: {}. ", path);
        createMetaFolder(metadataPath);
    }

    hdfsMetaPath = metadataPath;
    logger.info("hdfs meta path : {}", hdfsMetaPath);

}

Example 2

Source File: AppendTrieDictionary.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

private String convertToAbsolutePath(String path) {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    Path basicPath = new Path(path);
    if (basicPath.toUri().getScheme() == null)
        return kylinConfig.getHdfsWorkingDirectory() + path;

    String[] paths = path.split("/resources/GlobalDict/");
    if (paths.length == 2)
        return kylinConfig.getHdfsWorkingDirectory() + "/resources/GlobalDict/" + paths[1];

    paths = path.split("/resources/SegmentDict/");
    if (paths.length == 2) {
        return kylinConfig.getHdfsWorkingDirectory() + "/resources/SegmentDict/" + paths[1];
    } else {
        throw new RuntimeException(
                "the basic directory of global dictionary only support the format which contains '/resources/GlobalDict/' or '/resources/SegmentDict/'");
    }
}

Example 3

Source File: NSparkUpdateMetaAndCleanupAfterMergeStep.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    String cubeId = getParam(MetadataConstants.P_CUBE_ID);
    String[] segments = StringUtils.split(getParam(MetadataConstants.P_SEGMENT_NAMES), ",");
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    CubeInstance cube = CubeManager.getInstance(config).getCubeByUuid(cubeId);

    updateMetadataAfterMerge(cubeId);

    for (String segmentName : segments) {
        String path = config.getHdfsWorkingDirectory() + cube.getProject() + "/parquet/" + cube.getName() + "/" + segmentName;
        try {
            HadoopUtil.deletePath(HadoopUtil.getCurrentConfiguration(), new Path(path));
        } catch (IOException e) {
            throw new ExecuteException("Can not delete segment: " + segmentName + ", in cube: " + cube.getName());
        }
    }

    return ExecuteResult.createSucceed();
}

Example 4

Source File: HDFSResourceStore.java From kylin with Apache License 2.0

6 votes

public HDFSResourceStore(KylinConfig kylinConfig, StorageURL metadataUrl) throws Exception {
    super(kylinConfig);
    Preconditions.checkState(HDFS_SCHEME.equals(metadataUrl.getScheme()));

    String path = metadataUrl.getParameter("path");
    if (path == null) {
        // missing path is not expected, but don't fail it
        path = kylinConfig.getHdfsWorkingDirectory(null) + "tmp_metadata";
        logger.warn("Missing path, fall back to {}. ", path);
    }

    fs = HadoopUtil.getFileSystem(path);
    Path metadataPath = new Path(path);
    if (!fs.exists(metadataPath)) {
        logger.warn("Path not exist in HDFS, create it: {}. ", path);
        createMetaFolder(metadataPath);
    }

    hdfsMetaPath = metadataPath;
    logger.info("hdfs meta path : {}", hdfsMetaPath);

}

Example 5

Source File: AppendTrieDictionary.java From kylin with Apache License 2.0

6 votes

private String convertToAbsolutePath(String path) {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    Path basicPath = new Path(path);
    if (basicPath.toUri().getScheme() == null)
        return kylinConfig.getHdfsWorkingDirectory() + path;

    String[] paths = path.split("/resources/GlobalDict/");
    if (paths.length == 2)
        return kylinConfig.getHdfsWorkingDirectory() + "/resources/GlobalDict/" + paths[1];

    paths = path.split("/resources/SegmentDict/");
    if (paths.length == 2) {
        return kylinConfig.getHdfsWorkingDirectory() + "/resources/SegmentDict/" + paths[1];
    } else {
        throw new RuntimeException(
                "the basic directory of global dictionary only support the format which contains '/resources/GlobalDict/' or '/resources/SegmentDict/'");
    }
}

Example 6

Source File: AppendTrieDictionary.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

/**
 * JIRA: https://issues.apache.org/jira/browse/KYLIN-2945
 * if pass a absolute path, it may produce some problems like cannot find global dict after migration.
 * so convert to relative path can avoid it and be better to maintain flexibility.
 *
 */
private String convertToRelativePath(String path) {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    String hdfsWorkingDir = kylinConfig.getHdfsWorkingDirectory();
    if (!isSaveAbsolutePath && path.startsWith(hdfsWorkingDir)) {
        return path.substring(hdfsWorkingDir.length());
    }
    return path;
}

Example 7

Source File: CubingJob.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private String getCuboidRootPath(CubeSegment seg, KylinConfig kylinConfig) {
    String rootDir = kylinConfig.getHdfsWorkingDirectory();
    if (!rootDir.endsWith("/")) {
        rootDir = rootDir + "/";
    }
    String jobID = this.getId();
    return rootDir + "kylin-" + jobID + "/" + seg.getRealization().getName() + "/cuboid/";
}

Example 8

Source File: DeployCoprocessorCLI.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private static Path getCoprocessorHDFSDir(FileSystem fileSystem, KylinConfig config) throws IOException {
    String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
    hdfsWorkingDirectory = HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory);
    Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
    fileSystem.mkdirs(coprocessorDir);
    return coprocessorDir;
}

Example 9

Source File: NGlobalDictionaryV2Test.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private void roundTest(int size) throws IOException {
    System.out.println("GlobalDictionaryV2Test -> roundTest -> " + System.currentTimeMillis());
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    NGlobalDictionaryV2 dict1 = new NGlobalDictionaryV2("t1", "a", "spark", config.getHdfsWorkingDirectory());
    NGlobalDictionaryV2 dict2 = new NGlobalDictionaryV2("t2", "a", "local", config.getHdfsWorkingDirectory());
    List<String> stringList = generateRandomData(size);
    Collections.sort(stringList);
    runWithSparkBuildGlobalDict(dict1, stringList);
    runWithLocalBuildGlobalDict(dict2, stringList);
    compareTwoVersionDict(dict1, dict2);
    compareTwoModeVersionNum(dict1, dict2);
}

Example 10

Source File: AppendTrieDictionary.java From kylin with Apache License 2.0

5 votes

/**
 * JIRA: https://issues.apache.org/jira/browse/KYLIN-2945
 * if pass a absolute path, it may produce some problems like cannot find global dict after migration.
 * so convert to relative path can avoid it and be better to maintain flexibility.
 *
 */
private String convertToRelativePath(String path) {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    String hdfsWorkingDir = kylinConfig.getHdfsWorkingDirectory();
    if (!isSaveAbsolutePath && path.startsWith(hdfsWorkingDir)) {
        return path.substring(hdfsWorkingDir.length());
    }
    return path;
}

Example 11

Source File: CubingJob.java From kylin with Apache License 2.0

5 votes

private String getCuboidRootPath(CubeSegment seg, KylinConfig kylinConfig) {
    String rootDir = kylinConfig.getHdfsWorkingDirectory();
    if (!rootDir.endsWith("/")) {
        rootDir = rootDir + "/";
    }
    String jobID = this.getId();
    return rootDir + "kylin-" + jobID + "/" + seg.getRealization().getName() + "/cuboid/";
}

Example 12

Source File: HiveInputBase.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public BaseBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    this.flatDesc = flatDesc;
    this.flatTableDatabase = config.getHiveDatabaseForIntermediateTable();
    this.hdfsWorkingDir = config.getHdfsWorkingDirectory();
}

Example 13

Source File: PathManager.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

public static String getParquetStoragePath(KylinConfig config, String cubeName, String segName, String identifier, String cuboidId) {
    CubeInstance cube = CubeManager.getInstance(config).getCube(cubeName);
    String hdfsWorkDir = config.getHdfsWorkingDirectory(cube.getProject());
    return hdfsWorkDir + "parquet" + File.separator + cubeName + File.separator + segName + "_" + identifier + File.separator + cuboidId;
}

Example 14

Source File: HiveInputBase.java From kylin with Apache License 2.0

4 votes

public BaseBatchCubingInputSide(IJoinedFlatTableDesc flatDesc) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    this.flatDesc = flatDesc;
    this.flatTableDatabase = config.getHiveDatabaseForIntermediateTable();
    this.hdfsWorkingDir = config.getHdfsWorkingDirectory();
}

Example 15

Source File: DeployCoprocessorCLI.java From kylin with Apache License 2.0

4 votes

private static String getHDFSWorkingDirectory(KylinConfig config) {
    String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
    return HBaseConnection.makeQualifiedPathInHBaseCluster(hdfsWorkingDirectory);
}

Example 16

Source File: DeployCoprocessorCLI.java From Kylin with Apache License 2.0

4 votes

private static Path getCoprocessorHDFSDir(FileSystem fileSystem, KylinConfig config) throws IOException {
    String hdfsWorkingDirectory = config.getHdfsWorkingDirectory();
    Path coprocessorDir = new Path(hdfsWorkingDirectory, "coprocessor");
    fileSystem.mkdirs(coprocessorDir);
    return coprocessorDir;
}