Java Code Examples for org.apache.kylin.common.util.HBaseMetadataTestCase

The following examples show how to use org.apache.kylin.common.util.HBaseMetadataTestCase. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: kylin-on-parquet-v2   Source File: KylinTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected static void setupAll() throws Exception {
    //setup env
    HBaseMetadataTestCase.staticCreateTestMetadata();
    config = KylinConfig.getInstanceFromEnv();

    //setup cube conn
    String project = ProjectInstance.DEFAULT_PROJECT_NAME;
    cubeConnection = QueryConnection.getConnection(project);

    //setup h2
    h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++) + ";CACHE_SIZE=32072", "sa",
            "");
    // Load H2 Tables (inner join)
    H2Database h2DB = new H2Database(h2Connection, config, project);
    h2DB.loadAllTables();
}
 
Example 2
Source Project: kylin-on-parquet-v2   Source File: SandboxMetastoreCLI.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    System.setProperty("HADOOP_USER_NAME", "root");
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }

    if (args.length < 1) {
        printUsage();
        return;
    }

    if ("reset".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "reset" });
    } else if ("download".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "download", args[1] });
    } else if ("fetch".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "fetch", args[1], args[2] });
    } else if ("upload".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "upload", args[1] });
    } else {
        printUsage();
    }
}
 
Example 3
Source Project: kylin   Source File: KylinTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected static void setupAll() throws Exception {
    //setup env
    HBaseMetadataTestCase.staticCreateTestMetadata();
    config = KylinConfig.getInstanceFromEnv();

    //setup cube conn
    String project = ProjectInstance.DEFAULT_PROJECT_NAME;
    cubeConnection = QueryConnection.getConnection(project);

    //setup h2
    h2Connection = DriverManager.getConnection("jdbc:h2:mem:db" + (h2InstanceCount++) + ";CACHE_SIZE=32072", "sa",
            "");
    // Load H2 Tables (inner join)
    H2Database h2DB = new H2Database(h2Connection, config, project);
    h2DB.loadAllTables();
}
 
Example 4
Source Project: kylin   Source File: SandboxMetastoreCLI.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    System.setProperty("HADOOP_USER_NAME", "root");
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }

    if (args.length < 1) {
        printUsage();
        return;
    }

    if ("reset".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "reset" });
    } else if ("download".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "download", args[1] });
    } else if ("fetch".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "fetch", args[1], args[2] });
    } else if ("upload".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "upload", args[1] });
    } else {
        printUsage();
    }
}
 
Example 5
Source Project: Kylin   Source File: BuildCubeWithEngineTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();


    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(kylinConfig));
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    cubeManager = CubeManager.getInstance(kylinConfig);
    jobEngineConfig = new JobEngineConfig(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        if(jobService.getJob(jobId) instanceof CubingJob){
            jobService.deleteJob(jobId);
        }
    }

}
 
Example 6
@BeforeClass
public static void beforeClass() throws Exception {
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }
}
 
Example 7
@Before
public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();

}
 
Example 8
Source Project: kylin-on-parquet-v2   Source File: BuildCubeWithStream.java    License: Apache License 2.0 5 votes vote down vote up
public static void beforeClass() throws Exception {
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException(
                "No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }
    HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
 
Example 9
Source Project: kylin-on-parquet-v2   Source File: KylinTestBase.java    License: Apache License 2.0 5 votes vote down vote up
protected static void clean() {
    if (cubeConnection != null)
        closeConnection(cubeConnection);
    if (h2Connection != null)
        closeConnection(h2Connection);

    HBaseMetadataTestCase.staticCleanupTestMetadata();
    RemoveBlackoutRealizationsRule.blackList.clear();

}
 
Example 10
Source Project: kylin   Source File: DeployLocalMetaToRemoteTest.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }
}
 
Example 11
Source Project: kylin   Source File: DeployLocalMetaToRemoteTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();

}
 
Example 12
Source Project: kylin   Source File: BuildCubeWithStream.java    License: Apache License 2.0 5 votes vote down vote up
public static void beforeClass() throws Exception {
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException(
                "No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }
    HBaseMetadataTestCase.staticCreateTestMetadata(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
 
Example 13
Source Project: kylin   Source File: KylinTestBase.java    License: Apache License 2.0 5 votes vote down vote up
protected static void clean() {
    if (cubeConnection != null)
        closeConnection(cubeConnection);
    if (h2Connection != null)
        closeConnection(h2Connection);

    HBaseMetadataTestCase.staticCleanupTestMetadata();
    RemoveBlackoutRealizationsRule.blackList.clear();

}
 
Example 14
Source Project: Kylin   Source File: BuildCubeWithEngineTest.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, "../examples/test_case_data/sandbox");
    System.setProperty("hdp.version", "2.2.0.0-2041"); // mapred-site.xml ref this
}
 
Example 15
Source Project: Kylin   Source File: BuildIIWithEngineTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void before() throws Exception {
    HBaseMetadataTestCase.staticCreateTestMetadata(AbstractKylinTestCase.SANDBOX_TEST_DATA);

    DeployUtil.initCliWorkDir();
    //        DeployUtil.deployMetadata();
    DeployUtil.overrideJobJarLocations();

    final KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    jobService = ExecutableManager.getInstance(kylinConfig);
    scheduler = DefaultScheduler.getInstance();
    scheduler.init(new JobEngineConfig(kylinConfig));
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
    iiManager = IIManager.getInstance(kylinConfig);
    jobEngineConfig = new JobEngineConfig(kylinConfig);
    for (String jobId : jobService.getAllJobIds()) {
        if(jobService.getJob(jobId) instanceof IIJob){
            jobService.deleteJob(jobId);
        }
    }

    IIInstance ii = iiManager.getII(TEST_II_NAME);
    if (ii.getStatus() != RealizationStatusEnum.DISABLED) {
        ii.setStatus(RealizationStatusEnum.DISABLED);
        iiManager.updateII(ii);
    }
}
 
Example 16
Source Project: Kylin   Source File: KylinQueryTest.java    License: Apache License 2.0 5 votes vote down vote up
protected static void clean() {
    if (cubeConnection != null)
        closeConnection(cubeConnection);
    if (h2Connection != null)
        closeConnection(h2Connection);

    ObserverEnabler.forceCoprocessorUnset();
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
 
Example 17
@After
public void after() {
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
 
Example 18
Source Project: kylin-on-parquet-v2   Source File: BuildCubeWithStream.java    License: Apache License 2.0 4 votes vote down vote up
public void cleanup() throws Exception {
    cleanupOldStorage();
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
 
Example 19
Source Project: kylin-on-parquet-v2   Source File: BuildCubeWithEngine.java    License: Apache License 2.0 4 votes vote down vote up
public static void beforeClass() throws Exception {
    beforeClass(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
 
Example 20
Source Project: kylin-on-parquet-v2   Source File: BuildCubeWithEngine.java    License: Apache License 2.0 4 votes vote down vote up
public static void beforeClass(String confDir) throws Exception {
    logger.info("Adding to classpath: " + new File(confDir).getAbsolutePath());
    ClassUtil.addClasspath(new File(confDir).getAbsolutePath());

    fastBuildMode = isFastBuildMode();
    simpleBuildMode = isSimpleBuildMode();
    if (fastBuildMode) {
        logger.info("Will use fast build mode");
    }
    if (simpleBuildMode) {
        logger.info("Will use simple build mode");
    }

    String specifiedEngineType = System.getProperty("engineType");
    if (StringUtils.isNotEmpty(specifiedEngineType)) {
        engineType = Integer.parseInt(specifiedEngineType);
    }

    System.setProperty(KylinConfig.KYLIN_CONF, confDir);
    System.setProperty("SPARK_HOME", "/usr/local/spark"); // need manually create and put spark to this folder on Jenkins
    System.setProperty("kylin.hadoop.conf.dir", confDir);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException(
                "No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }

    HBaseMetadataTestCase.staticCreateTestMetadata(confDir);

    try {
        //check hdfs permission
        FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
        String hdfsWorkingDirectory = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
        Path coprocessorDir = new Path(hdfsWorkingDirectory);
        boolean success = fileSystem.mkdirs(coprocessorDir);
        if (!success) {
            throw new IOException("mkdir fails");
        }
    } catch (IOException e) {
        throw new RuntimeException(
                "failed to create kylin.env.hdfs-working-dir, Please make sure the user has right to access "
                        + KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(),
                e);
    }
}
 
Example 21
Source Project: kylin-on-parquet-v2   Source File: BuildCubeWithEngine.java    License: Apache License 2.0 4 votes vote down vote up
public static void afterClass() {
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
 
Example 22
Source Project: kylin-on-parquet-v2   Source File: BuildCubeWithStreamV2.java    License: Apache License 2.0 4 votes vote down vote up
public static void beforeClass() throws Exception {
    beforeClass(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
 
Example 23
Source Project: kylin-on-parquet-v2   Source File: BuildCubeWithStreamV2.java    License: Apache License 2.0 4 votes vote down vote up
public static void afterClass() {
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
 
Example 24
Source Project: kylin-on-parquet-v2   Source File: BuildCubeWithStreamV2.java    License: Apache License 2.0 4 votes vote down vote up
public void cleanup() throws Exception {
    cleanupOldStorage();
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
 
Example 25
@BeforeClass
public static void beforeClass() throws Exception {
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
 
Example 26
Source Project: kylin   Source File: DeployLocalMetaToRemoteTest.java    License: Apache License 2.0 4 votes vote down vote up
@After
public void after() {
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
 
Example 27
Source Project: kylin   Source File: BuildCubeWithStream.java    License: Apache License 2.0 4 votes vote down vote up
public void cleanup() throws Exception {
    cleanupOldStorage();
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}
 
Example 28
Source Project: kylin   Source File: BuildCubeWithEngine.java    License: Apache License 2.0 4 votes vote down vote up
public static void beforeClass() throws Exception {
    beforeClass(HBaseMetadataTestCase.SANDBOX_TEST_DATA);
}
 
Example 29
Source Project: kylin   Source File: BuildCubeWithEngine.java    License: Apache License 2.0 4 votes vote down vote up
public static void beforeClass(String confDir) throws Exception {
    logger.info("Adding to classpath: " + new File(confDir).getAbsolutePath());
    ClassUtil.addClasspath(new File(confDir).getAbsolutePath());

    fastBuildMode = isFastBuildMode();
    simpleBuildMode = isSimpleBuildMode();
    if (fastBuildMode) {
        logger.info("Will use fast build mode");
    }
    if (simpleBuildMode) {
        logger.info("Will use simple build mode");
    }

    String specifiedEngineType = System.getProperty("engineType");
    if (StringUtils.isNotEmpty(specifiedEngineType)) {
        engineType = Integer.parseInt(specifiedEngineType);
    }

    System.setProperty(KylinConfig.KYLIN_CONF, confDir);
    System.setProperty("SPARK_HOME", "/usr/local/spark"); // need manually create and put spark to this folder on Jenkins
    System.setProperty("kylin.hadoop.conf.dir", confDir);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException(
                "No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }

    HBaseMetadataTestCase.staticCreateTestMetadata(confDir);

    try {
        //check hdfs permission
        FileSystem fileSystem = HadoopUtil.getWorkingFileSystem();
        String hdfsWorkingDirectory = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
        Path coprocessorDir = new Path(hdfsWorkingDirectory);
        boolean success = fileSystem.mkdirs(coprocessorDir);
        if (!success) {
            throw new IOException("mkdir fails");
        }
    } catch (IOException e) {
        throw new RuntimeException(
                "failed to create kylin.env.hdfs-working-dir, Please make sure the user has right to access "
                        + KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(),
                e);
    }
}
 
Example 30
Source Project: kylin   Source File: BuildCubeWithEngine.java    License: Apache License 2.0 4 votes vote down vote up
public static void afterClass() {
    HBaseMetadataTestCase.staticCleanupTestMetadata();
}