org.apache.kylin.common.persistence.ResourceTool Java Examples

The following examples show how to use org.apache.kylin.common.persistence.ResourceTool. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: SandboxMetastoreCLI.java From kylin with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    System.setProperty("HADOOP_USER_NAME", "root");
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }

    if (args.length < 1) {
        printUsage();
        return;
    }

    if ("reset".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "reset" });
    } else if ("download".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "download", args[1] });
    } else if ("fetch".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "fetch", args[1], args[2] });
    } else if ("upload".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "upload", args[1] });
    } else {
        printUsage();
    }
}

Example #2

Source File: JobRelatedMetaUtil.java From kylin with Apache License 2.0

6 votes

public static void dumpAndUploadKylinPropsAndMetadata(Set<String> dumpList, KylinConfigExt kylinConfig, String metadataUrl)
        throws IOException {

    try (AutoDeleteDirectory tmpDir = new AutoDeleteDirectory("kylin_job_meta", "");
         AutoDeleteDirectory metaDir = tmpDir.child("meta")) {
        // dump metadata
        JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir.getFile(), dumpList);

        // dump metadata
        dumpResources(kylinConfig, metaDir.getFile(), dumpList);

        // write kylin.properties
        Properties props = kylinConfig.exportToProperties();
        props.setProperty("kylin.metadata.url", metadataUrl);
        File kylinPropsFile = new File(metaDir.getFile(), "kylin.properties");
        try (FileOutputStream os = new FileOutputStream(kylinPropsFile)) {
            props.store(os, kylinPropsFile.getAbsolutePath());
        }

        KylinConfig dstConfig = KylinConfig.createKylinConfig(props);
        //upload metadata
        new ResourceTool().copy(KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()), dstConfig);
    }
}

Example #3

Source File: JobRelatedMetaUtil.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public static void dumpAndUploadKylinPropsAndMetadata(Set<String> dumpList, KylinConfigExt kylinConfig, String metadataUrl)
        throws IOException {

    try (AutoDeleteDirectory tmpDir = new AutoDeleteDirectory("kylin_job_meta", "");
         AutoDeleteDirectory metaDir = tmpDir.child("meta")) {
        // dump metadata
        JobRelatedMetaUtil.dumpResources(kylinConfig, metaDir.getFile(), dumpList);

        // dump metadata
        dumpResources(kylinConfig, metaDir.getFile(), dumpList);

        // write kylin.properties
        Properties props = kylinConfig.exportToProperties();
        props.setProperty("kylin.metadata.url", metadataUrl);
        File kylinPropsFile = new File(metaDir.getFile(), "kylin.properties");
        try (FileOutputStream os = new FileOutputStream(kylinPropsFile)) {
            props.store(os, kylinPropsFile.getAbsolutePath());
        }

        KylinConfig dstConfig = KylinConfig.createKylinConfig(props);
        //upload metadata
        new ResourceTool().copy(KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()), dstConfig);
    }
}

Example #4

Source File: SandboxMetastoreCLI.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    System.setProperty("HADOOP_USER_NAME", "root");
    logger.info("Adding to classpath: " + new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    ClassUtil.addClasspath(new File(HBaseMetadataTestCase.SANDBOX_TEST_DATA).getAbsolutePath());
    System.setProperty(KylinConfig.KYLIN_CONF, HBaseMetadataTestCase.SANDBOX_TEST_DATA);
    if (StringUtils.isEmpty(System.getProperty("hdp.version"))) {
        throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.4.0.0-169");
    }

    if (args.length < 1) {
        printUsage();
        return;
    }

    if ("reset".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "reset" });
    } else if ("download".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "download", args[1] });
    } else if ("fetch".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "fetch", args[1], args[2] });
    } else if ("upload".equalsIgnoreCase(args[0])) {
        ResourceTool.main(new String[] { "upload", args[1] });
    } else {
        printUsage();
    }
}

Example #5

Source File: MetaDumpUtil.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public static void dumpAndUploadKylinPropsAndMetadata(Set<String> dumpList, KylinConfigExt kylinConfig,
        String metadataUrl) throws IOException {

    try (AutoDeleteDirectory tmpDir = new AutoDeleteDirectory("kylin_job_meta", "");
            AutoDeleteDirectory metaDir = tmpDir.child("meta")) {
        // dump metadata
        dumpResources(kylinConfig, metaDir.getFile().getAbsolutePath(), dumpList);

        // write kylin.properties
        Properties props = kylinConfig.exportToProperties();
        props.setProperty("kylin.metadata.url", metadataUrl);
        File kylinPropsFile = new File(metaDir.getFile(), "kylin.properties");
        try (FileOutputStream os = new FileOutputStream(kylinPropsFile)) {
            props.store(os, kylinPropsFile.getAbsolutePath());
        }

        KylinConfig dstConfig = KylinConfig.createKylinConfig(props);
        //upload metadata
        new ResourceTool().copy(KylinConfig.createInstanceFromUri(metaDir.getAbsolutePath()), dstConfig);
    }
}

Example #6

Source File: CubeMetaIngester.java From kylin with Apache License 2.0

5 votes

private void injest(File metaRoot) throws IOException {
    KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath());
    TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig);
    DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig);
    HybridManager srcHybridManager = HybridManager.getInstance(srcConfig);
    CubeManager srcCubeManager = CubeManager.getInstance(srcConfig);
    CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig);

    checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager);
    new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources));

    // clear the cache
    Broadcaster.getInstance(kylinConfig).notifyClearAll();
    
    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        logger.info("add " + tableDesc + " to " + targetProjectName);
        projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName);
    }

    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        logger.info("add " + cube + " to " + targetProjectName);
        projectManager.addModelToProject(cube.getModel().getName(), targetProjectName);
        projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null);
    }

}

Example #7

Source File: DeployUtil.java From Kylin with Apache License 2.0

5 votes

public static void deployMetadata() throws IOException {
    // install metadata to hbase
    ResourceTool.reset(config());
    ResourceTool.copy(KylinConfig.createInstanceFromUri(AbstractKylinTestCase.LOCALMETA_TEST_DATA), config());

    // update cube desc signature.
    for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) {
        cube.getDescriptor().setSignature(cube.getDescriptor().calculateSignature());
        CubeManager.getInstance(config()).updateCube(cube);
    }
}

Example #8

Source File: ITJDBCResourceStoreTest.java From kylin with Apache License 2.0

5 votes

@Before
public void setup() throws Exception {
    this.createTestMetadata();
    kylinConfig = KylinConfig.getInstanceFromEnv();
    KylinConfig configBackup = KylinConfig.createKylinConfig(kylinConfig);
    Statement statement = null;
    Connection conn = null;
    metadataUrlBackup = kylinConfig.getMetadataUrl();
    kylinConfig.setMetadataUrl(mainIdentifier + jdbcMetadataUrlNoIdentifier);
    JDBCSqlQueryFormat sqlQueryFormat = JDBCSqlQueryFormatProvider
            .createJDBCSqlQueriesFormat(KylinConfig.getInstanceFromEnv().getMetadataDialect());
    try {
        connectionManager = JDBCConnectionManager.getConnectionManager();
        conn = connectionManager.getConn();
        statement = conn.createStatement();
        String sql = new MessageFormat(sqlQueryFormat.getTestDropSql(), Locale.ROOT)
                .format(mainIdentifier, new StringBuffer(), new FieldPosition(0)).toString();
        statement.executeUpdate(sql);
        sql = new MessageFormat(sqlQueryFormat.getTestDropSql(), Locale.ROOT)
                .format(copyIdentifier, new StringBuffer(), new FieldPosition(0)).toString();
        statement.executeUpdate(sql);
        jdbcConnectable = true;
        new ResourceTool().copy(configBackup, kylinConfig);
    } catch (RuntimeException ex) {
        logger.info("Init connection manager failed, skip test cases");
    } finally {
        JDBCConnectionManager.closeQuietly(statement);
        JDBCConnectionManager.closeQuietly(conn);
    }
}

Example #9

Source File: DeployUtil.java From kylin with Apache License 2.0

5 votes

public static void deployMetadata(String localMetaData) throws IOException {
    // install metadata to hbase
    new ResourceTool().reset(config());
    new ResourceTool().copy(KylinConfig.createInstanceFromUri(localMetaData), config());

    // update cube desc signature.
    for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) {
        CubeDescManager.getInstance(config()).updateCubeDesc(cube.getDescriptor());//enforce signature updating
    }
}

Example #10

Source File: JobDiagnosisInfoCLI.java From kylin with Apache License 2.0

5 votes

private void extractResources(File destDir) {
    logger.info("The resource paths going to be extracted:");
    for (String s : requiredResources) {
        logger.info(s + "(required)");
    }

    try {
        KylinConfig srcConfig = KylinConfig.getInstanceFromEnv();
        KylinConfig dstConfig = KylinConfig.createInstanceFromUri(destDir.getAbsolutePath());
        new ResourceTool().copy(srcConfig, dstConfig, requiredResources);
    } catch (Exception e) {
        throw new RuntimeException("Failed to extract job resources. ", e);
    }
}

Example #11

Source File: CubeMetaIngester.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private void injest(File metaRoot) throws IOException {
    KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath());
    TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig);
    DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig);
    HybridManager srcHybridManager = HybridManager.getInstance(srcConfig);
    CubeManager srcCubeManager = CubeManager.getInstance(srcConfig);
    CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig);

    checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager);
    new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources));

    // clear the cache
    Broadcaster.getInstance(kylinConfig).notifyClearAll();
    
    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        logger.info("add " + tableDesc + " to " + targetProjectName);
        projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName);
    }

    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        logger.info("add " + cube + " to " + targetProjectName);
        projectManager.addModelToProject(cube.getModel().getName(), targetProjectName);
        projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null);
    }

}

Example #12

Source File: ITJDBCResourceStoreTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Before
public void setup() throws Exception {
    this.createTestMetadata();
    kylinConfig = KylinConfig.getInstanceFromEnv();
    KylinConfig configBackup = KylinConfig.createKylinConfig(kylinConfig);
    Statement statement = null;
    Connection conn = null;
    metadataUrlBackup = kylinConfig.getMetadataUrl();
    kylinConfig.setMetadataUrl(mainIdentifier + jdbcMetadataUrlNoIdentifier);
    JDBCSqlQueryFormat sqlQueryFormat = JDBCSqlQueryFormatProvider
            .createJDBCSqlQueriesFormat(KylinConfig.getInstanceFromEnv().getMetadataDialect());
    try {
        connectionManager = JDBCConnectionManager.getConnectionManager();
        conn = connectionManager.getConn();
        statement = conn.createStatement();
        String sql = new MessageFormat(sqlQueryFormat.getTestDropSql(), Locale.ROOT)
                .format(mainIdentifier, new StringBuffer(), new FieldPosition(0)).toString();
        statement.executeUpdate(sql);
        sql = new MessageFormat(sqlQueryFormat.getTestDropSql(), Locale.ROOT)
                .format(copyIdentifier, new StringBuffer(), new FieldPosition(0)).toString();
        statement.executeUpdate(sql);
        jdbcConnectable = true;
        new ResourceTool().copy(configBackup, kylinConfig);
    } catch (RuntimeException ex) {
        logger.info("Init connection manager failed, skip test cases");
    } finally {
        JDBCConnectionManager.closeQuietly(statement);
        JDBCConnectionManager.closeQuietly(conn);
    }
}

Example #13

Source File: DeployUtil.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

public static void deployMetadata(String localMetaData) throws IOException {
    // install metadata to hbase
    new ResourceTool().reset(config());
    new ResourceTool().copy(KylinConfig.createInstanceFromUri(localMetaData), config());

    // update cube desc signature.
    for (CubeInstance cube : CubeManager.getInstance(config()).listAllCubes()) {
        CubeDescManager.getInstance(config()).updateCubeDesc(cube.getDescriptor());//enforce signature updating
    }
}

Example #14

Source File: JobDiagnosisInfoCLI.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private void extractResources(File destDir) {
    logger.info("The resource paths going to be extracted:");
    for (String s : requiredResources) {
        logger.info(s + "(required)");
    }

    try {
        KylinConfig srcConfig = KylinConfig.getInstanceFromEnv();
        KylinConfig dstConfig = KylinConfig.createInstanceFromUri(destDir.getAbsolutePath());
        new ResourceTool().copy(srcConfig, dstConfig, requiredResources);
    } catch (Exception e) {
        throw new RuntimeException("Failed to extract job resources. ", e);
    }
}

Example #15

Source File: CubeMetaExtractorTest.java From kylin with Apache License 2.0

4 votes

@Test
public void testExtractorByPrj() throws IOException {
    folder.create();
    File tempDir = folder.getRoot();
    String tempDirAbsPath = tempDir.getAbsolutePath();
    List<String> args = new ArrayList<>();
    args.add("-destDir");
    args.add(tempDirAbsPath);
    args.add("-project");
    args.add("default");
    args.add("-compress");
    args.add("false");
    args.add("-packagetype");
    args.add("cubemeta");
    String[] cubeMetaArgs = new String[args.size()];
    args.toArray(cubeMetaArgs);

    CubeMetaExtractor cubeMetaExtractor = new CubeMetaExtractor();
    cubeMetaExtractor.execute(cubeMetaArgs);

    File[] files = tempDir.listFiles();
    Preconditions.checkState(files.length == 1);
    String dumpDir = files[0].getAbsolutePath();
    KylinConfig instanceFromUri = KylinConfig.createInstanceFromUri(dumpDir);
    NavigableSet<String> tables = new ResourceTool().list(instanceFromUri, "table");
    NavigableSet<String> tableExds = new ResourceTool().list(instanceFromUri, "table_exd");
    Set<String> expectTbl = Sets.newHashSet(
            "/table/DEFAULT.FIFTY_DIM.json", //
            "/table/DEFAULT.STREAMING_TABLE.json", //
            "/table/DEFAULT.STREAMING_CATEGORY.json", //
            "/table/DEFAULT.TEST_ACCOUNT.json", //
            "/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json", //
            "/table/DEFAULT.TEST_COUNTRY.json", //
            "/table/DEFAULT.TEST_KYLIN_FACT.json", //
            "/table/DEFAULT.TEST_ORDER.json", //
            "/table/EDW.TEST_CAL_DT.json", //
            "/table/EDW.TEST_SELLER_TYPE_DIM.json", //
            "/table/EDW.TEST_SITES.json", //
            "/table/SSB.CUSTOMER.json", //
            "/table/SSB.DATES.json", //
            "/table/SSB.PART.json", //
            "/table/SSB.SUPPLIER.json", //
            "/table/SSB.V_LINEORDER.json", //
            "/table/DEFAULT.STREAMING_V2_TABLE.json", //
            "/table/DEFAULT.STREAMING_V2_USER_INFO_TABLE.json"
    );
    Set<String> expectTblExd = Sets.newHashSet(
            "/table_exd/DEFAULT.TEST_COUNTRY.json", //
            "/table_exd/DEFAULT.TEST_KYLIN_FACT--default.json"); //
    Assert.assertEquals(expectTbl, tables);
    Assert.assertEquals(expectTblExd, tableExds);
}

Example #16

Source File: ITJDBCResourceStoreTest.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Test
public void testPerformanceWithResourceTool() throws Exception {
    Assume.assumeTrue(jdbcConnectable);
    KylinConfig tmpConfig = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
    tmpConfig.setMetadataUrl(copyIdentifier + jdbcMetadataUrlNoIdentifier);

    JDBCResourceStore store = (JDBCResourceStore) ResourceStore.getStore(kylinConfig);
    NavigableSet<String> executes = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
    NavigableSet<String> executeOutputs = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);

    long startTs = System.currentTimeMillis();

    for (String execute : executes) {
        String uuid = StringUtils.substringAfterLast(execute, "/");
        RawResource executeResource = store.getResource(execute);
        Map<String, RawResource> executeOutputResourceMap = new HashMap<>();

        for (String executeOutput : executeOutputs) {
            if (executeOutput.contains(uuid)) {
                RawResource executeOutputResource = store.getResource(executeOutput);
                executeOutputResourceMap.put(executeOutput, executeOutputResource);
            }
        }

        for (int i = 0; i < 200; i++) {
            String newUuid = UUID.randomUUID().toString();
            store.putResource(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + newUuid, executeResource.content(),
                    System.currentTimeMillis());

            for (String key : executeOutputResourceMap.keySet()) {
                String step = StringUtils.substringAfterLast(key, uuid);
                store.putResource(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + newUuid + step,
                        executeOutputResourceMap.get(key).content(), System.currentTimeMillis());
            }
        }
    }

    long queryNumBeforeCopy = store.getQueriedSqlNum();
    new ResourceTool().copy(kylinConfig, tmpConfig);
    long endTs = System.currentTimeMillis();
    long queryNumAfterCopy = store.getQueriedSqlNum();
    JDBCResourceStore resourceStoreCopy = (JDBCResourceStore) ResourceStore.getStore(tmpConfig);

    int executeNum = store.listResources("/execute").size();
    int executeOutputNum = store.listResources("/execute_output").size();

    assertEquals(executeNum, resourceStoreCopy.listResources("/execute").size());
    assertEquals(executeOutputNum, resourceStoreCopy.listResources("/execute_output").size());

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    String startTime = sdf.format(new Date(Long.parseLong(String.valueOf(startTs))));
    String endTime = sdf.format(new Date(Long.parseLong(String.valueOf(endTs))));

    logger.info("Test performance with ResourceTool done during " + startTime + " to " + endTime);
    logger.info("Now there is " + executeNum + " execute data and " + executeOutputNum
            + " execute_output data in resource store.");
    logger.info("Resource store run " + queryNumBeforeCopy + " sqls for metadata generation, and "
            + (queryNumAfterCopy - queryNumBeforeCopy) + " sqls for copy with ResourceTool.");
    assertTrue((queryNumAfterCopy - queryNumBeforeCopy) < queryNumBeforeCopy);
    logger.info("This test is expected to be done in 10 mins.");
    assertTrue((endTs - startTs) < 600000);
}

Example #17

Source File: ITJDBCResourceStoreTest.java From kylin with Apache License 2.0

4 votes

@Test
public void testPerformanceWithResourceTool() throws Exception {
    Assume.assumeTrue(jdbcConnectable);
    KylinConfig tmpConfig = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
    tmpConfig.setMetadataUrl(copyIdentifier + jdbcMetadataUrlNoIdentifier);

    JDBCResourceStore store = (JDBCResourceStore) ResourceStore.getStore(kylinConfig);
    NavigableSet<String> executes = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
    NavigableSet<String> executeOutputs = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);

    long startTs = System.currentTimeMillis();

    for (String execute : executes) {
        String uuid = StringUtils.substringAfterLast(execute, "/");
        RawResource executeResource = store.getResource(execute);
        Map<String, RawResource> executeOutputResourceMap = new HashMap<>();

        for (String executeOutput : executeOutputs) {
            if (executeOutput.contains(uuid)) {
                RawResource executeOutputResource = store.getResource(executeOutput);
                executeOutputResourceMap.put(executeOutput, executeOutputResource);
            }
        }

        for (int i = 0; i < 200; i++) {
            String newUuid = UUID.randomUUID().toString();
            store.putResource(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + newUuid, executeResource.content(),
                    System.currentTimeMillis());

            for (String key : executeOutputResourceMap.keySet()) {
                String step = StringUtils.substringAfterLast(key, uuid);
                store.putResource(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + newUuid + step,
                        executeOutputResourceMap.get(key).content(), System.currentTimeMillis());
            }
        }
    }

    long queryNumBeforeCopy = store.getQueriedSqlNum();
    new ResourceTool().copy(kylinConfig, tmpConfig);
    long endTs = System.currentTimeMillis();
    long queryNumAfterCopy = store.getQueriedSqlNum();
    JDBCResourceStore resourceStoreCopy = (JDBCResourceStore) ResourceStore.getStore(tmpConfig);

    int executeNum = store.listResources("/execute").size();
    int executeOutputNum = store.listResources("/execute_output").size();

    assertEquals(executeNum, resourceStoreCopy.listResources("/execute").size());
    assertEquals(executeOutputNum, resourceStoreCopy.listResources("/execute_output").size());

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    String startTime = sdf.format(new Date(Long.parseLong(String.valueOf(startTs))));
    String endTime = sdf.format(new Date(Long.parseLong(String.valueOf(endTs))));

    logger.info("Test performance with ResourceTool done during " + startTime + " to " + endTime);
    logger.info("Now there is " + executeNum + " execute data and " + executeOutputNum
            + " execute_output data in resource store.");
    logger.info("Resource store run " + queryNumBeforeCopy + " sqls for metadata generation, and "
            + (queryNumAfterCopy - queryNumBeforeCopy) + " sqls for copy with ResourceTool.");
    assertTrue((queryNumAfterCopy - queryNumBeforeCopy) < queryNumBeforeCopy);
    logger.info("This test is expected to be done in 10 mins.");
    assertTrue((endTs - startTs) < 600000);
}

Example #18

Source File: CubeMetaExtractorTest.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Test
public void testExtractorByPrj() throws IOException {
    folder.create();
    File tempDir = folder.getRoot();
    String tempDirAbsPath = tempDir.getAbsolutePath();
    List<String> args = new ArrayList<>();
    args.add("-destDir");
    args.add(tempDirAbsPath);
    args.add("-project");
    args.add("default");
    args.add("-compress");
    args.add("false");
    args.add("-packagetype");
    args.add("cubemeta");
    String[] cubeMetaArgs = new String[args.size()];
    args.toArray(cubeMetaArgs);

    CubeMetaExtractor cubeMetaExtractor = new CubeMetaExtractor();
    cubeMetaExtractor.execute(cubeMetaArgs);

    File[] files = tempDir.listFiles();
    Preconditions.checkState(files.length == 1);
    String dumpDir = files[0].getAbsolutePath();
    KylinConfig instanceFromUri = KylinConfig.createInstanceFromUri(dumpDir);
    NavigableSet<String> tables = new ResourceTool().list(instanceFromUri, "table");
    NavigableSet<String> tableExds = new ResourceTool().list(instanceFromUri, "table_exd");
    Set<String> expectTbl = Sets.newHashSet(
            "/table/DEFAULT.FIFTY_DIM.json", //
            "/table/DEFAULT.STREAMING_TABLE.json", //
            "/table/DEFAULT.STREAMING_CATEGORY.json", //
            "/table/DEFAULT.TEST_ACCOUNT.json", //
            "/table/DEFAULT.TEST_CATEGORY_GROUPINGS.json", //
            "/table/DEFAULT.TEST_COUNTRY.json", //
            "/table/DEFAULT.TEST_KYLIN_FACT.json", //
            "/table/DEFAULT.TEST_ORDER.json", //
            "/table/EDW.TEST_CAL_DT.json", //
            "/table/EDW.TEST_SELLER_TYPE_DIM.json", //
            "/table/EDW.TEST_SITES.json", //
            "/table/SSB.CUSTOMER.json", //
            "/table/SSB.DATES.json", //
            "/table/SSB.PART.json", //
            "/table/SSB.SUPPLIER.json", //
            "/table/SSB.V_LINEORDER.json", //
            "/table/DEFAULT.STREAMING_V2_TABLE.json", //
            "/table/DEFAULT.STREAMING_V2_USER_INFO_TABLE.json"
    );
    Set<String> expectTblExd = Sets.newHashSet(
            "/table_exd/DEFAULT.TEST_COUNTRY.json", //
            "/table_exd/DEFAULT.TEST_KYLIN_FACT--default.json"); //
    Assert.assertEquals(expectTbl, tables);
    Assert.assertEquals(expectTblExd, tableExds);
}