Java Code Examples for org.apache.kylin.cube.CubeManager#appendSegment()

The following examples show how to use org.apache.kylin.cube.CubeManager#appendSegment() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: HybridCubeCLITest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void testSegmentOverlap() throws IOException {
    thrown.expect(RuntimeException.class);
    thrown.expectMessage("Segments has overlap");

    HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
    Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
    HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });

    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
    CubeInstance cube2 = cubeManager.getCube("ssb_cube2");

    // 2012-01-01,2012-01-03
    cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
    // 2012-01-02,2012-01-04
    cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325462400000L, 1325635200000L));

    HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
}

Example 2

Source File: HybridCubeCLITest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void testSegmentNotOverlap() throws IOException {
    HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
    Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
    HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });

    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
    CubeInstance cube2 = cubeManager.getCube("ssb_cube2");

    // 2012-01-01,2012-01-03
    cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
    // 2012-01-03,2012-01-04
    cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325548800000L, 1325635200000L));

    // not throw exception
    HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
}

Example 3

Source File: JdbcHiveMRInputTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void testGenSqoopCmd_Partition() throws IOException {
    ISource source = SourceManager.getSource(new JdbcSourceAware());
    IMRInput input = source.adaptToBuildEngine(IMRInput.class);
    Assert.assertNotNull(input);

    CubeManager cubeManager = CubeManager.getInstance(getTestConfig());
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_inner_join_cube");
    CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()),
            new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L));
    CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg);
    JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input
            .getBatchCubingInputSide(flatDesc);

    AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp",
            cubeDesc.getName());
    Assert.assertNotNull(executable);

    String cmd = executable.getParam("cmd");
    Assert.assertTrue(cmd.contains("org.h2.Driver"));
    Assert.assertTrue(cmd.contains(
            "--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\")" + System.lineSeparator()
                    + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\""));
    source.close();
}

Example 4

Source File: JdbcHiveMRInputTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void testGenSqoopCmd_NoPartition() throws IOException {
    ISource source = SourceManager.getSource(new JdbcSourceAware());
    IMRInput input = source.adaptToBuildEngine(IMRInput.class);
    Assert.assertNotNull(input);

    CubeManager cubeManager = CubeManager.getInstance(getTestConfig());
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_left_join_cube");
    CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()),
            new SegmentRange.TSRange(0L, Long.MAX_VALUE));
    CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg);
    JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input
            .getBatchCubingInputSide(flatDesc);

    AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp",
            cubeDesc.getName());
    Assert.assertNotNull(executable);
    String cmd = executable.getParam("cmd");
    Assert.assertTrue(cmd.contains("org.h2.Driver"));
    Assert.assertTrue(
            cmd.contains("--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\")" + System.lineSeparator()
                    + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"\""));
    source.close();
}

Example 5

Source File: JdbcHiveMRInputTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void testGenSqoopCmd_WithLookupShardBy() throws IOException {
    ISource source = SourceManager.getSource(new JdbcSourceAware());
    IMRInput input = source.adaptToBuildEngine(IMRInput.class);
    Assert.assertNotNull(input);

    CubeManager cubeManager = CubeManager.getInstance(getTestConfig());
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_jdbc_shard");
    CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()),
            new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L));
    CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg);
    JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input
            .getBatchCubingInputSide(flatDesc);

    AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp",
            cubeDesc.getName());
    Assert.assertNotNull(executable);

    String cmd = executable.getParam("cmd");
    Assert.assertTrue(cmd.contains("org.h2.Driver"));
    Assert.assertTrue(cmd.contains(
            "--boundary-query \"SELECT MIN(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\"), MAX(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\")" + System.lineSeparator()
                    + "FROM \\\"DEFAULT\\\".\\\"TEST_CATEGORY_GROUPINGS\\\" AS \\\"TEST_CATEGORY_GROUPINGS\\\"\""));

    source.close();
}

Example 6

Source File: JobStepFactoryTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

@Test
public void testAddStepInCubing() throws IOException {
    CubeManager cubeMgr = CubeManager.getInstance(config);
    CubeInstance cube = cubeMgr.getCube(CUBE_NAME);
    cleanupSegments(CUBE_NAME);
    CubeSegment oneSeg = cubeMgr.appendSegment(cube, new SegmentRange.TSRange(0L, Long.MAX_VALUE));
    Set<CubeSegment> segments = Sets.newHashSet(oneSeg);
    NSparkCubingJob job = NSparkCubingJob.create(segments, "ADMIN");
    Assert.assertEquals(CUBE_NAME, job.getParam(MetadataConstants.P_CUBE_NAME));

    NSparkExecutable resourceDetectStep = job.getResourceDetectStep();
    Assert.assertEquals(ResourceDetectBeforeCubingJob.class.getName(),
            resourceDetectStep.getSparkSubmitClassName());
    Assert.assertEquals(ExecutableConstants.STEP_NAME_DETECT_RESOURCE, resourceDetectStep.getName());
    job.getParams().forEach((key, value) -> Assert.assertEquals(value, resourceDetectStep.getParam(key)));
    Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), resourceDetectStep.getId()).toString(),
            resourceDetectStep.getDistMetaUrl());

    NSparkExecutable cubeStep = job.getSparkCubingStep();
    Assert.assertEquals(config.getSparkBuildClassName(), cubeStep.getSparkSubmitClassName());
    Assert.assertEquals(ExecutableConstants.STEP_NAME_BUILD_SPARK_CUBE, cubeStep.getName());
    job.getParams().forEach((key, value) -> Assert.assertEquals(value, cubeStep.getParam(key)));
    Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), cubeStep.getId()).toString(),
            cubeStep.getDistMetaUrl());
}

Example 7

Source File: LocalWithSparkSessionTest.java From kylin-on-parquet-v2 with Apache License 2.0

6 votes

public ExecutableState buildCuboid(String cubeName, SegmentRange.TSRange tsRange) throws Exception {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    CubeManager cubeMgr = CubeManager.getInstance(config);
    CubeInstance cube = cubeMgr.getCube(cubeName);
    ExecutableManager execMgr = ExecutableManager.getInstance(config);
    DataModelManager.getInstance(config).getModels();
    // ready cube, segment, cuboid layout
    CubeSegment oneSeg = cubeMgr.appendSegment(cube, tsRange);
    NSparkCubingJob job = NSparkCubingJob.create(Sets.newHashSet(oneSeg), "ADMIN");
    NSparkCubingStep sparkStep = job.getSparkCubingStep();
    StorageURL distMetaUrl = StorageURL.valueOf(sparkStep.getDistMetaUrl());
    Assert.assertEquals("hdfs", distMetaUrl.getScheme());
    Assert.assertTrue(distMetaUrl.getParameter("path").startsWith(config.getHdfsWorkingDirectory()));

    // launch the job
    execMgr.addJob(job);

    return wait(job);
}

Example 8

Source File: HybridCubeCLITest.java From kylin with Apache License 2.0

6 votes

@Test
public void testSegmentOverlap() throws IOException {
    thrown.expect(RuntimeException.class);
    thrown.expectMessage("Segments has overlap");

    HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
    Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
    HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });

    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
    CubeInstance cube2 = cubeManager.getCube("ssb_cube2");

    // 2012-01-01,2012-01-03
    cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
    // 2012-01-02,2012-01-04
    cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325462400000L, 1325635200000L));

    HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
}

Example 9

Source File: HybridCubeCLITest.java From kylin with Apache License 2.0

6 votes

@Test
public void testSegmentNotOverlap() throws IOException {
    HybridManager hybridManager = HybridManager.getInstance(KylinConfig.getInstanceFromEnv());
    Assert.assertNull(hybridManager.getHybridInstance("ssb_hybrid"));
    HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "create" });

    CubeManager cubeManager = CubeManager.getInstance(KylinConfig.getInstanceFromEnv());
    CubeInstance cube1 = cubeManager.getCube("ssb_cube1");
    CubeInstance cube2 = cubeManager.getCube("ssb_cube2");

    // 2012-01-01,2012-01-03
    cubeManager.appendSegment(cube1, new SegmentRange.TSRange(1325376000000L, 1325548800000L));
    // 2012-01-03,2012-01-04
    cubeManager.appendSegment(cube2, new SegmentRange.TSRange(1325548800000L, 1325635200000L));

    // not throw exception
    HybridCubeCLI.main(new String[] { "-name", "ssb_hybrid", "-project", "default", "-model", "ssb", "-cubes", "ssb_cube1,ssb_cube2", "-action", "update" });
}

Example 10

Source File: JdbcHiveMRInputTest.java From kylin with Apache License 2.0

6 votes

@Test
public void testGenSqoopCmd_Partition() throws IOException {
    ISource source = SourceManager.getSource(new JdbcSourceAware());
    IMRInput input = source.adaptToBuildEngine(IMRInput.class);
    Assert.assertNotNull(input);

    CubeManager cubeManager = CubeManager.getInstance(getTestConfig());
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_inner_join_cube");
    CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()),
            new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L));
    CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg);
    JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input
            .getBatchCubingInputSide(flatDesc);

    AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp",
            cubeDesc.getName());
    Assert.assertNotNull(executable);

    String cmd = executable.getParam("cmd");
    Assert.assertTrue(cmd.contains("org.h2.Driver"));
    Assert.assertTrue(cmd.contains(
            "--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"LEAF_CATEG_ID\\\")" + System.lineSeparator()
                    + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\""));
    source.close();
}

Example 11

Source File: JdbcHiveMRInputTest.java From kylin with Apache License 2.0

6 votes

@Test
public void testGenSqoopCmd_NoPartition() throws IOException {
    ISource source = SourceManager.getSource(new JdbcSourceAware());
    IMRInput input = source.adaptToBuildEngine(IMRInput.class);
    Assert.assertNotNull(input);

    CubeManager cubeManager = CubeManager.getInstance(getTestConfig());
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ci_left_join_cube");
    CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()),
            new SegmentRange.TSRange(0L, Long.MAX_VALUE));
    CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg);
    JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input
            .getBatchCubingInputSide(flatDesc);

    AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp",
            cubeDesc.getName());
    Assert.assertNotNull(executable);
    String cmd = executable.getParam("cmd");
    Assert.assertTrue(cmd.contains("org.h2.Driver"));
    Assert.assertTrue(
            cmd.contains("--boundary-query \"SELECT MIN(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\"), MAX(\\\"TEST_KYLIN_FACT\\\".\\\"CAL_DT\\\")" + System.lineSeparator()
                    + "FROM \\\"DEFAULT\\\".\\\"TEST_KYLIN_FACT\\\" AS \\\"TEST_KYLIN_FACT\\\"\""));
    source.close();
}

Example 12

Source File: JdbcHiveMRInputTest.java From kylin with Apache License 2.0

6 votes

@Test
public void testGenSqoopCmd_WithLookupShardBy() throws IOException {
    ISource source = SourceManager.getSource(new JdbcSourceAware());
    IMRInput input = source.adaptToBuildEngine(IMRInput.class);
    Assert.assertNotNull(input);

    CubeManager cubeManager = CubeManager.getInstance(getTestConfig());
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("ut_jdbc_shard");
    CubeSegment seg = cubeManager.appendSegment(cubeManager.getCube(cubeDesc.getName()),
            new SegmentRange.TSRange(System.currentTimeMillis() - 100L, System.currentTimeMillis() + 100L));
    CubeJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(seg);
    JdbcHiveMRInput.JdbcMRBatchCubingInputSide inputSide = (JdbcHiveMRInput.JdbcMRBatchCubingInputSide) input
            .getBatchCubingInputSide(flatDesc);

    AbstractExecutable executable = new MockInputSide(flatDesc, inputSide).createSqoopToFlatHiveStep("/tmp",
            cubeDesc.getName());
    Assert.assertNotNull(executable);

    String cmd = executable.getParam("cmd");
    Assert.assertTrue(cmd.contains("org.h2.Driver"));
    Assert.assertTrue(cmd.contains(
            "--boundary-query \"SELECT MIN(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\"), MAX(\\\"TEST_CATEGORY_GROUPINGS\\\".\\\"META_CATEG_NAME\\\")" + System.lineSeparator()
                    + "FROM \\\"DEFAULT\\\".\\\"TEST_CATEGORY_GROUPINGS\\\" AS \\\"TEST_CATEGORY_GROUPINGS\\\"\""));

    source.close();
}

Example 13

Source File: CsvSourceTest.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

@Test
public void testGetFlatTable() throws IOException {
    System.out.println(getTestConfig().getMetadataUrl());
    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    CubeInstance cube = cubeMgr.getCube(CUBE_NAME);
    cleanupSegments(CUBE_NAME);
    DataModelDesc model = cube.getModel();
    CubeSegment segment = cubeMgr.appendSegment(cube, new SegmentRange.TSRange(dateToLong("2010-01-01"), dateToLong("2013-01-01")));
    Dataset<Row> ds = initFlatTable(segment);
    ds.show(10);
    StructType schema = ds.schema();

    SegmentInfo segmentInfo = MetadataConverter.getSegmentInfo(segment.getCubeInstance(), segment.getUuid(),
            segment.getName(), segment.getStorageLocationIdentifier());
    scala.collection.immutable.Map<String, String> map = BuildUtils.getColumnIndexMap(segmentInfo);
    for (StructField field : schema.fields()) {
        Assert.assertNotNull(model.findColumn(map.apply(field.name())));
    }

    for (LayoutEntity layoutEntity : MetadataConverter.extractEntityList2JavaList(cube)) {
        Set<Integer> dims = layoutEntity.getOrderedDimensions().keySet();
        Column[] modelCols = new Column[dims.size()];
        int index = 0;
        for (int id : dims) {
            modelCols[index] = new Column(String.valueOf(id));
            index++;
        }
        ds.select(modelCols).show(10);
    }
}

Example 14

Source File: CacheServiceTest.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Test
public void testCubeCRUD() throws Exception {
    final Broadcaster broadcaster = Broadcaster.getInstance(configA);
    broadcaster.getCounterAndClear();

    getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");

    //create cube

    final String cubeName = "test_kylin_cube_a_new_one";
    final CubeManager cubeManager = getCubeManager(configA);
    final CubeManager cubeManagerB = getCubeManager(configB);
    final ProjectManager projectManager = getProjectManager(configA);
    final ProjectManager projectManagerB = getProjectManager(configB);
    final CubeDescManager cubeDescManager = getCubeDescManager(configA);
    final CubeDescManager cubeDescManagerB = getCubeDescManager(configB);
    final CubeDesc cubeDesc = getCubeDescManager(configA).getCubeDesc("test_kylin_cube_with_slr_desc");

    assertTrue(cubeManager.getCube(cubeName) == null);
    assertTrue(cubeManagerB.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    cubeManager.createCube(cubeName, ProjectInstance.DEFAULT_PROJECT_NAME, cubeDesc, null);
    //one for cube update, one for project update
    assertEquals(2, broadcaster.getCounterAndClear());
    waitForCounterAndClear(2);

    assertNotNull(cubeManager.getCube(cubeName));
    assertNotNull(cubeManagerB.getCube(cubeName));
    assertTrue(containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    assertTrue(containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));

    //update cube
    CubeInstance cube = cubeManager.getCube(cubeName);
    assertEquals(0, cube.getSegments().size());
    assertEquals(0, cubeManagerB.getCube(cubeName).getSegments().size());
    CubeSegment segment = cubeManager.appendSegment(cube, new TSRange(0L, 1000L));
    //one for cube update
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertEquals(1, cubeManagerB.getCube(cubeName).getSegments().size());
    assertEquals(segment.getName(), cubeManagerB.getCube(cubeName).getSegments().get(0).getName());

    //delete cube
    cubeManager.dropCube(cubeName, false);
    //one for cube update, one for project update
    assertEquals(2, broadcaster.getCounterAndClear());
    waitForCounterAndClear(2);

    assertTrue(cubeManager.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    assertTrue(cubeManagerB.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));

    final String cubeDescName = "test_cube_desc";
    cubeDesc.setName(cubeDescName);
    cubeDesc.setLastModified(0);
    assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
    assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
    cubeDescManager.createCubeDesc(cubeDesc);
    //one for add cube desc
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertNotNull(cubeDescManager.getCubeDesc(cubeDescName));
    assertNotNull(cubeDescManagerB.getCubeDesc(cubeDescName));

    cubeDesc.setNotifyList(Arrays.asList("[email protected]", "[email protected]", "[email protected]"));
    cubeDescManager.updateCubeDesc(cubeDesc);
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertEquals(cubeDesc.getNotifyList(), cubeDescManagerB.getCubeDesc(cubeDescName).getNotifyList());

    cubeDescManager.removeCubeDesc(cubeDesc);
    //one for add cube desc
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
    assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);

    getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");
}

Example 15

Source File: JobStepFactoryTest.java From kylin-on-parquet-v2 with Apache License 2.0

4 votes

@Test
public void testAddStepInMerging() throws Exception {
    CubeManager cubeMgr = CubeManager.getInstance(config);
    CubeInstance cube = cubeMgr.getCube(CUBE_NAME);

    cleanupSegments(CUBE_NAME);
    /**
     * Round1. Add 2 segment
     */
    CubeSegment segment1 = cubeMgr.appendSegment(cube, new SegmentRange.TSRange(dateToLong("2010-01-01"), dateToLong("2013-01-01")));
    CubeSegment segment2 = cubeMgr.appendSegment(cube, new SegmentRange.TSRange(dateToLong("2013-01-01"), dateToLong("2015-01-01")));
    segment1.setStatus(SegmentStatusEnum.READY);
    segment2.setStatus(SegmentStatusEnum.READY);

    CubeInstance reloadCube = cube.latestCopyForWrite();
    Segments segments = new Segments();
    segments.add(segment1);
    segments.add(segment2);
    reloadCube.setSegments(segments);
    CubeUpdate update = new CubeUpdate(reloadCube);
    cubeMgr.updateCube(update);

    /**
     * Round2. Merge two segments
     */

    reloadCube = cubeMgr.reloadCube(CUBE_NAME);
    CubeSegment mergedSegment = cubeMgr.mergeSegments(reloadCube, new SegmentRange.TSRange(dateToLong("2010-01-01"), dateToLong("2015-01-01"))
            , null, true);
    NSparkMergingJob job = NSparkMergingJob.merge(mergedSegment, "ADMIN");
    Assert.assertEquals(CUBE_NAME, job.getParam(MetadataConstants.P_CUBE_NAME));

    NSparkExecutable resourceDetectStep = job.getResourceDetectStep();
    Assert.assertEquals(ResourceDetectBeforeMergingJob.class.getName(),
            resourceDetectStep.getSparkSubmitClassName());
    Assert.assertEquals(ExecutableConstants.STEP_NAME_DETECT_RESOURCE, resourceDetectStep.getName());
    job.getParams().forEach((key, value) -> Assert.assertEquals(value, resourceDetectStep.getParam(key)));
    Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), resourceDetectStep.getId()).toString(),
            resourceDetectStep.getDistMetaUrl());

    NSparkExecutable mergeStep = job.getSparkMergingStep();
    Assert.assertEquals(config.getSparkMergeClassName(), mergeStep.getSparkSubmitClassName());
    Assert.assertEquals(ExecutableConstants.STEP_NAME_MERGER_SPARK_SEGMENT, mergeStep.getName());
    job.getParams().forEach((key, value) -> Assert.assertEquals(value, mergeStep.getParam(key)));
    Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), mergeStep.getId()).toString(),
            mergeStep.getDistMetaUrl());

    CubeInstance cubeInstance = cubeMgr.reloadCube(CUBE_NAME);
    NSparkUpdateMetaAndCleanupAfterMergeStep cleanStep = job.getCleanUpAfterMergeStep();
    job.getParams().forEach((key, value) -> {
        if (key.equalsIgnoreCase(MetadataConstants.P_SEGMENT_IDS)) {
            final List<String> needDeleteSegmentNames = cubeInstance.getMergingSegments(mergedSegment).stream()
                    .map(CubeSegment::getName).collect(Collectors.toList());
            Assert.assertEquals(needDeleteSegmentNames, Arrays.asList(cleanStep.getParam(MetadataConstants.P_SEGMENT_NAMES).split(",")));
        } else {
            Assert.assertEquals(value, mergeStep.getParam(key));
        }
    });
    Assert.assertEquals(config.getJobTmpMetaStoreUrl(getProject(), cleanStep.getId()).toString(),
            cleanStep.getDistMetaUrl());
}

Example 16

Source File: CacheServiceTest.java From kylin with Apache License 2.0

4 votes

@Test
public void testCubeCRUD() throws Exception {
    final Broadcaster broadcaster = Broadcaster.getInstance(configA);
    broadcaster.getCounterAndClear();

    getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");

    //create cube

    final String cubeName = "test_kylin_cube_a_new_one";
    final CubeManager cubeManager = getCubeManager(configA);
    final CubeManager cubeManagerB = getCubeManager(configB);
    final ProjectManager projectManager = getProjectManager(configA);
    final ProjectManager projectManagerB = getProjectManager(configB);
    final CubeDescManager cubeDescManager = getCubeDescManager(configA);
    final CubeDescManager cubeDescManagerB = getCubeDescManager(configB);
    final CubeDesc cubeDesc = getCubeDescManager(configA).getCubeDesc("test_kylin_cube_with_slr_desc");

    assertTrue(cubeManager.getCube(cubeName) == null);
    assertTrue(cubeManagerB.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    cubeManager.createCube(cubeName, ProjectInstance.DEFAULT_PROJECT_NAME, cubeDesc, null);
    //one for cube update, one for project update
    assertEquals(2, broadcaster.getCounterAndClear());
    waitForCounterAndClear(2);

    assertNotNull(cubeManager.getCube(cubeName));
    assertNotNull(cubeManagerB.getCube(cubeName));
    assertTrue(containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    assertTrue(containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));

    //update cube
    CubeInstance cube = cubeManager.getCube(cubeName);
    assertEquals(0, cube.getSegments().size());
    assertEquals(0, cubeManagerB.getCube(cubeName).getSegments().size());
    CubeSegment segment = cubeManager.appendSegment(cube, new TSRange(0L, 1000L));
    //one for cube update
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertEquals(1, cubeManagerB.getCube(cubeName).getSegments().size());
    assertEquals(segment.getName(), cubeManagerB.getCube(cubeName).getSegments().get(0).getName());

    //delete cube
    cubeManager.dropCube(cubeName, false);
    //one for cube update, one for project update
    assertEquals(2, broadcaster.getCounterAndClear());
    waitForCounterAndClear(2);

    assertTrue(cubeManager.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManager.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));
    assertTrue(cubeManagerB.getCube(cubeName) == null);
    assertTrue(!containsRealization(projectManagerB.listAllRealizations(ProjectInstance.DEFAULT_PROJECT_NAME),
            RealizationType.CUBE, cubeName));

    final String cubeDescName = "test_cube_desc";
    cubeDesc.setName(cubeDescName);
    cubeDesc.setLastModified(0);
    assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
    assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);
    cubeDescManager.createCubeDesc(cubeDesc);
    //one for add cube desc
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertNotNull(cubeDescManager.getCubeDesc(cubeDescName));
    assertNotNull(cubeDescManagerB.getCubeDesc(cubeDescName));

    cubeDesc.setNotifyList(Arrays.asList("[email protected]", "[email protected]", "[email protected]"));
    cubeDescManager.updateCubeDesc(cubeDesc);
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertEquals(cubeDesc.getNotifyList(), cubeDescManagerB.getCubeDesc(cubeDescName).getNotifyList());

    cubeDescManager.removeCubeDesc(cubeDesc);
    //one for add cube desc
    assertEquals(1, broadcaster.getCounterAndClear());
    waitForCounterAndClear(1);
    assertTrue(cubeDescManager.getCubeDesc(cubeDescName) == null);
    assertTrue(cubeDescManagerB.getCubeDesc(cubeDescName) == null);

    getStore().deleteResource("/cube/test_kylin_cube_a_new_one.json");
}