Java Code Examples for org.apache.kylin.cube.model.CubeDesc

The following examples show how to use org.apache.kylin.cube.model.CubeDesc. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: kylin-on-parquet-v2   Source File: FlinkCubingByLayer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
    KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
            .setAndUnsetThreadLocalConfig(kConfig)) {
        CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
        CubeDesc cubeDesc = cubeInstance.getDescriptor();
        CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
        CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich(
                EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
        Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
        baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc,
                AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid),
                MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
    }
}
 
Example 2
@Test
public void testGoodDesc() throws IOException {
    AggregationGroupRule rule = getAggregationGroupRule();

    for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/").listFiles()) {
        if (!f.getName().endsWith("json")) {
            continue;
        }
        CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
        desc.init(getTestConfig());
        ValidateContext vContext = new ValidateContext();
        rule.validate(desc, vContext);
        //vContext.print(System.out);
        assertTrue(vContext.getResults().length == 0);
    }
}
 
Example 3
Source Project: kylin-on-parquet-v2   Source File: CuboidTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testIsValid2() {
    CubeDesc cube = getTestKylinCubeWithoutSeller();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();

    assertEquals(false, cuboidScheduler.isValid(toLong("111111111")));

    // base
    assertEquals(false, cuboidScheduler.isValid(0));
    assertEquals(true, cuboidScheduler.isValid(toLong("11111111")));

    // aggregation group & zero tail
    assertEquals(true, cuboidScheduler.isValid(toLong("10000111")));
    assertEquals(false, cuboidScheduler.isValid(toLong("10001111")));
    assertEquals(false, cuboidScheduler.isValid(toLong("11001111")));
    assertEquals(true, cuboidScheduler.isValid(toLong("10000001")));
    assertEquals(true, cuboidScheduler.isValid(toLong("10000101")));

    // hierarchy
    assertEquals(true, cuboidScheduler.isValid(toLong("10100000")));
    assertEquals(true, cuboidScheduler.isValid(toLong("10110000")));
    assertEquals(true, cuboidScheduler.isValid(toLong("10111000")));
    assertEquals(false, cuboidScheduler.isValid(toLong("10001000")));
    assertEquals(false, cuboidScheduler.isValid(toLong("10011000")));
}
 
Example 4
Source Project: kylin   Source File: TableSchemaUpdaterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDealWithMappingForCubeDesc() throws IOException {
    CubeDescManager cubeDescManager = CubeDescManager.getInstance(getTestConfig());
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc("ci_left_join_cube");

    CubeDesc updated = TableSchemaUpdater.dealWithMappingForCubeDesc(cubeDesc, mappings);
    updated = reinit(updated, cubeDescManager.CUBE_DESC_SERIALIZER);

    try (DataInputStream bis = new DataInputStream(
            new FileInputStream(new File(mappingRootPath + CubeDesc.concatResourcePath(updated.getName()))))) {
        CubeDesc expected = cubeDescManager.CUBE_DESC_SERIALIZER.deserialize(bis);
        Assert.assertTrue(expected.equalsRaw(updated));
    } catch (Exception e) {
        Assert.fail("CubeDesc is not updated correctly");
    }
}
 
Example 5
Source Project: Kylin   Source File: CuboidSchedulerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFindSmallerSibling2() {
    CubeDesc cube = getTestKylinCubeWithSeller();
    CuboidScheduler scheduler = new CuboidScheduler(cube);

    Collection<Long> siblings;

    siblings = scheduler.findSmallerSibling(511);
    assertEquals("[]", siblings.toString());

    siblings = scheduler.findSmallerSibling(toLong("110111111"));
    assertEquals("[383]", siblings.toString());

    siblings = scheduler.findSmallerSibling(toLong("101110111"));
    assertEquals("[319]", siblings.toString());

    siblings = scheduler.findSmallerSibling(toLong("111111000"));
    assertEquals("[]", siblings.toString());

    siblings = scheduler.findSmallerSibling(toLong("111111000"));
    assertEquals("[]", siblings.toString());

    siblings = scheduler.findSmallerSibling(toLong("110000000"));
    assertEquals("[288, 320]", sortToString(siblings));
}
 
Example 6
Source Project: kylin   Source File: CuboidTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFindCuboidByIdWithSingleAggrGroup2() {
    CubeDesc cube = getTestKylinCubeWithSeller();
    Cuboid cuboid;

    cuboid = Cuboid.findById(cube, 0);
    assertEquals(toLong("101000000"), cuboid.getId());

    cuboid = Cuboid.findById(cube, 1);
    assertEquals(toLong("100000111"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("010"));
    assertEquals(toLong("100000111"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("0100000"));
    assertEquals(toLong("100100000"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("1001000"));
    assertEquals(toLong("101111000"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("1000111"));
    assertEquals(toLong("101000111"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("111111111"));
    assertEquals(toLong("111111111"), cuboid.getId());
}
 
Example 7
Source Project: kylin   Source File: MergeDictionaryStep.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * For the new segment, we need to create new dimension dictionaries by merging underlying
 * dictionaries. (https://issues.apache.org/jira/browse/KYLIN-2457, https://issues.apache.org/jira/browse/KYLIN-2800)
 * @param cube
 * @param newSeg
 * @throws IOException
 */
private void makeDictForNewSegment(KylinConfig conf, CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) throws IOException {
    DictionaryManager dictMgr = DictionaryManager.getInstance(conf);
    CubeDesc cubeDesc = cube.getDescriptor();

    for (TblColRef col : cubeDesc.getAllColumnsNeedDictionaryBuilt()) {
        logger.info("Merging fact table dictionary on : " + col);
        List<DictionaryInfo> dictInfos = new ArrayList<DictionaryInfo>();
        for (CubeSegment segment : mergingSegments) {
            logger.info("Including fact table dictionary of segment : " + segment);
            if (segment.getDictResPath(col) != null) {
                DictionaryInfo dictInfo = dictMgr.getDictionaryInfo(segment.getDictResPath(col));
                if (dictInfo != null && !dictInfos.contains(dictInfo)) {
                    dictInfos.add(dictInfo);
                } else {
                    logger.warn("Failed to load DictionaryInfo from " + segment.getDictResPath(col));
                }
            }
        }
        mergeDictionaries(dictMgr, newSeg, dictInfos, col);
    }
}
 
Example 8
Source Project: kylin-on-parquet-v2   Source File: HBaseLookupMRSteps.java    License: Apache License 2.0 6 votes vote down vote up
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) {
    CubeDesc cubeDesc = cube.getDescriptor();
    Set<String> allLookupTables = Sets.newHashSet();
    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        TableRef table = dim.getTableRef();
        if (cubeDesc.getModel().isLookupTable(table)) {
            allLookupTables.add(table.getTableIdentity());
        }
    }
    List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList();
    for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) {
        if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType())
                && allLookupTables.contains(snapshotTableDesc.getTableName())) {
            addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc);
        }
    }
}
 
Example 9
Source Project: kylin-on-parquet-v2   Source File: CuboidSchedulerTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds, long[] expectChildren) {
    Set<Long> totalSpanning = Sets.newHashSet();
    for (long cuboidId : cuboidIds) {
        List<Long> spannings = scheduler.getSpanningCuboid(cuboidId);
        totalSpanning.addAll(spannings);
        System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " + toString(spannings));

        for (long child : spannings) {
            assertTrue(scheduler.isValid(child));
        }
    }

    long[] spanningsArray = Longs.toArray(totalSpanning);

    Arrays.parallelSort(spanningsArray);
    Arrays.parallelSort(expectChildren);
    assertArrayEquals(expectChildren, spanningsArray);
}
 
Example 10
Source Project: kylin   Source File: CubeDescManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
        throws IOException {
    String cubeDescName = cacheKey;
    CubeDesc cubeDesc = getCubeDesc(cubeDescName);
    String modelName = cubeDesc == null ? null : cubeDesc.getModelName();

    if (event == Event.DROP)
        removeLocalCubeDesc(cubeDescName);
    else
        reloadCubeDescQuietly(cubeDescName);

    for (ProjectInstance prj : ProjectManager.getInstance(config).findProjectsByModel(modelName)) {
        broadcaster.notifyProjectSchemaUpdate(prj.getName());
    }
}
 
Example 11
Source Project: kylin   Source File: LookupTableToHFileJob.java    License: Apache License 2.0 6 votes vote down vote up
private String[] getLookupKeyColumns(CubeInstance cube, String tableName) {
    CubeDesc cubeDesc = cube.getDescriptor();
    DataModelDesc modelDesc = cubeDesc.getModel();
    TableRef lookupTableRef = null;
    for (TableRef tableRef : modelDesc.getLookupTables()) {
        if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) {
            lookupTableRef = tableRef;
            break;
        }
    }
    if (lookupTableRef == null) {
        throw new IllegalStateException("cannot find table in model:" + tableName);
    }
    JoinDesc joinDesc = modelDesc.getJoinByPKSide(lookupTableRef);
    TblColRef[] keyColRefs = joinDesc.getPrimaryKeyColumns();
    String[] result = new String[keyColRefs.length];
    for (int i = 0; i < keyColRefs.length; i++) {
        result[i] = keyColRefs[i].getName();
    }
    return result;
}
 
Example 12
Source Project: Kylin   Source File: CubeMetadataUpgrade.java    License: Apache License 2.0 6 votes vote down vote up
private void upgradeCubeDesc() {
    logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));

    List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT);
    for (String path : paths) {

        try {
            CubeDescUpgrader upgrade = new CubeDescUpgrader(path);
            CubeDesc ndesc = upgrade.upgrade();
            ndesc.setSignature(ndesc.calculateSignature());

            getStore().putResource(ndesc.getModel().getResourcePath(), ndesc.getModel(), MetadataManager.MODELDESC_SERIALIZER);
            getStore().putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER);
            updatedResources.add(ndesc.getResourcePath());
        } catch (IOException e) {
            e.printStackTrace();
            errorMsgs.add("Upgrade CubeDesc at '" + path + "' failed: " + e.getLocalizedMessage());
        }
    }

}
 
Example 13
Source Project: Kylin   Source File: Cuboid.java    License: Apache License 2.0 6 votes vote down vote up
private static long translateToValidCuboid(CubeDesc cube, long cuboidID) {
    if (Cuboid.isValid(cube, cuboidID)) {
        return cuboidID;
    }

    HashSet<Long> dedupped = new HashSet<Long>();
    Queue<Long> queue = new LinkedList<Long>();
    List<Long> parents = Cuboid.getAllPossibleParents(cube, cuboidID);

    // check each parent
    addToQueue(queue, parents, dedupped);
    while (queue.size() > 0) {
        long parent = pollFromQueue(queue, dedupped);
        if (Cuboid.isValid(cube, parent)) {
            return parent;
        } else {
            addToQueue(queue, Cuboid.getAllPossibleParents(cube, parent), dedupped);
        }
    }
    return -1;
}
 
Example 14
Source Project: kylin   Source File: DictionaryRuleTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testDictionaryDesc(String expectMessage, DictionaryDesc... descs) throws IOException {
    DictionaryRule rule = new DictionaryRule();
    File f = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/test_kylin_cube_without_slr_left_join_desc.json");
    CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);

    List<DictionaryDesc> newDicts = Lists.newArrayList(desc.getDictionaries());
    for (DictionaryDesc dictDesc : descs) {
        newDicts.add(dictDesc);
    }
    desc.setDictionaries(newDicts);

    desc.init(config);
    ValidateContext vContext = new ValidateContext();
    rule.validate(desc, vContext);

    if (expectMessage == null) {
        assertTrue(vContext.getResults().length == 0);
    } else {
        assertTrue(vContext.getResults().length == 1);
        String actualMessage = vContext.getResults()[0].getMessage();
        assertTrue(actualMessage.startsWith(expectMessage));
    }
}
 
Example 15
Source Project: kylin-on-parquet-v2   Source File: TopNMeasureTypeTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void test() {

    CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc");

    MeasureDesc topSellerMeasure = null;

    for (MeasureDesc measureDesc : desc.getMeasures()) {
        if (measureDesc.getName().equals("TOP_SELLER")) {
            topSellerMeasure = measureDesc;
            break;
        }
    }
    TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType());

    topSellerMeasure.getFunction().getConfiguration().clear();
    List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());

    assertTrue(colsNeedDict != null && colsNeedDict.size() == 1);

    TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1);
    topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6");
    colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());

    assertTrue(colsNeedDict.size() == 0);
}
 
Example 16
Source Project: Kylin   Source File: CuboidTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCuboid2() {
    CubeDesc cube = getTestKylinCubeWithSeller();
    Cuboid cuboid;

    cuboid = Cuboid.findById(cube, 0);
    assertEquals(toLong("100100000"), cuboid.getId());

    cuboid = Cuboid.findById(cube, 1);
    assertEquals(toLong("100000111"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("010"));
    assertEquals(toLong("100000111"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("0100000"));
    assertEquals(toLong("100100000"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("1001000"));
    assertEquals(toLong("101111000"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("1000111"));
    assertEquals(toLong("101000111"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("111111111"));
    assertEquals(toLong("111111111"), cuboid.getId());
}
 
Example 17
Source Project: Kylin   Source File: CubeHFileMapper.java    License: Apache License 2.0 6 votes vote down vote up
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

            cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
            qBytes = Bytes.toBytes(colDesc.getQualifier());
            timestamp = System.currentTimeMillis();

            List<MeasureDesc> measures = cubeDesc.getMeasures();
            String[] measureNames = getMeasureNames(cubeDesc);
            String[] refs = colDesc.getMeasureRefs();

            refIndex = new int[refs.length];
            refMeasures = new MeasureDesc[refs.length];
            for (int i = 0; i < refs.length; i++) {
                refIndex[i] = indexOf(measureNames, refs[i]);
                refMeasures[i] = measures.get(refIndex[i]);
            }

            codec = new MeasureCodec(refMeasures);
            colValues = new Object[refs.length];

            isFullCopy = true;
            for (int i = 0; i < measures.size(); i++) {
                if (refIndex.length <= i || refIndex[i] != i)
                    isFullCopy = false;
            }
        }
 
Example 18
Source Project: kylin   Source File: QueryGenerator.java    License: Apache License 2.0 6 votes vote down vote up
public static String generateQuery(CubeDesc cubeDesc, Set<BitSet> selected, int maxNumOfDimension) {
    IJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(cubeDesc);

    String dimensionStatement = createDimensionStatement(cubeDesc.getDimensions(), selected, maxNumOfDimension);
    String measureStatement = createMeasureStatement(cubeDesc.getMeasures());

    StringBuilder sql = new StringBuilder();
    sql.append("SELECT" + "\n");
    sql.append(dimensionStatement);
    sql.append(measureStatement);

    StringBuilder joinPart = new StringBuilder();
    JoinedFlatTable.appendJoinStatement(flatDesc, joinPart, false, null);
    sql.append(joinPart.toString().replaceAll("DEFAULT\\.", ""));

    sql.append("GROUP BY" + "\n");
    sql.append(dimensionStatement);
    String ret = sql.toString();
    ret = ret.replaceAll("`", "\"");
    return ret;
}
 
Example 19
Source Project: kylin   Source File: CubeManager.java    License: Apache License 2.0 6 votes vote down vote up
public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner)
        throws IOException {
    try (AutoLock lock = cubeMapLock.lockForWrite()) {
        logger.info("Creating cube '{}-->{}' from desc '{}'", projectName, cubeName, desc.getName());

        // save cube resource
        CubeInstance cube = CubeInstance.create(cubeName, desc);
        cube.setOwner(owner);
        updateCubeWithRetry(new CubeUpdate(cube), 0);

        ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName,
                owner);

        return cube;
    }
}
 
Example 20
Source Project: kylin-on-parquet-v2   Source File: HiveInputBase.java    License: Apache License 2.0 5 votes vote down vote up
protected static AbstractExecutable createRedistributeFlatHiveTableStep(String hiveInitStatements, String cubeName,
        IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) {
    RedistributeFlatHiveTableStep step = new RedistributeFlatHiveTableStep();
    step.setInitStatement(hiveInitStatements);
    step.setIntermediateTable(flatDesc.getTableName());
    step.setRedistributeDataStatement(JoinedFlatTable.generateRedistributeFlatTableStatement(flatDesc, cubeDesc));
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setName(ExecutableConstants.STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE);
    return step;
}
 
Example 21
Source Project: Kylin   Source File: RowKeyEncoderTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEncodeWithSlr2() throws Exception {
    CubeInstance cube = CubeManager.getInstance(getTestConfig()).getCube("TEST_KYLIN_CUBE_WITH_SLR_READY");
    // CubeSegment seg = cube.getTheOnlySegment();
    CubeDesc cubeDesc = cube.getDescriptor();
    // String data =
    // "1234567892013-08-18Abbigliamento e accessoriDonna: AccessoriSciarpFoulard e ScialliAuctionItalyRegular";
    byte[][] data = new byte[9][];
    data[0] = Bytes.toBytes("123456789");
    data[1] = null;
    data[2] = null;
    data[3] = null;
    data[4] = null;
    data[5] = null;
    data[6] = null;
    data[7] = null;
    data[8] = null;

    long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
    Cuboid baseCuboid = Cuboid.findById(cubeDesc, baseCuboidId);
    AbstractRowKeyEncoder rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cube.getFirstSegment(), baseCuboid);

    byte[] encodedKey = rowKeyEncoder.encode(data);
    assertEquals(48, encodedKey.length);
    byte[] sellerId = Arrays.copyOfRange(encodedKey, 8, 26);
    byte[] cuboidId = Arrays.copyOfRange(encodedKey, 0, 8);
    byte[] rest = Arrays.copyOfRange(encodedKey, 26, encodedKey.length);
    assertTrue(Bytes.toString(sellerId).startsWith("123456789"));
    assertEquals(511, Bytes.toLong(cuboidId));
    assertArrayEquals(new byte[] { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, rest);
}
 
Example 22
Source Project: kylin   Source File: HiveInputBase.java    License: Apache License 2.0 5 votes vote down vote up
protected static AbstractExecutable createRedistributeFlatHiveTableByLivyStep(String hiveInitStatements,
                                                                              String cubeName, IJoinedFlatTableDesc flatDesc, CubeDesc cubeDesc) {
    RedistributeFlatHiveTableByLivyStep step = new RedistributeFlatHiveTableByLivyStep();
    step.setInitStatement(hiveInitStatements);
    step.setIntermediateTable(flatDesc.getTableName());
    step.setRedistributeDataStatement(JoinedFlatTable.generateRedistributeFlatTableStatement(flatDesc, cubeDesc));
    CubingExecutableUtil.setCubeName(cubeName, step.getParams());
    step.setName(ExecutableConstants.STEP_NAME_REDISTRIBUTE_FLAT_HIVE_TABLE);
    return step;
}
 
Example 23
Source Project: kylin   Source File: AggregationGroupRuleTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGoodDesc2() throws IOException {

    ValidateContext vContext = new ValidateContext();
    CubeDesc desc = JsonUtil.readValue(new FileInputStream(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/test_kylin_cube_with_slr_desc.json"), CubeDesc.class);
    desc.getAggregationGroups().get(0).getSelectRule().jointDims = new String[][] { //
            new String[] { "lstg_format_name", "lstg_site_id", "slr_segment_cd", "CATEG_LVL2_NAME" } };

    IValidatorRule<CubeDesc> rule = getAggregationGroupRule();
    rule.validate(desc, vContext);
    //vContext.print(System.out);
    assertEquals(1, vContext.getResults().length);
}
 
Example 24
Source Project: kylin   Source File: StreamingCubeRule.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void validate(CubeDesc cube, ValidateContext context) {
    DataModelDesc model = cube.getModel();
    
    if (model.getRootFactTable().getTableDesc().getSourceType() != ISourceAware.ID_STREAMING
            && !model.getRootFactTable().getTableDesc().isStreamingTable()) {
        return;
    }

    if (model.getPartitionDesc() == null || model.getPartitionDesc().getPartitionDateColumn() == null) {
        context.addResult(ResultLevel.ERROR, "Must define a partition column.");
        return;
    }

    final TblColRef partitionCol = model.getPartitionDesc().getPartitionDateColumnRef();
    boolean found = false;
    for (DimensionDesc dimensionDesc : cube.getDimensions()) {
        for (TblColRef dimCol : dimensionDesc.getColumnRefs()) {
            if (dimCol.equals(partitionCol)) {
                found = true;
                break;
            }
        }
    }

    if (found == false) {
        context.addResult(ResultLevel.ERROR, "Partition column '" + partitionCol + "' isn't in dimension list.");
        return;
    }

}
 
Example 25
Source Project: Kylin   Source File: CubeSizeEstimationCLI.java    License: Apache License 2.0 5 votes vote down vote up
private static RowKeyColInfo extractRowKeyInfo(CubeDesc cubeDesc) {
    RowKeyDesc rowKeyDesc = cubeDesc.getRowkey();
    RowKeyColInfo info = new RowKeyColInfo();
    info.hierachyColBitIndice = new ArrayList<List<Integer>>();
    info.nonHierachyColBitIndice = new ArrayList<Integer>();
    HashSet<Integer> heirachyIndexSet = new HashSet<Integer>();

    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        if (dim.getHierarchy() != null) {
            LinkedList<Integer> hlist = new LinkedList<Integer>();
            for (HierarchyDesc hierarchyDesc : dim.getHierarchy()) {
                int index = rowKeyDesc.getColumnBitIndex(hierarchyDesc.getColumnRef());
                hlist.add(index);
                heirachyIndexSet.add(index);
            }
            info.hierachyColBitIndice.add(hlist);
        }
    }

    for (int i = 0; i < rowKeyDesc.getRowKeyColumns().length; ++i) {
        if (!heirachyIndexSet.contains(i)) {
            info.nonHierachyColBitIndice.add(i);
        }
    }

    return info;
}
 
Example 26
Source Project: Kylin   Source File: RowKeyAttrRuleTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testOneMandatoryColumn() {
    IValidatorRule<CubeDesc> rule = new RowKeyAttrRule();
    rule.validate(cube, vContext);
    vContext.print(System.out);
    assertTrue("Failed to validate rowkey", vContext.getResults().length == 1);
    assertTrue("Failed to validate mandatory error", vContext.getResults()[0].getMessage().startsWith("Rowkey column"));
}
 
Example 27
Source Project: Kylin   Source File: CubeReducerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReducer() throws Exception {

    reduceDriver.getConfiguration().set(BatchConstants.CFG_CUBE_NAME, "test_kylin_cube_with_slr_ready");

    CubeDesc cubeDesc = CubeManager.getInstance(getTestConfig()).getCube("test_kylin_cube_with_slr_ready").getDescriptor();
    MeasureCodec codec = new MeasureCodec(cubeDesc.getMeasures());

    Text key1 = new Text("72010ustech");
    List<Text> values1 = new ArrayList<Text>();
    values1.add(newValueText(codec, "15.09", "15.09", "15.09", 1));
    values1.add(newValueText(codec, "20.34", "20.34", "20.34", 1));
    values1.add(newValueText(codec, "10", "10", "10", 1));

    Text key2 = new Text("1tech");
    List<Text> values2 = new ArrayList<Text>();
    values2.add(newValueText(codec, "15.09", "15.09", "15.09", 1));
    values2.add(newValueText(codec, "20.34", "20.34", "20.34", 1));

    Text key3 = new Text("0");
    List<Text> values3 = new ArrayList<Text>();
    values3.add(newValueText(codec, "146.52", "146.52", "146.52", 4));

    reduceDriver.withInput(key1, values1);
    reduceDriver.withInput(key2, values2);
    reduceDriver.withInput(key3, values3);

    List<Pair<Text, Text>> result = reduceDriver.run();

    Pair<Text, Text> p1 = new Pair<Text, Text>(new Text("72010ustech"), newValueText(codec, "45.43", "10", "20.34", 3));
    Pair<Text, Text> p2 = new Pair<Text, Text>(new Text("1tech"), newValueText(codec, "35.43", "15.09", "20.34", 2));
    Pair<Text, Text> p3 = new Pair<Text, Text>(new Text("0"), newValueText(codec, "146.52", "146.52", "146.52", 4));

    assertEquals(3, result.size());

    assertTrue(result.contains(p1));
    assertTrue(result.contains(p2));
    assertTrue(result.contains(p3));
}
 
Example 28
Source Project: kylin-on-parquet-v2   Source File: CubeCapabilityChecker.java    License: Apache License 2.0 5 votes vote down vote up
private static Set<FunctionDesc> unmatchedAggregations(Collection<FunctionDesc> aggregations, CubeInstance cube) {
    HashSet<FunctionDesc> result = Sets.newHashSet(aggregations);

    CubeDesc cubeDesc = cube.getDescriptor();
    List<FunctionDesc> definedFuncs = cubeDesc.listAllFunctions();

    // check normal aggregations
    result.removeAll(definedFuncs);

    // check dynamic aggregations
    Iterator<FunctionDesc> funcIterator = result.iterator();
    while (funcIterator.hasNext()) {
        FunctionDesc entry = funcIterator.next();
        if (entry instanceof DynamicFunctionDesc) {
            DynamicFunctionDesc dynFunc = (DynamicFunctionDesc) entry;
            // Filter columns cannot be derived
            Collection<TblColRef> definedCols = dynFunc.ifFriendlyForDerivedFilter()
                    ? cubeDesc.listDimensionColumnsIncludingDerived()
                    : cubeDesc.listDimensionColumnsExcludingDerived(true);
            Set<TblColRef> filterCols = Sets.newHashSet(dynFunc.getFilterColumnSet());
            filterCols.removeAll(definedCols);
            if (!filterCols.isEmpty()) {
                continue;
            }

            // All inner funcs should be defined
            Set<FunctionDesc> innerFuncSet = Sets.newHashSet(dynFunc.getRuntimeFuncs());
            innerFuncSet.removeAll(definedFuncs);
            if (!innerFuncSet.isEmpty()) {
                continue;
            }

            funcIterator.remove();
        }
    }
    return result;
}
 
Example 29
Source Project: kylin   Source File: StreamingDataQueryPlanner.java    License: Apache License 2.0 5 votes vote down vote up
public StreamingDataQueryPlanner(CubeDesc cubeDesc, TupleFilter filter) {
    this.cubeDesc = cubeDesc;
    this.filter = filter;
    this.flatFilter = flattenToOrAndFilter(filter);
    String timezoneName = cubeDesc.getConfig().getStreamingDerivedTimeTimezone();
    if (timezoneName == null || timezoneName.length() == 0) {
        timezoneOffset = 0;
    } else {
        timezoneOffset = TimeZone.getTimeZone(timezoneName).getRawOffset();
    }
}
 
Example 30
Source Project: kylin-on-parquet-v2   Source File: CubeDescTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBadInit3() throws Exception {
    thrown.expect(IllegalStateException.class);
    thrown.expectMessage("Aggregation group 1 'includes' dimensions not include all the dimensions:");
    CubeDesc cubeDesc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc(CUBE_WITH_SLR_DESC);
    String[] temp = Arrays.asList(cubeDesc.getAggregationGroups().get(0).getIncludes()).subList(0, 3)
            .toArray(new String[3]);
    cubeDesc.getAggregationGroups().get(0).setIncludes(temp);

    cubeDesc.init(getTestConfig());
}