org.apache.kylin.cube.model.CubeDesc Java Examples

The following examples show how to use org.apache.kylin.cube.model.CubeDesc. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CuboidSchedulerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds, long[] expectChildren) {
    Set<Long> totalSpanning = Sets.newHashSet();
    for (long cuboidId : cuboidIds) {
        List<Long> spannings = scheduler.getSpanningCuboid(cuboidId);
        totalSpanning.addAll(spannings);
        System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " + toString(spannings));

        for (long child : spannings) {
            assertTrue(scheduler.isValid(child));
        }
    }

    long[] spanningsArray = Longs.toArray(totalSpanning);

    Arrays.parallelSort(spanningsArray);
    Arrays.parallelSort(expectChildren);
    assertArrayEquals(expectChildren, spanningsArray);
}
 
Example #2
Source File: FlinkCubingByLayer.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
    KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl);
    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
            .setAndUnsetThreadLocalConfig(kConfig)) {
        CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName);
        CubeDesc cubeDesc = cubeInstance.getDescriptor();
        CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId);
        CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich(
                EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc);
        long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc);
        Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId);
        baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc,
                AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid),
                MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap());
    }
}
 
Example #3
Source File: AggregationGroupRuleTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void testGoodDesc() throws IOException {
    AggregationGroupRule rule = getAggregationGroupRule();

    for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/").listFiles()) {
        if (!f.getName().endsWith("json")) {
            continue;
        }
        CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
        desc.init(getTestConfig());
        ValidateContext vContext = new ValidateContext();
        rule.validate(desc, vContext);
        //vContext.print(System.out);
        assertTrue(vContext.getResults().length == 0);
    }
}
 
Example #4
Source File: QueryGenerator.java    From kylin with Apache License 2.0 6 votes vote down vote up
public static String generateQuery(CubeDesc cubeDesc, Set<BitSet> selected, int maxNumOfDimension) {
    IJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(cubeDesc);

    String dimensionStatement = createDimensionStatement(cubeDesc.getDimensions(), selected, maxNumOfDimension);
    String measureStatement = createMeasureStatement(cubeDesc.getMeasures());

    StringBuilder sql = new StringBuilder();
    sql.append("SELECT" + "\n");
    sql.append(dimensionStatement);
    sql.append(measureStatement);

    StringBuilder joinPart = new StringBuilder();
    JoinedFlatTable.appendJoinStatement(flatDesc, joinPart, false, null);
    sql.append(joinPart.toString().replaceAll("DEFAULT\\.", ""));

    sql.append("GROUP BY" + "\n");
    sql.append(dimensionStatement);
    String ret = sql.toString();
    ret = ret.replaceAll("`", "\"");
    return ret;
}
 
Example #5
Source File: CubeHFileMapper.java    From Kylin with Apache License 2.0 6 votes vote down vote up
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) {

            cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName());
            qBytes = Bytes.toBytes(colDesc.getQualifier());
            timestamp = System.currentTimeMillis();

            List<MeasureDesc> measures = cubeDesc.getMeasures();
            String[] measureNames = getMeasureNames(cubeDesc);
            String[] refs = colDesc.getMeasureRefs();

            refIndex = new int[refs.length];
            refMeasures = new MeasureDesc[refs.length];
            for (int i = 0; i < refs.length; i++) {
                refIndex[i] = indexOf(measureNames, refs[i]);
                refMeasures[i] = measures.get(refIndex[i]);
            }

            codec = new MeasureCodec(refMeasures);
            colValues = new Object[refs.length];

            isFullCopy = true;
            for (int i = 0; i < measures.size(); i++) {
                if (refIndex.length <= i || refIndex[i] != i)
                    isFullCopy = false;
            }
        }
 
Example #6
Source File: CuboidTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void testIsValid2() {
    CubeDesc cube = getTestKylinCubeWithoutSeller();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();

    assertEquals(false, cuboidScheduler.isValid(toLong("111111111")));

    // base
    assertEquals(false, cuboidScheduler.isValid(0));
    assertEquals(true, cuboidScheduler.isValid(toLong("11111111")));

    // aggregation group & zero tail
    assertEquals(true, cuboidScheduler.isValid(toLong("10000111")));
    assertEquals(false, cuboidScheduler.isValid(toLong("10001111")));
    assertEquals(false, cuboidScheduler.isValid(toLong("11001111")));
    assertEquals(true, cuboidScheduler.isValid(toLong("10000001")));
    assertEquals(true, cuboidScheduler.isValid(toLong("10000101")));

    // hierarchy
    assertEquals(true, cuboidScheduler.isValid(toLong("10100000")));
    assertEquals(true, cuboidScheduler.isValid(toLong("10110000")));
    assertEquals(true, cuboidScheduler.isValid(toLong("10111000")));
    assertEquals(false, cuboidScheduler.isValid(toLong("10001000")));
    assertEquals(false, cuboidScheduler.isValid(toLong("10011000")));
}
 
Example #7
Source File: CubeManager.java    From kylin with Apache License 2.0 6 votes vote down vote up
public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner)
        throws IOException {
    try (AutoLock lock = cubeMapLock.lockForWrite()) {
        logger.info("Creating cube '{}-->{}' from desc '{}'", projectName, cubeName, desc.getName());

        // save cube resource
        CubeInstance cube = CubeInstance.create(cubeName, desc);
        cube.setOwner(owner);
        updateCubeWithRetry(new CubeUpdate(cube), 0);

        ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName,
                owner);

        return cube;
    }
}
 
Example #8
Source File: TableSchemaUpdaterTest.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void testDealWithMappingForCubeDesc() throws IOException {
    CubeDescManager cubeDescManager = CubeDescManager.getInstance(getTestConfig());
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc("ci_left_join_cube");

    CubeDesc updated = TableSchemaUpdater.dealWithMappingForCubeDesc(cubeDesc, mappings);
    updated = reinit(updated, cubeDescManager.CUBE_DESC_SERIALIZER);

    try (DataInputStream bis = new DataInputStream(
            new FileInputStream(new File(mappingRootPath + CubeDesc.concatResourcePath(updated.getName()))))) {
        CubeDesc expected = cubeDescManager.CUBE_DESC_SERIALIZER.deserialize(bis);
        Assert.assertTrue(expected.equalsRaw(updated));
    } catch (Exception e) {
        Assert.fail("CubeDesc is not updated correctly");
    }
}
 
Example #9
Source File: CuboidSchedulerTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void testFindSmallerSibling2() {
    CubeDesc cube = getTestKylinCubeWithSeller();
    CuboidScheduler scheduler = new CuboidScheduler(cube);

    Collection<Long> siblings;

    siblings = scheduler.findSmallerSibling(511);
    assertEquals("[]", siblings.toString());

    siblings = scheduler.findSmallerSibling(toLong("110111111"));
    assertEquals("[383]", siblings.toString());

    siblings = scheduler.findSmallerSibling(toLong("101110111"));
    assertEquals("[319]", siblings.toString());

    siblings = scheduler.findSmallerSibling(toLong("111111000"));
    assertEquals("[]", siblings.toString());

    siblings = scheduler.findSmallerSibling(toLong("111111000"));
    assertEquals("[]", siblings.toString());

    siblings = scheduler.findSmallerSibling(toLong("110000000"));
    assertEquals("[288, 320]", sortToString(siblings));
}
 
Example #10
Source File: CuboidTest.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void testCuboid2() {
    CubeDesc cube = getTestKylinCubeWithSeller();
    Cuboid cuboid;

    cuboid = Cuboid.findById(cube, 0);
    assertEquals(toLong("100100000"), cuboid.getId());

    cuboid = Cuboid.findById(cube, 1);
    assertEquals(toLong("100000111"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("010"));
    assertEquals(toLong("100000111"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("0100000"));
    assertEquals(toLong("100100000"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("1001000"));
    assertEquals(toLong("101111000"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("1000111"));
    assertEquals(toLong("101000111"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("111111111"));
    assertEquals(toLong("111111111"), cuboid.getId());
}
 
Example #11
Source File: MergeDictionaryStep.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * For the new segment, we need to create new dimension dictionaries by merging underlying
 * dictionaries. (https://issues.apache.org/jira/browse/KYLIN-2457, https://issues.apache.org/jira/browse/KYLIN-2800)
 * @param cube
 * @param newSeg
 * @throws IOException
 */
private void makeDictForNewSegment(KylinConfig conf, CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) throws IOException {
    DictionaryManager dictMgr = DictionaryManager.getInstance(conf);
    CubeDesc cubeDesc = cube.getDescriptor();

    for (TblColRef col : cubeDesc.getAllColumnsNeedDictionaryBuilt()) {
        logger.info("Merging fact table dictionary on : " + col);
        List<DictionaryInfo> dictInfos = new ArrayList<DictionaryInfo>();
        for (CubeSegment segment : mergingSegments) {
            logger.info("Including fact table dictionary of segment : " + segment);
            if (segment.getDictResPath(col) != null) {
                DictionaryInfo dictInfo = dictMgr.getDictionaryInfo(segment.getDictResPath(col));
                if (dictInfo != null && !dictInfos.contains(dictInfo)) {
                    dictInfos.add(dictInfo);
                } else {
                    logger.warn("Failed to load DictionaryInfo from " + segment.getDictResPath(col));
                }
            }
        }
        mergeDictionaries(dictMgr, newSeg, dictInfos, col);
    }
}
 
Example #12
Source File: TopNMeasureTypeTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void test() {

    CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc");

    MeasureDesc topSellerMeasure = null;

    for (MeasureDesc measureDesc : desc.getMeasures()) {
        if (measureDesc.getName().equals("TOP_SELLER")) {
            topSellerMeasure = measureDesc;
            break;
        }
    }
    TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType());

    topSellerMeasure.getFunction().getConfiguration().clear();
    List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());

    assertTrue(colsNeedDict != null && colsNeedDict.size() == 1);

    TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1);
    topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6");
    colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction());

    assertTrue(colsNeedDict.size() == 0);
}
 
Example #13
Source File: CubeDescManager.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey)
        throws IOException {
    String cubeDescName = cacheKey;
    CubeDesc cubeDesc = getCubeDesc(cubeDescName);
    String modelName = cubeDesc == null ? null : cubeDesc.getModelName();

    if (event == Event.DROP)
        removeLocalCubeDesc(cubeDescName);
    else
        reloadCubeDescQuietly(cubeDescName);

    for (ProjectInstance prj : ProjectManager.getInstance(config).findProjectsByModel(modelName)) {
        broadcaster.notifyProjectSchemaUpdate(prj.getName());
    }
}
 
Example #14
Source File: HBaseLookupMRSteps.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) {
    CubeDesc cubeDesc = cube.getDescriptor();
    Set<String> allLookupTables = Sets.newHashSet();
    for (DimensionDesc dim : cubeDesc.getDimensions()) {
        TableRef table = dim.getTableRef();
        if (cubeDesc.getModel().isLookupTable(table)) {
            allLookupTables.add(table.getTableIdentity());
        }
    }
    List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList();
    for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) {
        if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType())
                && allLookupTables.contains(snapshotTableDesc.getTableName())) {
            addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc);
        }
    }
}
 
Example #15
Source File: LookupTableToHFileJob.java    From kylin with Apache License 2.0 6 votes vote down vote up
private String[] getLookupKeyColumns(CubeInstance cube, String tableName) {
    CubeDesc cubeDesc = cube.getDescriptor();
    DataModelDesc modelDesc = cubeDesc.getModel();
    TableRef lookupTableRef = null;
    for (TableRef tableRef : modelDesc.getLookupTables()) {
        if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) {
            lookupTableRef = tableRef;
            break;
        }
    }
    if (lookupTableRef == null) {
        throw new IllegalStateException("cannot find table in model:" + tableName);
    }
    JoinDesc joinDesc = modelDesc.getJoinByPKSide(lookupTableRef);
    TblColRef[] keyColRefs = joinDesc.getPrimaryKeyColumns();
    String[] result = new String[keyColRefs.length];
    for (int i = 0; i < keyColRefs.length; i++) {
        result[i] = keyColRefs[i].getName();
    }
    return result;
}
 
Example #16
Source File: CubeMetadataUpgrade.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private void upgradeCubeDesc() {
    logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));

    List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT);
    for (String path : paths) {

        try {
            CubeDescUpgrader upgrade = new CubeDescUpgrader(path);
            CubeDesc ndesc = upgrade.upgrade();
            ndesc.setSignature(ndesc.calculateSignature());

            getStore().putResource(ndesc.getModel().getResourcePath(), ndesc.getModel(), MetadataManager.MODELDESC_SERIALIZER);
            getStore().putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER);
            updatedResources.add(ndesc.getResourcePath());
        } catch (IOException e) {
            e.printStackTrace();
            errorMsgs.add("Upgrade CubeDesc at '" + path + "' failed: " + e.getLocalizedMessage());
        }
    }

}
 
Example #17
Source File: Cuboid.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private static long translateToValidCuboid(CubeDesc cube, long cuboidID) {
    if (Cuboid.isValid(cube, cuboidID)) {
        return cuboidID;
    }

    HashSet<Long> dedupped = new HashSet<Long>();
    Queue<Long> queue = new LinkedList<Long>();
    List<Long> parents = Cuboid.getAllPossibleParents(cube, cuboidID);

    // check each parent
    addToQueue(queue, parents, dedupped);
    while (queue.size() > 0) {
        long parent = pollFromQueue(queue, dedupped);
        if (Cuboid.isValid(cube, parent)) {
            return parent;
        } else {
            addToQueue(queue, Cuboid.getAllPossibleParents(cube, parent), dedupped);
        }
    }
    return -1;
}
 
Example #18
Source File: DictionaryRuleTest.java    From kylin with Apache License 2.0 6 votes vote down vote up
private void testDictionaryDesc(String expectMessage, DictionaryDesc... descs) throws IOException {
    DictionaryRule rule = new DictionaryRule();
    File f = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/test_kylin_cube_without_slr_left_join_desc.json");
    CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);

    List<DictionaryDesc> newDicts = Lists.newArrayList(desc.getDictionaries());
    for (DictionaryDesc dictDesc : descs) {
        newDicts.add(dictDesc);
    }
    desc.setDictionaries(newDicts);

    desc.init(config);
    ValidateContext vContext = new ValidateContext();
    rule.validate(desc, vContext);

    if (expectMessage == null) {
        assertTrue(vContext.getResults().length == 0);
    } else {
        assertTrue(vContext.getResults().length == 1);
        String actualMessage = vContext.getResults()[0].getMessage();
        assertTrue(actualMessage.startsWith(expectMessage));
    }
}
 
Example #19
Source File: CuboidTest.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Test
public void testFindCuboidByIdWithSingleAggrGroup2() {
    CubeDesc cube = getTestKylinCubeWithSeller();
    Cuboid cuboid;

    cuboid = Cuboid.findById(cube, 0);
    assertEquals(toLong("101000000"), cuboid.getId());

    cuboid = Cuboid.findById(cube, 1);
    assertEquals(toLong("100000111"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("010"));
    assertEquals(toLong("100000111"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("0100000"));
    assertEquals(toLong("100100000"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("1001000"));
    assertEquals(toLong("101111000"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("1000111"));
    assertEquals(toLong("101000111"), cuboid.getId());

    cuboid = Cuboid.findById(cube, toLong("111111111"));
    assertEquals(toLong("111111111"), cuboid.getId());
}
 
Example #20
Source File: CuboidSchedulerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
private CubeDesc getFiftyDimFiveCapCubeDesc() {
    File metaFile = new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc/fifty_dim_five_cap.json.bad");
    assertTrue(metaFile.exists());
    String path = metaFile.getPath();
    metaFile.renameTo(new File(path.substring(0, path.length() - 4)));
    return CubeDescManager.getInstance(getTestConfig()).getCubeDesc("fifty_dim_five_cap");
}
 
Example #21
Source File: UpdateCubeAfterSnapshotStep.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig kylinConfig = context.getConfig();
    CubeManager cubeManager = CubeManager.getInstance(kylinConfig);

    CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams()));
    List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams());
    String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams());

    String extLookupSnapshotStr = this.getParam(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO);
    if (extLookupSnapshotStr == null || extLookupSnapshotStr.isEmpty()) {
        return new ExecuteResult();
    }

    Map<String, String> extLookupSnapshotMap = LookupMaterializeContext.parseLookupSnapshots(extLookupSnapshotStr);
    String snapshotResPath = extLookupSnapshotMap.get(lookupTableName);
    if (snapshotResPath == null) {
        logger.info("no snapshot path exist in the context, so no need to update snapshot path");
        return new ExecuteResult();
    }
    CubeDesc cubeDesc = cube.getDescriptor();
    try {
        logger.info("update snapshot path:{} to cube:{}", snapshotResPath, cube.getName());
        if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) {
            if (!snapshotResPath.equals(cube.getSnapshotResPath(lookupTableName))) {
                LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName,
                        snapshotResPath);
            }
        } else {
            LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName,
                    snapshotResPath);
        }
        return new ExecuteResult();
    } catch (IOException e) {
        logger.error("fail to save cuboid statistics", e);
        return ExecuteResult.createError(e);
    }
}
 
Example #22
Source File: MigrationRuleSet.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Override
public void apply(Context ctx) throws RuleValidationException {
    CubeDesc cubeDesc = ctx.getCubeInstance().getDescriptor();
    List<String> notifyList = cubeDesc.getNotifyList();
    if (notifyList == null || notifyList.size() == 0) {
        throw new RuleValidationException("Cube email notification list is not set or empty.");
    }
}
 
Example #23
Source File: DataController.java    From kylin with Apache License 2.0 5 votes vote down vote up
private Set<FunctionDesc> convertMetrics(CubeDesc cubeDesc, List<FunctionDesc> metrics) {
    Set<FunctionDesc> result = Sets.newHashSet();
    for (FunctionDesc metric : metrics) {
        result.add(findAggrFuncFromCubeDesc(cubeDesc, metric));
    }
    return result;
}
 
Example #24
Source File: SparkFactDistinct.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private void initDictColDeduper(CubeDesc cubeDesc) {
    // setup dict col deduper
    dictColDeduper = new DictColDeduper();
    Set<TblColRef> dictCols = cubeDesc.getAllColumnsNeedDictionaryBuilt();
    for (int i = 0; i < allCols.size(); i++) {
        if (dictCols.contains(allCols.get(i)))
            dictColDeduper.setIsDictCol(i);
    }
}
 
Example #25
Source File: CuboidSchedulerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test(expected=RuntimeException.class)
public void testTooManyCombination() {
    File twentyFile = new File(new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc"), "twenty_dim");
    twentyFile.renameTo(new File(twentyFile.getPath().substring(0, twentyFile.getPath().length() - 4)));
    CubeDesc cube = getTwentyDimCubeDesc();
    CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler();
    cuboidScheduler.getCuboidCount();
    twentyFile.renameTo(new File(twentyFile.getPath() + ".bad"));
}
 
Example #26
Source File: SparkFactDistinct.java    From kylin with Apache License 2.0 5 votes vote down vote up
private void initDictColDeduper(CubeDesc cubeDesc) {
    // setup dict col deduper
    dictColDeduper = new DictColDeduper();
    Set<TblColRef> dictCols = cubeDesc.getAllColumnsNeedDictionaryBuilt();
    for (int i = 0; i < allCols.size(); i++) {
        if (dictCols.contains(allCols.get(i)))
            dictColDeduper.setIsDictCol(i);
    }
}
 
Example #27
Source File: CacheControllerTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testClearCacheForCubeMigration() throws IOException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String CUBENAME = "test_kylin_cube_without_slr_desc";

    CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
    CubeDesc cubeDesc = cubeDescManager.getCubeDesc(CUBENAME);
    DataModelDesc modelDesc = cubeDesc.getModel();
    Map<String, String> tableToProjects = new HashMap<>();
    for (TableRef tableRef : modelDesc.getAllTables()) {
        tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject());
    }

    String uuid = cubeDesc.getUuid();
    String signature = cubeDesc.getSignature();

    assertEquals(cubeDesc.getRetentionRange(), 0);

    //update cubeDesc
    cubeDesc.setRetentionRange(2018);
    cubeDesc.updateRandomUuid();

    //directly update metadata in store to simulate cube migration
    Serializer<CubeDesc> cubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class);
    getStore().checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, cubeDescSerializer);

    CubeMigrationRequest request = new CubeMigrationRequest();
    request.setCube(cubeDesc.getName());
    request.setModel(modelDesc.getName());
    request.setProject(modelDesc.getProject());
    request.setTableToProjects(tableToProjects);

    cacheController.clearCacheForCubeMigration(request);

    assertEquals(2018, cubeDescManager.getCubeDesc(CUBENAME).getRetentionRange());
    assertEquals(signature, cubeDescManager.getCubeDesc(CUBENAME).getSignature());
    assertNotEquals(uuid, cubeDescManager.getCubeDesc(CUBENAME).getUuid());
}
 
Example #28
Source File: CubeScanRangePlanner.java    From kylin with Apache License 2.0 5 votes vote down vote up
private Set<TblColRef> replaceDerivedColumns(Set<TblColRef> input, CubeDesc cubeDesc) {
    Set<TblColRef> ret = Sets.newHashSet();
    for (TblColRef col : input) {
        if (cubeDesc.hasHostColumn(col)) {
            for (TblColRef host : cubeDesc.getHostInfo(col).columns) {
                ret.add(host);
            }
        } else {
            ret.add(col);
        }
    }
    return ret;
}
 
Example #29
Source File: RowRecordReader.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public RowRecordReader(CubeDesc cubeDesc, Path path, FileSystem fileSystem) throws IOException {
    super(fileSystem, path);
    this.cubeDesc = cubeDesc;
    this.folderPath = path;
    this.fs = fileSystem;
    this.currentRowRecord = new RowRecord();
    initReaders();
}
 
Example #30
Source File: DictionaryRuleTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
@Test
public void testGoodDesc() throws IOException {
    DictionaryRule rule = new DictionaryRule();

    for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/").listFiles()) {
        if (!f.getName().endsWith("json")) {
            continue;
        }
        CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class);
        desc.init(config);
        ValidateContext vContext = new ValidateContext();
        rule.validate(desc, vContext);
        assertTrue(vContext.getResults().length == 0);
    }
}