org.apache.kylin.cube.model.CubeDesc Java Examples
The following examples show how to use
org.apache.kylin.cube.model.CubeDesc.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CuboidSchedulerTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private void testSpanningAndGetParent(CuboidScheduler scheduler, CubeDesc cube, long[] cuboidIds, long[] expectChildren) { Set<Long> totalSpanning = Sets.newHashSet(); for (long cuboidId : cuboidIds) { List<Long> spannings = scheduler.getSpanningCuboid(cuboidId); totalSpanning.addAll(spannings); System.out.println("Spanning result for " + cuboidId + "(" + Long.toBinaryString(cuboidId) + "): " + toString(spannings)); for (long child : spannings) { assertTrue(scheduler.isValid(child)); } } long[] spanningsArray = Longs.toArray(totalSpanning); Arrays.parallelSort(spanningsArray); Arrays.parallelSort(expectChildren); assertArrayEquals(expectChildren, spanningsArray); }
Example #2
Source File: FlinkCubingByLayer.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
@Override public void open(Configuration parameters) throws Exception { KylinConfig kConfig = AbstractHadoopJob.loadKylinConfigFromHdfs(conf, metaUrl); try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(kConfig)) { CubeInstance cubeInstance = CubeManager.getInstance(kConfig).getCube(cubeName); CubeDesc cubeDesc = cubeInstance.getDescriptor(); CubeSegment cubeSegment = cubeInstance.getSegmentById(segmentId); CubeJoinedFlatTableEnrich interDesc = new CubeJoinedFlatTableEnrich( EngineFactory.getJoinedFlatTableDesc(cubeSegment), cubeDesc); long baseCuboidId = Cuboid.getBaseCuboidId(cubeDesc); Cuboid baseCuboid = Cuboid.findForMandatory(cubeDesc, baseCuboidId); baseCuboidBuilder = new BaseCuboidBuilder(kConfig, cubeDesc, cubeSegment, interDesc, AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid), MeasureIngester.create(cubeDesc.getMeasures()), cubeSegment.buildDictionaryMap()); } }
Example #3
Source File: AggregationGroupRuleTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
@Test public void testGoodDesc() throws IOException { AggregationGroupRule rule = getAggregationGroupRule(); for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA + "/cube_desc/").listFiles()) { if (!f.getName().endsWith("json")) { continue; } CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class); desc.init(getTestConfig()); ValidateContext vContext = new ValidateContext(); rule.validate(desc, vContext); //vContext.print(System.out); assertTrue(vContext.getResults().length == 0); } }
Example #4
Source File: QueryGenerator.java From kylin with Apache License 2.0 | 6 votes |
public static String generateQuery(CubeDesc cubeDesc, Set<BitSet> selected, int maxNumOfDimension) { IJoinedFlatTableDesc flatDesc = new CubeJoinedFlatTableDesc(cubeDesc); String dimensionStatement = createDimensionStatement(cubeDesc.getDimensions(), selected, maxNumOfDimension); String measureStatement = createMeasureStatement(cubeDesc.getMeasures()); StringBuilder sql = new StringBuilder(); sql.append("SELECT" + "\n"); sql.append(dimensionStatement); sql.append(measureStatement); StringBuilder joinPart = new StringBuilder(); JoinedFlatTable.appendJoinStatement(flatDesc, joinPart, false, null); sql.append(joinPart.toString().replaceAll("DEFAULT\\.", "")); sql.append("GROUP BY" + "\n"); sql.append(dimensionStatement); String ret = sql.toString(); ret = ret.replaceAll("`", "\""); return ret; }
Example #5
Source File: CubeHFileMapper.java From Kylin with Apache License 2.0 | 6 votes |
public KeyValueCreator(CubeDesc cubeDesc, HBaseColumnDesc colDesc) { cfBytes = Bytes.toBytes(colDesc.getColumnFamilyName()); qBytes = Bytes.toBytes(colDesc.getQualifier()); timestamp = System.currentTimeMillis(); List<MeasureDesc> measures = cubeDesc.getMeasures(); String[] measureNames = getMeasureNames(cubeDesc); String[] refs = colDesc.getMeasureRefs(); refIndex = new int[refs.length]; refMeasures = new MeasureDesc[refs.length]; for (int i = 0; i < refs.length; i++) { refIndex[i] = indexOf(measureNames, refs[i]); refMeasures[i] = measures.get(refIndex[i]); } codec = new MeasureCodec(refMeasures); colValues = new Object[refs.length]; isFullCopy = true; for (int i = 0; i < measures.size(); i++) { if (refIndex.length <= i || refIndex[i] != i) isFullCopy = false; } }
Example #6
Source File: CuboidTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
@Test public void testIsValid2() { CubeDesc cube = getTestKylinCubeWithoutSeller(); CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); assertEquals(false, cuboidScheduler.isValid(toLong("111111111"))); // base assertEquals(false, cuboidScheduler.isValid(0)); assertEquals(true, cuboidScheduler.isValid(toLong("11111111"))); // aggregation group & zero tail assertEquals(true, cuboidScheduler.isValid(toLong("10000111"))); assertEquals(false, cuboidScheduler.isValid(toLong("10001111"))); assertEquals(false, cuboidScheduler.isValid(toLong("11001111"))); assertEquals(true, cuboidScheduler.isValid(toLong("10000001"))); assertEquals(true, cuboidScheduler.isValid(toLong("10000101"))); // hierarchy assertEquals(true, cuboidScheduler.isValid(toLong("10100000"))); assertEquals(true, cuboidScheduler.isValid(toLong("10110000"))); assertEquals(true, cuboidScheduler.isValid(toLong("10111000"))); assertEquals(false, cuboidScheduler.isValid(toLong("10001000"))); assertEquals(false, cuboidScheduler.isValid(toLong("10011000"))); }
Example #7
Source File: CubeManager.java From kylin with Apache License 2.0 | 6 votes |
public CubeInstance createCube(String cubeName, String projectName, CubeDesc desc, String owner) throws IOException { try (AutoLock lock = cubeMapLock.lockForWrite()) { logger.info("Creating cube '{}-->{}' from desc '{}'", projectName, cubeName, desc.getName()); // save cube resource CubeInstance cube = CubeInstance.create(cubeName, desc); cube.setOwner(owner); updateCubeWithRetry(new CubeUpdate(cube), 0); ProjectManager.getInstance(config).moveRealizationToProject(RealizationType.CUBE, cubeName, projectName, owner); return cube; } }
Example #8
Source File: TableSchemaUpdaterTest.java From kylin with Apache License 2.0 | 6 votes |
@Test public void testDealWithMappingForCubeDesc() throws IOException { CubeDescManager cubeDescManager = CubeDescManager.getInstance(getTestConfig()); CubeDesc cubeDesc = cubeDescManager.getCubeDesc("ci_left_join_cube"); CubeDesc updated = TableSchemaUpdater.dealWithMappingForCubeDesc(cubeDesc, mappings); updated = reinit(updated, cubeDescManager.CUBE_DESC_SERIALIZER); try (DataInputStream bis = new DataInputStream( new FileInputStream(new File(mappingRootPath + CubeDesc.concatResourcePath(updated.getName()))))) { CubeDesc expected = cubeDescManager.CUBE_DESC_SERIALIZER.deserialize(bis); Assert.assertTrue(expected.equalsRaw(updated)); } catch (Exception e) { Assert.fail("CubeDesc is not updated correctly"); } }
Example #9
Source File: CuboidSchedulerTest.java From Kylin with Apache License 2.0 | 6 votes |
@Test public void testFindSmallerSibling2() { CubeDesc cube = getTestKylinCubeWithSeller(); CuboidScheduler scheduler = new CuboidScheduler(cube); Collection<Long> siblings; siblings = scheduler.findSmallerSibling(511); assertEquals("[]", siblings.toString()); siblings = scheduler.findSmallerSibling(toLong("110111111")); assertEquals("[383]", siblings.toString()); siblings = scheduler.findSmallerSibling(toLong("101110111")); assertEquals("[319]", siblings.toString()); siblings = scheduler.findSmallerSibling(toLong("111111000")); assertEquals("[]", siblings.toString()); siblings = scheduler.findSmallerSibling(toLong("111111000")); assertEquals("[]", siblings.toString()); siblings = scheduler.findSmallerSibling(toLong("110000000")); assertEquals("[288, 320]", sortToString(siblings)); }
Example #10
Source File: CuboidTest.java From Kylin with Apache License 2.0 | 6 votes |
@Test public void testCuboid2() { CubeDesc cube = getTestKylinCubeWithSeller(); Cuboid cuboid; cuboid = Cuboid.findById(cube, 0); assertEquals(toLong("100100000"), cuboid.getId()); cuboid = Cuboid.findById(cube, 1); assertEquals(toLong("100000111"), cuboid.getId()); cuboid = Cuboid.findById(cube, toLong("010")); assertEquals(toLong("100000111"), cuboid.getId()); cuboid = Cuboid.findById(cube, toLong("0100000")); assertEquals(toLong("100100000"), cuboid.getId()); cuboid = Cuboid.findById(cube, toLong("1001000")); assertEquals(toLong("101111000"), cuboid.getId()); cuboid = Cuboid.findById(cube, toLong("1000111")); assertEquals(toLong("101000111"), cuboid.getId()); cuboid = Cuboid.findById(cube, toLong("111111111")); assertEquals(toLong("111111111"), cuboid.getId()); }
Example #11
Source File: MergeDictionaryStep.java From kylin with Apache License 2.0 | 6 votes |
/** * For the new segment, we need to create new dimension dictionaries by merging underlying * dictionaries. (https://issues.apache.org/jira/browse/KYLIN-2457, https://issues.apache.org/jira/browse/KYLIN-2800) * @param cube * @param newSeg * @throws IOException */ private void makeDictForNewSegment(KylinConfig conf, CubeInstance cube, CubeSegment newSeg, List<CubeSegment> mergingSegments) throws IOException { DictionaryManager dictMgr = DictionaryManager.getInstance(conf); CubeDesc cubeDesc = cube.getDescriptor(); for (TblColRef col : cubeDesc.getAllColumnsNeedDictionaryBuilt()) { logger.info("Merging fact table dictionary on : " + col); List<DictionaryInfo> dictInfos = new ArrayList<DictionaryInfo>(); for (CubeSegment segment : mergingSegments) { logger.info("Including fact table dictionary of segment : " + segment); if (segment.getDictResPath(col) != null) { DictionaryInfo dictInfo = dictMgr.getDictionaryInfo(segment.getDictResPath(col)); if (dictInfo != null && !dictInfos.contains(dictInfo)) { dictInfos.add(dictInfo); } else { logger.warn("Failed to load DictionaryInfo from " + segment.getDictResPath(col)); } } } mergeDictionaries(dictMgr, newSeg, dictInfos, col); } }
Example #12
Source File: TopNMeasureTypeTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
@Test public void test() { CubeDesc desc = CubeDescManager.getInstance(getTestConfig()).getCubeDesc("test_kylin_cube_without_slr_left_join_desc"); MeasureDesc topSellerMeasure = null; for (MeasureDesc measureDesc : desc.getMeasures()) { if (measureDesc.getName().equals("TOP_SELLER")) { topSellerMeasure = measureDesc; break; } } TopNMeasureType measureType = (TopNMeasureType) MeasureTypeFactory.create(topSellerMeasure.getFunction().getExpression(), topSellerMeasure.getFunction().getReturnDataType()); topSellerMeasure.getFunction().getConfiguration().clear(); List<TblColRef> colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); assertTrue(colsNeedDict != null && colsNeedDict.size() == 1); TblColRef sellerColRef = topSellerMeasure.getFunction().getParameter().getColRefs().get(1); topSellerMeasure.getFunction().getConfiguration().put(TopNMeasureType.CONFIG_ENCODING_PREFIX + sellerColRef.getIdentity(), "int:6"); colsNeedDict = measureType.getColumnsNeedDictionary(topSellerMeasure.getFunction()); assertTrue(colsNeedDict.size() == 0); }
Example #13
Source File: CubeDescManager.java From kylin with Apache License 2.0 | 6 votes |
@Override public void onEntityChange(Broadcaster broadcaster, String entity, Event event, String cacheKey) throws IOException { String cubeDescName = cacheKey; CubeDesc cubeDesc = getCubeDesc(cubeDescName); String modelName = cubeDesc == null ? null : cubeDesc.getModelName(); if (event == Event.DROP) removeLocalCubeDesc(cubeDescName); else reloadCubeDescQuietly(cubeDescName); for (ProjectInstance prj : ProjectManager.getInstance(config).findProjectsByModel(modelName)) { broadcaster.notifyProjectSchemaUpdate(prj.getName()); } }
Example #14
Source File: HBaseLookupMRSteps.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
public void addMaterializeLookupTablesSteps(LookupMaterializeContext context) { CubeDesc cubeDesc = cube.getDescriptor(); Set<String> allLookupTables = Sets.newHashSet(); for (DimensionDesc dim : cubeDesc.getDimensions()) { TableRef table = dim.getTableRef(); if (cubeDesc.getModel().isLookupTable(table)) { allLookupTables.add(table.getTableIdentity()); } } List<SnapshotTableDesc> snapshotTableDescs = cubeDesc.getSnapshotTableDescList(); for (SnapshotTableDesc snapshotTableDesc : snapshotTableDescs) { if (ExtTableSnapshotInfo.STORAGE_TYPE_HBASE.equals(snapshotTableDesc.getStorageType()) && allLookupTables.contains(snapshotTableDesc.getTableName())) { addMaterializeLookupTableSteps(context, snapshotTableDesc.getTableName(), snapshotTableDesc); } } }
Example #15
Source File: LookupTableToHFileJob.java From kylin with Apache License 2.0 | 6 votes |
private String[] getLookupKeyColumns(CubeInstance cube, String tableName) { CubeDesc cubeDesc = cube.getDescriptor(); DataModelDesc modelDesc = cubeDesc.getModel(); TableRef lookupTableRef = null; for (TableRef tableRef : modelDesc.getLookupTables()) { if (tableRef.getTableIdentity().equalsIgnoreCase(tableName)) { lookupTableRef = tableRef; break; } } if (lookupTableRef == null) { throw new IllegalStateException("cannot find table in model:" + tableName); } JoinDesc joinDesc = modelDesc.getJoinByPKSide(lookupTableRef); TblColRef[] keyColRefs = joinDesc.getPrimaryKeyColumns(); String[] result = new String[keyColRefs.length]; for (int i = 0; i < keyColRefs.length; i++) { result[i] = keyColRefs[i].getName(); } return result; }
Example #16
Source File: CubeMetadataUpgrade.java From Kylin with Apache License 2.0 | 6 votes |
private void upgradeCubeDesc() { logger.info("Reloading Cube Metadata from folder " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT)); List<String> paths = listResourceStore(ResourceStore.CUBE_DESC_RESOURCE_ROOT); for (String path : paths) { try { CubeDescUpgrader upgrade = new CubeDescUpgrader(path); CubeDesc ndesc = upgrade.upgrade(); ndesc.setSignature(ndesc.calculateSignature()); getStore().putResource(ndesc.getModel().getResourcePath(), ndesc.getModel(), MetadataManager.MODELDESC_SERIALIZER); getStore().putResource(ndesc.getResourcePath(), ndesc, CubeDescManager.CUBE_DESC_SERIALIZER); updatedResources.add(ndesc.getResourcePath()); } catch (IOException e) { e.printStackTrace(); errorMsgs.add("Upgrade CubeDesc at '" + path + "' failed: " + e.getLocalizedMessage()); } } }
Example #17
Source File: Cuboid.java From Kylin with Apache License 2.0 | 6 votes |
private static long translateToValidCuboid(CubeDesc cube, long cuboidID) { if (Cuboid.isValid(cube, cuboidID)) { return cuboidID; } HashSet<Long> dedupped = new HashSet<Long>(); Queue<Long> queue = new LinkedList<Long>(); List<Long> parents = Cuboid.getAllPossibleParents(cube, cuboidID); // check each parent addToQueue(queue, parents, dedupped); while (queue.size() > 0) { long parent = pollFromQueue(queue, dedupped); if (Cuboid.isValid(cube, parent)) { return parent; } else { addToQueue(queue, Cuboid.getAllPossibleParents(cube, parent), dedupped); } } return -1; }
Example #18
Source File: DictionaryRuleTest.java From kylin with Apache License 2.0 | 6 votes |
private void testDictionaryDesc(String expectMessage, DictionaryDesc... descs) throws IOException { DictionaryRule rule = new DictionaryRule(); File f = new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/test_kylin_cube_without_slr_left_join_desc.json"); CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class); List<DictionaryDesc> newDicts = Lists.newArrayList(desc.getDictionaries()); for (DictionaryDesc dictDesc : descs) { newDicts.add(dictDesc); } desc.setDictionaries(newDicts); desc.init(config); ValidateContext vContext = new ValidateContext(); rule.validate(desc, vContext); if (expectMessage == null) { assertTrue(vContext.getResults().length == 0); } else { assertTrue(vContext.getResults().length == 1); String actualMessage = vContext.getResults()[0].getMessage(); assertTrue(actualMessage.startsWith(expectMessage)); } }
Example #19
Source File: CuboidTest.java From kylin with Apache License 2.0 | 6 votes |
@Test public void testFindCuboidByIdWithSingleAggrGroup2() { CubeDesc cube = getTestKylinCubeWithSeller(); Cuboid cuboid; cuboid = Cuboid.findById(cube, 0); assertEquals(toLong("101000000"), cuboid.getId()); cuboid = Cuboid.findById(cube, 1); assertEquals(toLong("100000111"), cuboid.getId()); cuboid = Cuboid.findById(cube, toLong("010")); assertEquals(toLong("100000111"), cuboid.getId()); cuboid = Cuboid.findById(cube, toLong("0100000")); assertEquals(toLong("100100000"), cuboid.getId()); cuboid = Cuboid.findById(cube, toLong("1001000")); assertEquals(toLong("101111000"), cuboid.getId()); cuboid = Cuboid.findById(cube, toLong("1000111")); assertEquals(toLong("101000111"), cuboid.getId()); cuboid = Cuboid.findById(cube, toLong("111111111")); assertEquals(toLong("111111111"), cuboid.getId()); }
Example #20
Source File: CuboidSchedulerTest.java From kylin with Apache License 2.0 | 5 votes |
private CubeDesc getFiftyDimFiveCapCubeDesc() { File metaFile = new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc/fifty_dim_five_cap.json.bad"); assertTrue(metaFile.exists()); String path = metaFile.getPath(); metaFile.renameTo(new File(path.substring(0, path.length() - 4))); return CubeDescManager.getInstance(getTestConfig()).getCubeDesc("fifty_dim_five_cap"); }
Example #21
Source File: UpdateCubeAfterSnapshotStep.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Override protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException { KylinConfig kylinConfig = context.getConfig(); CubeManager cubeManager = CubeManager.getInstance(kylinConfig); CubeInstance cube = cubeManager.getCube(LookupExecutableUtil.getCubeName(this.getParams())); List<String> segmentIDs = LookupExecutableUtil.getSegments(this.getParams()); String lookupTableName = LookupExecutableUtil.getLookupTableName(this.getParams()); String extLookupSnapshotStr = this.getParam(BatchConstants.ARG_EXT_LOOKUP_SNAPSHOTS_INFO); if (extLookupSnapshotStr == null || extLookupSnapshotStr.isEmpty()) { return new ExecuteResult(); } Map<String, String> extLookupSnapshotMap = LookupMaterializeContext.parseLookupSnapshots(extLookupSnapshotStr); String snapshotResPath = extLookupSnapshotMap.get(lookupTableName); if (snapshotResPath == null) { logger.info("no snapshot path exist in the context, so no need to update snapshot path"); return new ExecuteResult(); } CubeDesc cubeDesc = cube.getDescriptor(); try { logger.info("update snapshot path:{} to cube:{}", snapshotResPath, cube.getName()); if (cubeDesc.isGlobalSnapshotTable(lookupTableName)) { if (!snapshotResPath.equals(cube.getSnapshotResPath(lookupTableName))) { LookupExecutableUtil.updateSnapshotPathToCube(cubeManager, cube, lookupTableName, snapshotResPath); } } else { LookupExecutableUtil.updateSnapshotPathToSegments(cubeManager, cube, segmentIDs, lookupTableName, snapshotResPath); } return new ExecuteResult(); } catch (IOException e) { logger.error("fail to save cuboid statistics", e); return ExecuteResult.createError(e); } }
Example #22
Source File: MigrationRuleSet.java From kylin with Apache License 2.0 | 5 votes |
@Override public void apply(Context ctx) throws RuleValidationException { CubeDesc cubeDesc = ctx.getCubeInstance().getDescriptor(); List<String> notifyList = cubeDesc.getNotifyList(); if (notifyList == null || notifyList.size() == 0) { throw new RuleValidationException("Cube email notification list is not set or empty."); } }
Example #23
Source File: DataController.java From kylin with Apache License 2.0 | 5 votes |
private Set<FunctionDesc> convertMetrics(CubeDesc cubeDesc, List<FunctionDesc> metrics) { Set<FunctionDesc> result = Sets.newHashSet(); for (FunctionDesc metric : metrics) { result.add(findAggrFuncFromCubeDesc(cubeDesc, metric)); } return result; }
Example #24
Source File: SparkFactDistinct.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
private void initDictColDeduper(CubeDesc cubeDesc) { // setup dict col deduper dictColDeduper = new DictColDeduper(); Set<TblColRef> dictCols = cubeDesc.getAllColumnsNeedDictionaryBuilt(); for (int i = 0; i < allCols.size(); i++) { if (dictCols.contains(allCols.get(i))) dictColDeduper.setIsDictCol(i); } }
Example #25
Source File: CuboidSchedulerTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Test(expected=RuntimeException.class) public void testTooManyCombination() { File twentyFile = new File(new File(LocalFileMetadataTestCase.LOCALMETA_TEMP_DATA, "cube_desc"), "twenty_dim"); twentyFile.renameTo(new File(twentyFile.getPath().substring(0, twentyFile.getPath().length() - 4))); CubeDesc cube = getTwentyDimCubeDesc(); CuboidScheduler cuboidScheduler = cube.getInitialCuboidScheduler(); cuboidScheduler.getCuboidCount(); twentyFile.renameTo(new File(twentyFile.getPath() + ".bad")); }
Example #26
Source File: SparkFactDistinct.java From kylin with Apache License 2.0 | 5 votes |
private void initDictColDeduper(CubeDesc cubeDesc) { // setup dict col deduper dictColDeduper = new DictColDeduper(); Set<TblColRef> dictCols = cubeDesc.getAllColumnsNeedDictionaryBuilt(); for (int i = 0; i < allCols.size(); i++) { if (dictCols.contains(allCols.get(i))) dictColDeduper.setIsDictCol(i); } }
Example #27
Source File: CacheControllerTest.java From kylin with Apache License 2.0 | 5 votes |
@Test public void testClearCacheForCubeMigration() throws IOException { KylinConfig config = KylinConfig.getInstanceFromEnv(); String CUBENAME = "test_kylin_cube_without_slr_desc"; CubeDescManager cubeDescManager = CubeDescManager.getInstance(config); CubeDesc cubeDesc = cubeDescManager.getCubeDesc(CUBENAME); DataModelDesc modelDesc = cubeDesc.getModel(); Map<String, String> tableToProjects = new HashMap<>(); for (TableRef tableRef : modelDesc.getAllTables()) { tableToProjects.put(tableRef.getTableIdentity(), tableRef.getTableDesc().getProject()); } String uuid = cubeDesc.getUuid(); String signature = cubeDesc.getSignature(); assertEquals(cubeDesc.getRetentionRange(), 0); //update cubeDesc cubeDesc.setRetentionRange(2018); cubeDesc.updateRandomUuid(); //directly update metadata in store to simulate cube migration Serializer<CubeDesc> cubeDescSerializer = new JsonSerializer<CubeDesc>(CubeDesc.class); getStore().checkAndPutResource(cubeDesc.getResourcePath(), cubeDesc, cubeDescSerializer); CubeMigrationRequest request = new CubeMigrationRequest(); request.setCube(cubeDesc.getName()); request.setModel(modelDesc.getName()); request.setProject(modelDesc.getProject()); request.setTableToProjects(tableToProjects); cacheController.clearCacheForCubeMigration(request); assertEquals(2018, cubeDescManager.getCubeDesc(CUBENAME).getRetentionRange()); assertEquals(signature, cubeDescManager.getCubeDesc(CUBENAME).getSignature()); assertNotEquals(uuid, cubeDescManager.getCubeDesc(CUBENAME).getUuid()); }
Example #28
Source File: CubeScanRangePlanner.java From kylin with Apache License 2.0 | 5 votes |
private Set<TblColRef> replaceDerivedColumns(Set<TblColRef> input, CubeDesc cubeDesc) { Set<TblColRef> ret = Sets.newHashSet(); for (TblColRef col : input) { if (cubeDesc.hasHostColumn(col)) { for (TblColRef host : cubeDesc.getHostInfo(col).columns) { ret.add(host); } } else { ret.add(col); } } return ret; }
Example #29
Source File: RowRecordReader.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
public RowRecordReader(CubeDesc cubeDesc, Path path, FileSystem fileSystem) throws IOException { super(fileSystem, path); this.cubeDesc = cubeDesc; this.folderPath = path; this.fs = fileSystem; this.currentRowRecord = new RowRecord(); initReaders(); }
Example #30
Source File: DictionaryRuleTest.java From kylin with Apache License 2.0 | 5 votes |
@Test public void testGoodDesc() throws IOException { DictionaryRule rule = new DictionaryRule(); for (File f : new File(LocalFileMetadataTestCase.LOCALMETA_TEST_DATA + "/cube_desc/").listFiles()) { if (!f.getName().endsWith("json")) { continue; } CubeDesc desc = JsonUtil.readValue(new FileInputStream(f), CubeDesc.class); desc.init(config); ValidateContext vContext = new ValidateContext(); rule.validate(desc, vContext); assertTrue(vContext.getResults().length == 0); } }