Java Code Examples for org.apache.kylin.common.persistence.ResourceStore#getStore()
The following examples show how to use
org.apache.kylin.common.persistence.ResourceStore#getStore() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CubeStatsReader.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
/** * @param cuboidScheduler if it's null, part of it's functions will not be supported */ public CubeStatsReader(CubeSegment cubeSegment, CuboidScheduler cuboidScheduler, KylinConfig kylinConfig) throws IOException { ResourceStore store = ResourceStore.getStore(kylinConfig); String statsKey = cubeSegment.getStatisticsResourcePath(); RawResource resource = store.getResource(statsKey); if (resource == null) throw new IllegalStateException("Missing resource at " + statsKey); File tmpSeqFile = writeTmpSeqFile(resource.content()); Path path = new Path(HadoopUtil.fixWindowsPath("file://" + tmpSeqFile.getAbsolutePath())); CubeStatsResult cubeStatsResult = new CubeStatsResult(path, kylinConfig.getCubeStatsHLLPrecision()); tmpSeqFile.delete(); this.seg = cubeSegment; this.cuboidScheduler = cuboidScheduler; this.samplingPercentage = cubeStatsResult.getPercentage(); this.mapperNumberOfFirstBuild = cubeStatsResult.getMapperNumber(); this.mapperOverlapRatioOfFirstBuild = cubeStatsResult.getMapperOverlapRatio(); this.cuboidRowEstimatesHLL = cubeStatsResult.getCounterMap(); this.sourceRowCount = cubeStatsResult.getSourceRecordCount(); }
Example 2
Source File: MetaDumpUtil.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
public static void dumpResources(KylinConfig kylinConfig, String metaOutDir, Set<String> dumpList) throws IOException { long startTime = System.currentTimeMillis(); ResourceStore from = ResourceStore.getStore(kylinConfig); KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaOutDir); ResourceStore to = ResourceStore.getStore(localConfig); final String[] tolerantResources = { "/table_exd" }; for (String path : dumpList) { RawResource res = from.getResource(path); if (res == null) { if (StringUtils.startsWithAny(path, tolerantResources)) { continue; } else { throw new IllegalStateException("No resource found at -- " + path); } } to.putResource(path, res.content(), res.lastModified()); res.content().close(); } logger.debug("Dump resources to {} took {} ms", metaOutDir, System.currentTimeMillis() - startTime); }
Example 3
Source File: SparkExecutable.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private void attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws IOException { Set<String> dumpList = new LinkedHashSet<>( JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance())); ResourceStore rs = ResourceStore.getStore(segments.get(0).getConfig()); for (CubeSegment segment : segments) { dumpList.addAll(segment.getDictionaryPaths()); if (rs.exists(segment.getStatisticsResourcePath())) { // cube statistics is not available for new segment dumpList.add(segment.getStatisticsResourcePath()); } //tiretree global domain dic CubeDescTiretreeGlobalDomainDictUtil.cuboidJob(segment.getCubeDesc(), dumpList); } JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segments.get(0).getConfig(), this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt())); }
Example 4
Source File: MigrationRuleSet.java From kylin with Apache License 2.0 | 6 votes |
public Context(QueryService queryService, CubeInstance cubeInstance, String targetHost, String tgtProjectName) { this.queryService = queryService; this.cubeInstance = cubeInstance; this.targetAddress = targetHost; KylinConfig targetConfig = KylinConfig.createInstanceFromUri(targetHost); this.targetResourceStore = ResourceStore.getStore(targetConfig); this.tgtProjectName = tgtProjectName; List<ProjectInstance> projList = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv()) .findProjects(cubeInstance.getType(), cubeInstance.getName()); if (projList.size() != 1) { throw new InternalErrorException("Cube " + cubeInstance.getName() + " should belong to only one project. However, it's belong to " + projList); } this.srcProjectName = projList.get(0).getName(); }
Example 5
Source File: ModelDataGenerator.java From kylin with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws IOException { String modelName = args[0]; int nRows = Integer.parseInt(args[1]); String outputDir = args.length > 2 ? args[2] : null; KylinConfig conf = KylinConfig.getInstanceFromEnv(); DataModelDesc model = DataModelManager.getInstance(conf).getDataModelDesc(modelName); ResourceStore store = outputDir == null ? ResourceStore.getStore(conf) : ResourceStore.getStore(mockup(outputDir)); ModelDataGenerator gen = new ModelDataGenerator(model, nRows, store); gen.generate(); }
Example 6
Source File: ExtendCubeToHybridCLI.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
public ExtendCubeToHybridCLI() { this.kylinConfig = KylinConfig.getInstanceFromEnv(); this.store = ResourceStore.getStore(kylinConfig); this.cubeManager = CubeManager.getInstance(kylinConfig); this.cubeDescManager = CubeDescManager.getInstance(kylinConfig); this.metadataManager = DataModelManager.getInstance(kylinConfig); }
Example 7
Source File: FlinkExecutable.java From kylin with Apache License 2.0 | 5 votes |
private void attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws IOException { Set<String> dumpList = new LinkedHashSet<>(); dumpList.addAll(JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance())); ResourceStore rs = ResourceStore.getStore(segments.get(0).getConfig()); for (CubeSegment segment : segments) { dumpList.addAll(segment.getDictionaryPaths()); if (rs.exists(segment.getStatisticsResourcePath())) { // cube statistics is not available for new segment dumpList.add(segment.getStatisticsResourcePath()); } } JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segments.get(0).getConfig(), this.getParam(FlinkCubingByLayer.OPTION_META_URL.getOpt())); }
Example 8
Source File: CubeDescUpgrader.java From Kylin with Apache License 2.0 | 4 votes |
protected static ResourceStore getStore() { return ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); }
Example 9
Source File: ITJDBCResourceStoreTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
@Test public void testPerformanceWithResourceTool() throws Exception { Assume.assumeTrue(jdbcConnectable); KylinConfig tmpConfig = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv()); tmpConfig.setMetadataUrl(copyIdentifier + jdbcMetadataUrlNoIdentifier); JDBCResourceStore store = (JDBCResourceStore) ResourceStore.getStore(kylinConfig); NavigableSet<String> executes = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT); NavigableSet<String> executeOutputs = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT); long startTs = System.currentTimeMillis(); for (String execute : executes) { String uuid = StringUtils.substringAfterLast(execute, "/"); RawResource executeResource = store.getResource(execute); Map<String, RawResource> executeOutputResourceMap = new HashMap<>(); for (String executeOutput : executeOutputs) { if (executeOutput.contains(uuid)) { RawResource executeOutputResource = store.getResource(executeOutput); executeOutputResourceMap.put(executeOutput, executeOutputResource); } } for (int i = 0; i < 200; i++) { String newUuid = UUID.randomUUID().toString(); store.putResource(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + newUuid, executeResource.content(), System.currentTimeMillis()); for (String key : executeOutputResourceMap.keySet()) { String step = StringUtils.substringAfterLast(key, uuid); store.putResource(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + newUuid + step, executeOutputResourceMap.get(key).content(), System.currentTimeMillis()); } } } long queryNumBeforeCopy = store.getQueriedSqlNum(); new ResourceTool().copy(kylinConfig, tmpConfig); long endTs = System.currentTimeMillis(); long queryNumAfterCopy = store.getQueriedSqlNum(); JDBCResourceStore resourceStoreCopy = (JDBCResourceStore) ResourceStore.getStore(tmpConfig); int executeNum = store.listResources("/execute").size(); int executeOutputNum = store.listResources("/execute_output").size(); assertEquals(executeNum, resourceStoreCopy.listResources("/execute").size()); assertEquals(executeOutputNum, resourceStoreCopy.listResources("/execute_output").size()); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT); String startTime = sdf.format(new Date(Long.parseLong(String.valueOf(startTs)))); String endTime = sdf.format(new Date(Long.parseLong(String.valueOf(endTs)))); logger.info("Test performance with ResourceTool done during " + startTime + " to " + endTime); logger.info("Now there is " + executeNum + " execute data and " + executeOutputNum + " execute_output data in resource store."); logger.info("Resource store run " + queryNumBeforeCopy + " sqls for metadata generation, and " + (queryNumAfterCopy - queryNumBeforeCopy) + " sqls for copy with ResourceTool."); assertTrue((queryNumAfterCopy - queryNumBeforeCopy) < queryNumBeforeCopy); logger.info("This test is expected to be done in 10 mins."); assertTrue((endTs - startTs) < 600000); }
Example 10
Source File: MetadataUpgradeTest.java From Kylin with Apache License 2.0 | 4 votes |
@Test public void testTableDescUpgrade() throws Exception { MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv()); TableDesc fact = metaMgr.getTableDesc("default.test_kylin_fact"); @SuppressWarnings("deprecation") String oldResLocation = fact.getResourcePathV1(); String newResLocation = fact.getResourcePath(); ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv()); Assert.assertTrue(store.exists(newResLocation)); Assert.assertTrue(!store.exists(oldResLocation)); String oldExdResLocation = TableDesc.concatExdResourcePath("test_kylin_fact".toUpperCase()); String newExdResLocation = TableDesc.concatExdResourcePath("default.test_kylin_fact".toUpperCase()); Assert.assertTrue(store.exists(newExdResLocation)); Assert.assertTrue(!store.exists(oldExdResLocation)); }
Example 11
Source File: StreamingSourceConfigManager.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
private ResourceStore getStore() { return ResourceStore.getStore(this.config); }
Example 12
Source File: MetadataCleanupJob.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
private ResourceStore getStore() { return ResourceStore.getStore(config); }
Example 13
Source File: StreamingManager.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
private ResourceStore getStore() { return ResourceStore.getStore(this.config); }
Example 14
Source File: TempStatementManager.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
private ResourceStore getStore() { return ResourceStore.getStore(this.config); }
Example 15
Source File: ToolUtil.java From kylin with Apache License 2.0 | 4 votes |
public static String getMetaStoreId() throws IOException { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ResourceStore store = ResourceStore.getStore(kylinConfig); return store.getMetaStoreUUID(); }
Example 16
Source File: ProjectManager.java From kylin with Apache License 2.0 | 4 votes |
ResourceStore getStore() { return ResourceStore.getStore(this.config); }
Example 17
Source File: IIManager.java From Kylin with Apache License 2.0 | 4 votes |
private ResourceStore getStore() { return ResourceStore.getStore(this.config); }
Example 18
Source File: CubeMetadataUpgrade.java From Kylin with Apache License 2.0 | 4 votes |
private ResourceStore getStore() { return ResourceStore.getStore(config); }
Example 19
Source File: QueryService.java From kylin-on-parquet-v2 with Apache License 2.0 | 4 votes |
public QueryService() { queryStore = ResourceStore.getStore(getConfig()); preparedContextPool = createPreparedContextPool(); badQueryDetector.start(); }
Example 20
Source File: MetadataCleanupJob.java From kylin with Apache License 2.0 | 4 votes |
private ResourceStore getStore() { return ResourceStore.getStore(config); }