Java Code Examples for org.apache.kylin.common.persistence.ResourceStore#getStore()

The following examples show how to use org.apache.kylin.common.persistence.ResourceStore#getStore() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CubeStatsReader.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
/**
 * @param cuboidScheduler if it's null, part of it's functions will not be supported
 */
public CubeStatsReader(CubeSegment cubeSegment, CuboidScheduler cuboidScheduler, KylinConfig kylinConfig)
        throws IOException {
    ResourceStore store = ResourceStore.getStore(kylinConfig);
    String statsKey = cubeSegment.getStatisticsResourcePath();
    RawResource resource = store.getResource(statsKey);
    if (resource == null)
        throw new IllegalStateException("Missing resource at " + statsKey);

    File tmpSeqFile = writeTmpSeqFile(resource.content());
    Path path = new Path(HadoopUtil.fixWindowsPath("file://" + tmpSeqFile.getAbsolutePath()));

    CubeStatsResult cubeStatsResult = new CubeStatsResult(path, kylinConfig.getCubeStatsHLLPrecision());
    tmpSeqFile.delete();

    this.seg = cubeSegment;
    this.cuboidScheduler = cuboidScheduler;
    this.samplingPercentage = cubeStatsResult.getPercentage();
    this.mapperNumberOfFirstBuild = cubeStatsResult.getMapperNumber();
    this.mapperOverlapRatioOfFirstBuild = cubeStatsResult.getMapperOverlapRatio();
    this.cuboidRowEstimatesHLL = cubeStatsResult.getCounterMap();
    this.sourceRowCount = cubeStatsResult.getSourceRecordCount();
}
 
Example 2
Source File: MetaDumpUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public static void dumpResources(KylinConfig kylinConfig, String metaOutDir, Set<String> dumpList)
        throws IOException {
    long startTime = System.currentTimeMillis();

    ResourceStore from = ResourceStore.getStore(kylinConfig);
    KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaOutDir);
    ResourceStore to = ResourceStore.getStore(localConfig);
    final String[] tolerantResources = { "/table_exd" };

    for (String path : dumpList) {
        RawResource res = from.getResource(path);
        if (res == null) {
            if (StringUtils.startsWithAny(path, tolerantResources)) {
                continue;
            } else {
                throw new IllegalStateException("No resource found at -- " + path);
            }
        }
        to.putResource(path, res.content(), res.lastModified());
        res.content().close();
    }

    logger.debug("Dump resources to {} took {} ms", metaOutDir, System.currentTimeMillis() - startTime);
}
 
Example 3
Source File: SparkExecutable.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private void attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>(
            JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance()));
    ResourceStore rs = ResourceStore.getStore(segments.get(0).getConfig());
    for (CubeSegment segment : segments) {
        dumpList.addAll(segment.getDictionaryPaths());
        if (rs.exists(segment.getStatisticsResourcePath())) {
            // cube statistics is not available for new segment
            dumpList.add(segment.getStatisticsResourcePath());
        }
        //tiretree global domain dic
        CubeDescTiretreeGlobalDomainDictUtil.cuboidJob(segment.getCubeDesc(), dumpList);
    }
    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segments.get(0).getConfig(),
            this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt()));
}
 
Example 4
Source File: MigrationRuleSet.java    From kylin with Apache License 2.0 6 votes vote down vote up
public Context(QueryService queryService, CubeInstance cubeInstance, String targetHost, String tgtProjectName) {
    this.queryService = queryService;
    this.cubeInstance = cubeInstance;
    this.targetAddress = targetHost;
    KylinConfig targetConfig = KylinConfig.createInstanceFromUri(targetHost);
    this.targetResourceStore = ResourceStore.getStore(targetConfig);
    this.tgtProjectName = tgtProjectName;

    List<ProjectInstance> projList = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
            .findProjects(cubeInstance.getType(), cubeInstance.getName());
    if (projList.size() != 1) {
        throw new InternalErrorException("Cube " + cubeInstance.getName()
                + " should belong to only one project. However, it's belong to " + projList);
    }
    this.srcProjectName = projList.get(0).getName();
}
 
Example 5
Source File: ModelDataGenerator.java    From kylin with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws IOException {
    String modelName = args[0];
    int nRows = Integer.parseInt(args[1]);
    String outputDir = args.length > 2 ? args[2] : null;

    KylinConfig conf = KylinConfig.getInstanceFromEnv();
    DataModelDesc model = DataModelManager.getInstance(conf).getDataModelDesc(modelName);
    ResourceStore store = outputDir == null ? ResourceStore.getStore(conf)
            : ResourceStore.getStore(mockup(outputDir));

    ModelDataGenerator gen = new ModelDataGenerator(model, nRows, store);
    gen.generate();
}
 
Example 6
Source File: ExtendCubeToHybridCLI.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public ExtendCubeToHybridCLI() {
    this.kylinConfig = KylinConfig.getInstanceFromEnv();
    this.store = ResourceStore.getStore(kylinConfig);
    this.cubeManager = CubeManager.getInstance(kylinConfig);
    this.cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    this.metadataManager = DataModelManager.getInstance(kylinConfig);
}
 
Example 7
Source File: FlinkExecutable.java    From kylin with Apache License 2.0 5 votes vote down vote up
private void attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>();
    dumpList.addAll(JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance()));
    ResourceStore rs = ResourceStore.getStore(segments.get(0).getConfig());
    for (CubeSegment segment : segments) {
        dumpList.addAll(segment.getDictionaryPaths());
        if (rs.exists(segment.getStatisticsResourcePath())) {
            // cube statistics is not available for new segment
            dumpList.add(segment.getStatisticsResourcePath());
        }
    }

    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segments.get(0).getConfig(),
            this.getParam(FlinkCubingByLayer.OPTION_META_URL.getOpt()));
}
 
Example 8
Source File: CubeDescUpgrader.java    From Kylin with Apache License 2.0 4 votes vote down vote up
protected static ResourceStore getStore() {
    return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
}
 
Example 9
Source File: ITJDBCResourceStoreTest.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up
@Test
public void testPerformanceWithResourceTool() throws Exception {
    Assume.assumeTrue(jdbcConnectable);
    KylinConfig tmpConfig = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
    tmpConfig.setMetadataUrl(copyIdentifier + jdbcMetadataUrlNoIdentifier);

    JDBCResourceStore store = (JDBCResourceStore) ResourceStore.getStore(kylinConfig);
    NavigableSet<String> executes = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
    NavigableSet<String> executeOutputs = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);

    long startTs = System.currentTimeMillis();

    for (String execute : executes) {
        String uuid = StringUtils.substringAfterLast(execute, "/");
        RawResource executeResource = store.getResource(execute);
        Map<String, RawResource> executeOutputResourceMap = new HashMap<>();

        for (String executeOutput : executeOutputs) {
            if (executeOutput.contains(uuid)) {
                RawResource executeOutputResource = store.getResource(executeOutput);
                executeOutputResourceMap.put(executeOutput, executeOutputResource);
            }
        }

        for (int i = 0; i < 200; i++) {
            String newUuid = UUID.randomUUID().toString();
            store.putResource(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + newUuid, executeResource.content(),
                    System.currentTimeMillis());

            for (String key : executeOutputResourceMap.keySet()) {
                String step = StringUtils.substringAfterLast(key, uuid);
                store.putResource(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + newUuid + step,
                        executeOutputResourceMap.get(key).content(), System.currentTimeMillis());
            }
        }
    }

    long queryNumBeforeCopy = store.getQueriedSqlNum();
    new ResourceTool().copy(kylinConfig, tmpConfig);
    long endTs = System.currentTimeMillis();
    long queryNumAfterCopy = store.getQueriedSqlNum();
    JDBCResourceStore resourceStoreCopy = (JDBCResourceStore) ResourceStore.getStore(tmpConfig);

    int executeNum = store.listResources("/execute").size();
    int executeOutputNum = store.listResources("/execute_output").size();

    assertEquals(executeNum, resourceStoreCopy.listResources("/execute").size());
    assertEquals(executeOutputNum, resourceStoreCopy.listResources("/execute_output").size());

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    String startTime = sdf.format(new Date(Long.parseLong(String.valueOf(startTs))));
    String endTime = sdf.format(new Date(Long.parseLong(String.valueOf(endTs))));

    logger.info("Test performance with ResourceTool done during " + startTime + " to " + endTime);
    logger.info("Now there is " + executeNum + " execute data and " + executeOutputNum
            + " execute_output data in resource store.");
    logger.info("Resource store run " + queryNumBeforeCopy + " sqls for metadata generation, and "
            + (queryNumAfterCopy - queryNumBeforeCopy) + " sqls for copy with ResourceTool.");
    assertTrue((queryNumAfterCopy - queryNumBeforeCopy) < queryNumBeforeCopy);
    logger.info("This test is expected to be done in 10 mins.");
    assertTrue((endTs - startTs) < 600000);
}
 
Example 10
Source File: MetadataUpgradeTest.java    From Kylin with Apache License 2.0 4 votes vote down vote up
@Test
public void testTableDescUpgrade() throws Exception {

    MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
    TableDesc fact = metaMgr.getTableDesc("default.test_kylin_fact");
    
    @SuppressWarnings("deprecation")
    String oldResLocation = fact.getResourcePathV1();
    String newResLocation = fact.getResourcePath();
    
    ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
    
    Assert.assertTrue(store.exists(newResLocation));
    Assert.assertTrue(!store.exists(oldResLocation));
    
    
    String oldExdResLocation = TableDesc.concatExdResourcePath("test_kylin_fact".toUpperCase());
    String newExdResLocation = TableDesc.concatExdResourcePath("default.test_kylin_fact".toUpperCase());
    
    Assert.assertTrue(store.exists(newExdResLocation));
    Assert.assertTrue(!store.exists(oldExdResLocation));
    
}
 
Example 11
Source File: StreamingSourceConfigManager.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up
private ResourceStore getStore() {
    return ResourceStore.getStore(this.config);
}
 
Example 12
Source File: MetadataCleanupJob.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up
private ResourceStore getStore() {
    return ResourceStore.getStore(config);
}
 
Example 13
Source File: StreamingManager.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up
private ResourceStore getStore() {
    return ResourceStore.getStore(this.config);
}
 
Example 14
Source File: TempStatementManager.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up
private ResourceStore getStore() {
    return ResourceStore.getStore(this.config);
}
 
Example 15
Source File: ToolUtil.java    From kylin with Apache License 2.0 4 votes vote down vote up
public static String getMetaStoreId() throws IOException {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    ResourceStore store = ResourceStore.getStore(kylinConfig);
    return store.getMetaStoreUUID();
}
 
Example 16
Source File: ProjectManager.java    From kylin with Apache License 2.0 4 votes vote down vote up
ResourceStore getStore() {
    return ResourceStore.getStore(this.config);
}
 
Example 17
Source File: IIManager.java    From Kylin with Apache License 2.0 4 votes vote down vote up
private ResourceStore getStore() {
    return ResourceStore.getStore(this.config);
}
 
Example 18
Source File: CubeMetadataUpgrade.java    From Kylin with Apache License 2.0 4 votes vote down vote up
private ResourceStore getStore() {
    return ResourceStore.getStore(config);
}
 
Example 19
Source File: QueryService.java    From kylin-on-parquet-v2 with Apache License 2.0 4 votes vote down vote up
public QueryService() {
    queryStore = ResourceStore.getStore(getConfig());
    preparedContextPool = createPreparedContextPool();
    badQueryDetector.start();
}
 
Example 20
Source File: MetadataCleanupJob.java    From kylin with Apache License 2.0 4 votes vote down vote up
private ResourceStore getStore() {
    return ResourceStore.getStore(config);
}