Java Code Examples for org.apache.kylin.common.persistence.ResourceStore#getStore()

/**
 * @param cuboidScheduler if it's null, part of it's functions will not be supported
 */
public CubeStatsReader(CubeSegment cubeSegment, CuboidScheduler cuboidScheduler, KylinConfig kylinConfig)
        throws IOException {
    ResourceStore store = ResourceStore.getStore(kylinConfig);
    String statsKey = cubeSegment.getStatisticsResourcePath();
    RawResource resource = store.getResource(statsKey);
    if (resource == null)
        throw new IllegalStateException("Missing resource at " + statsKey);

    File tmpSeqFile = writeTmpSeqFile(resource.content());
    Path path = new Path(HadoopUtil.fixWindowsPath("file://" + tmpSeqFile.getAbsolutePath()));

    CubeStatsResult cubeStatsResult = new CubeStatsResult(path, kylinConfig.getCubeStatsHLLPrecision());
    tmpSeqFile.delete();

    this.seg = cubeSegment;
    this.cuboidScheduler = cuboidScheduler;
    this.samplingPercentage = cubeStatsResult.getPercentage();
    this.mapperNumberOfFirstBuild = cubeStatsResult.getMapperNumber();
    this.mapperOverlapRatioOfFirstBuild = cubeStatsResult.getMapperOverlapRatio();
    this.cuboidRowEstimatesHLL = cubeStatsResult.getCounterMap();
    this.sourceRowCount = cubeStatsResult.getSourceRecordCount();
}
 

public static void dumpResources(KylinConfig kylinConfig, String metaOutDir, Set<String> dumpList)
        throws IOException {
    long startTime = System.currentTimeMillis();

    ResourceStore from = ResourceStore.getStore(kylinConfig);
    KylinConfig localConfig = KylinConfig.createInstanceFromUri(metaOutDir);
    ResourceStore to = ResourceStore.getStore(localConfig);
    final String[] tolerantResources = { "/table_exd" };

    for (String path : dumpList) {
        RawResource res = from.getResource(path);
        if (res == null) {
            if (StringUtils.startsWithAny(path, tolerantResources)) {
                continue;
            } else {
                throw new IllegalStateException("No resource found at -- " + path);
            }
        }
        to.putResource(path, res.content(), res.lastModified());
        res.content().close();
    }

    logger.debug("Dump resources to {} took {} ms", metaOutDir, System.currentTimeMillis() - startTime);
}
 

private void attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>(
            JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance()));
    ResourceStore rs = ResourceStore.getStore(segments.get(0).getConfig());
    for (CubeSegment segment : segments) {
        dumpList.addAll(segment.getDictionaryPaths());
        if (rs.exists(segment.getStatisticsResourcePath())) {
            // cube statistics is not available for new segment
            dumpList.add(segment.getStatisticsResourcePath());
        }
        //tiretree global domain dic
        CubeDescTiretreeGlobalDomainDictUtil.cuboidJob(segment.getCubeDesc(), dumpList);
    }
    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segments.get(0).getConfig(),
            this.getParam(SparkCubingByLayer.OPTION_META_URL.getOpt()));
}
 

public Context(QueryService queryService, CubeInstance cubeInstance, String targetHost, String tgtProjectName) {
    this.queryService = queryService;
    this.cubeInstance = cubeInstance;
    this.targetAddress = targetHost;
    KylinConfig targetConfig = KylinConfig.createInstanceFromUri(targetHost);
    this.targetResourceStore = ResourceStore.getStore(targetConfig);
    this.tgtProjectName = tgtProjectName;

    List<ProjectInstance> projList = ProjectManager.getInstance(KylinConfig.getInstanceFromEnv())
            .findProjects(cubeInstance.getType(), cubeInstance.getName());
    if (projList.size() != 1) {
        throw new InternalErrorException("Cube " + cubeInstance.getName()
                + " should belong to only one project. However, it's belong to " + projList);
    }
    this.srcProjectName = projList.get(0).getName();
}
 

public static void main(String[] args) throws IOException {
    String modelName = args[0];
    int nRows = Integer.parseInt(args[1]);
    String outputDir = args.length > 2 ? args[2] : null;

    KylinConfig conf = KylinConfig.getInstanceFromEnv();
    DataModelDesc model = DataModelManager.getInstance(conf).getDataModelDesc(modelName);
    ResourceStore store = outputDir == null ? ResourceStore.getStore(conf)
            : ResourceStore.getStore(mockup(outputDir));

    ModelDataGenerator gen = new ModelDataGenerator(model, nRows, store);
    gen.generate();
}
 

public ExtendCubeToHybridCLI() {
    this.kylinConfig = KylinConfig.getInstanceFromEnv();
    this.store = ResourceStore.getStore(kylinConfig);
    this.cubeManager = CubeManager.getInstance(kylinConfig);
    this.cubeDescManager = CubeDescManager.getInstance(kylinConfig);
    this.metadataManager = DataModelManager.getInstance(kylinConfig);
}
 

private void attachSegmentsMetadataWithDict(List<CubeSegment> segments) throws IOException {
    Set<String> dumpList = new LinkedHashSet<>();
    dumpList.addAll(JobRelatedMetaUtil.collectCubeMetadata(segments.get(0).getCubeInstance()));
    ResourceStore rs = ResourceStore.getStore(segments.get(0).getConfig());
    for (CubeSegment segment : segments) {
        dumpList.addAll(segment.getDictionaryPaths());
        if (rs.exists(segment.getStatisticsResourcePath())) {
            // cube statistics is not available for new segment
            dumpList.add(segment.getStatisticsResourcePath());
        }
    }

    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segments.get(0).getConfig(),
            this.getParam(FlinkCubingByLayer.OPTION_META_URL.getOpt()));
}
 

protected static ResourceStore getStore() {
    return ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
}
 

@Test
public void testPerformanceWithResourceTool() throws Exception {
    Assume.assumeTrue(jdbcConnectable);
    KylinConfig tmpConfig = KylinConfig.createKylinConfig(KylinConfig.getInstanceFromEnv());
    tmpConfig.setMetadataUrl(copyIdentifier + jdbcMetadataUrlNoIdentifier);

    JDBCResourceStore store = (JDBCResourceStore) ResourceStore.getStore(kylinConfig);
    NavigableSet<String> executes = store.listResources(ResourceStore.EXECUTE_RESOURCE_ROOT);
    NavigableSet<String> executeOutputs = store.listResources(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT);

    long startTs = System.currentTimeMillis();

    for (String execute : executes) {
        String uuid = StringUtils.substringAfterLast(execute, "/");
        RawResource executeResource = store.getResource(execute);
        Map<String, RawResource> executeOutputResourceMap = new HashMap<>();

        for (String executeOutput : executeOutputs) {
            if (executeOutput.contains(uuid)) {
                RawResource executeOutputResource = store.getResource(executeOutput);
                executeOutputResourceMap.put(executeOutput, executeOutputResource);
            }
        }

        for (int i = 0; i < 200; i++) {
            String newUuid = UUID.randomUUID().toString();
            store.putResource(ResourceStore.EXECUTE_RESOURCE_ROOT + "/" + newUuid, executeResource.content(),
                    System.currentTimeMillis());

            for (String key : executeOutputResourceMap.keySet()) {
                String step = StringUtils.substringAfterLast(key, uuid);
                store.putResource(ResourceStore.EXECUTE_OUTPUT_RESOURCE_ROOT + "/" + newUuid + step,
                        executeOutputResourceMap.get(key).content(), System.currentTimeMillis());
            }
        }
    }

    long queryNumBeforeCopy = store.getQueriedSqlNum();
    new ResourceTool().copy(kylinConfig, tmpConfig);
    long endTs = System.currentTimeMillis();
    long queryNumAfterCopy = store.getQueriedSqlNum();
    JDBCResourceStore resourceStoreCopy = (JDBCResourceStore) ResourceStore.getStore(tmpConfig);

    int executeNum = store.listResources("/execute").size();
    int executeOutputNum = store.listResources("/execute_output").size();

    assertEquals(executeNum, resourceStoreCopy.listResources("/execute").size());
    assertEquals(executeOutputNum, resourceStoreCopy.listResources("/execute_output").size());

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT);
    String startTime = sdf.format(new Date(Long.parseLong(String.valueOf(startTs))));
    String endTime = sdf.format(new Date(Long.parseLong(String.valueOf(endTs))));

    logger.info("Test performance with ResourceTool done during " + startTime + " to " + endTime);
    logger.info("Now there is " + executeNum + " execute data and " + executeOutputNum
            + " execute_output data in resource store.");
    logger.info("Resource store run " + queryNumBeforeCopy + " sqls for metadata generation, and "
            + (queryNumAfterCopy - queryNumBeforeCopy) + " sqls for copy with ResourceTool.");
    assertTrue((queryNumAfterCopy - queryNumBeforeCopy) < queryNumBeforeCopy);
    logger.info("This test is expected to be done in 10 mins.");
    assertTrue((endTs - startTs) < 600000);
}
 

@Test
public void testTableDescUpgrade() throws Exception {

    MetadataManager metaMgr = MetadataManager.getInstance(KylinConfig.getInstanceFromEnv());
    TableDesc fact = metaMgr.getTableDesc("default.test_kylin_fact");
    
    @SuppressWarnings("deprecation")
    String oldResLocation = fact.getResourcePathV1();
    String newResLocation = fact.getResourcePath();
    
    ResourceStore store = ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
    
    Assert.assertTrue(store.exists(newResLocation));
    Assert.assertTrue(!store.exists(oldResLocation));
    
    
    String oldExdResLocation = TableDesc.concatExdResourcePath("test_kylin_fact".toUpperCase());
    String newExdResLocation = TableDesc.concatExdResourcePath("default.test_kylin_fact".toUpperCase());
    
    Assert.assertTrue(store.exists(newExdResLocation));
    Assert.assertTrue(!store.exists(oldExdResLocation));
    
}
 

private ResourceStore getStore() {
    return ResourceStore.getStore(this.config);
}
 

private ResourceStore getStore() {
    return ResourceStore.getStore(config);
}
 

private ResourceStore getStore() {
    return ResourceStore.getStore(this.config);
}
 

private ResourceStore getStore() {
    return ResourceStore.getStore(this.config);
}
 

public static String getMetaStoreId() throws IOException {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    ResourceStore store = ResourceStore.getStore(kylinConfig);
    return store.getMetaStoreUUID();
}
 

ResourceStore getStore() {
    return ResourceStore.getStore(this.config);
}
 

private ResourceStore getStore() {
    return ResourceStore.getStore(this.config);
}
 

private ResourceStore getStore() {
    return ResourceStore.getStore(config);
}
 

public QueryService() {
    queryStore = ResourceStore.getStore(getConfig());
    preparedContextPool = createPreparedContextPool();
    badQueryDetector.start();
}
 

private ResourceStore getStore() {
    return ResourceStore.getStore(config);
}