org.apache.kylin.common.KylinConfig Java Examples

The following examples show how to use org.apache.kylin.common.KylinConfig. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BuildLayoutWithUpdate.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public void updateLayout(SegmentInfo seg, KylinConfig config) {
    for (int i = 0; i < currentLayoutsNum; i++) {
        try {
            logger.info("Wait to take job result.");
            JobResult result = completionService.take().get();
            logger.info("Take job result successful.");
            if (result.isFailed()) {
                shutDownPool();
                throw new RuntimeException(result.getThrowable());
            }
            seg.updateLayout(result.layout);
        } catch (InterruptedException | ExecutionException e) {
            shutDownPool();
            throw new RuntimeException(e);
        }
    }
    currentLayoutsNum = 0;
}
 
Example #2
Source File: CubeStatsReader.java    From kylin with Apache License 2.0 6 votes vote down vote up
/**
 * @param cuboidScheduler if it's null, part of it's functions will not be supported
 */
public CubeStatsReader(CubeSegment cubeSegment, CuboidScheduler cuboidScheduler, KylinConfig kylinConfig)
        throws IOException {
    ResourceStore store = ResourceStore.getStore(kylinConfig);
    String statsKey = cubeSegment.getStatisticsResourcePath();
    RawResource resource = store.getResource(statsKey);
    if (resource == null)
        throw new IllegalStateException("Missing resource at " + statsKey);

    File tmpSeqFile = writeTmpSeqFile(resource.content());
    Path path = new Path(HadoopUtil.fixWindowsPath("file://" + tmpSeqFile.getAbsolutePath()));

    CubeStatsResult cubeStatsResult = new CubeStatsResult(path, kylinConfig.getCubeStatsHLLPrecision());
    tmpSeqFile.delete();

    this.seg = cubeSegment;
    this.cuboidScheduler = cuboidScheduler;
    this.samplingPercentage = cubeStatsResult.getPercentage();
    this.mapperNumberOfFirstBuild = cubeStatsResult.getMapperNumber();
    this.mapperOverlapRatioOfFirstBuild = cubeStatsResult.getMapperOverlapRatio();
    this.cuboidRowEstimatesHLL = cubeStatsResult.getCounterMap();
    this.sourceRowCount = cubeStatsResult.getSourceRecordCount();
}
 
Example #3
Source File: BaseCuboidBuilder.java    From kylin with Apache License 2.0 6 votes vote down vote up
public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, CubeSegment cubeSegment,
        CubeJoinedFlatTableEnrich intermediateTableDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) {
    this.kylinConfig = kylinConfig;
    this.cubeDesc = cubeDesc;
    this.cubeSegment = cubeSegment;
    this.intermediateTableDesc = intermediateTableDesc;
    this.dictionaryMap = dictionaryMap;

    Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc);
    rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid);

    measureDescList = cubeDesc.getMeasures();
    aggrIngesters = MeasureIngester.create(measureDescList);
    measureCodec = new BufferedMeasureCodec(measureDescList);

    kvBuilder = new KeyValueBuilder(intermediateTableDesc);
    checkHiveGlobalDictionaryColumn();
}
 
Example #4
Source File: AbstractInfoExtractor.java    From kylin with Apache License 2.0 6 votes vote down vote up
private void dumpBasicDiagInfo() throws IOException {
    try {
        for (String commitSHA1File : COMMIT_SHA1_FILES) {
            File commitFile = new File(KylinConfig.getKylinHome(), commitSHA1File);
            if (commitFile.exists()) {
                FileUtils.copyFileToDirectory(commitFile, exportDir);
            }
        }
    } catch (IOException e) {
        logger.warn("Failed to copy commit_SHA1 file.", e);
    }

    String output = KylinVersion.getKylinClientInformation() + "\n";
    FileUtils.writeStringToFile(new File(exportDir, "kylin_env"), output, Charset.defaultCharset());

    StringBuilder basicSb = new StringBuilder();
    basicSb.append("MetaStoreID: ").append(ToolUtil.getMetaStoreId()).append("\n");
    basicSb.append("PackageType: ").append(packageType.toUpperCase(Locale.ROOT)).append("\n");
    SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z", Locale.ROOT);
    basicSb.append("PackageTimestamp: ").append(format.format(new Date())).append("\n");
    basicSb.append("Host: ").append(ToolUtil.getHostName()).append("\n");
    FileUtils.writeStringToFile(new File(exportDir, "info"), basicSb.toString(), Charset.defaultCharset());
}
 
Example #5
Source File: QueryConnection.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public static Connection getConnection(String project) throws SQLException {
    if (!isRegister) {
        try {
            Class<?> aClass = Thread.currentThread().getContextClassLoader()
                    .loadClass("org.apache.calcite.jdbc.Driver");
            Driver o = (Driver) aClass.getDeclaredConstructor().newInstance();
            DriverManager.registerDriver(o);
        } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
            e.printStackTrace();
        }
        isRegister = true;
    }
    File olapTmp = OLAPSchemaFactory.createTempOLAPJson(project, KylinConfig.getInstanceFromEnv());
    Properties info = new Properties();
    info.putAll(KylinConfig.getInstanceFromEnv().getCalciteExtrasProperties());
    // Import calcite props from jdbc client(override the kylin.properties)
    info.putAll(BackdoorToggles.getJdbcDriverClientCalciteProps());
    info.put("model", olapTmp.getAbsolutePath());
    info.put("typeSystem", "org.apache.kylin.query.calcite.KylinRelDataTypeSystem");
    return DriverManager.getConnection("jdbc:calcite:", info);
}
 
Example #6
Source File: RealizationSignature.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
static HybridSignature getHybridSignature(KylinConfig config, String realizationName) {
    HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(realizationName);
    if (hybridInstance == null) {
        return null;
    }
    IRealization[] realizations = hybridInstance.getRealizations();
    Set<RealizationSignature> realizationSignatureSet = Sets.newHashSetWithExpectedSize(realizations.length);
    for (IRealization realization : realizations) {
        RealizationSignature realizationSignature = null;
        if (realization.getType() == RealizationType.CUBE) {
            realizationSignature = CubeSignature.getCubeSignature(config, realization.getName());
        } else if (realization.getType() == RealizationType.HYBRID) {
            realizationSignature = getHybridSignature(config, realization.getName());
        }
        if (realizationSignature != null) {
            realizationSignatureSet.add(realizationSignature);
        }
    }
    return new HybridSignature(realizationName, realizationSignatureSet);
}
 
Example #7
Source File: QueryMetricsFacade.java    From kylin with Apache License 2.0 6 votes vote down vote up
private static QueryMetrics getQueryMetrics(String name) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    int[] intervals = config.getQueryMetricsPercentilesIntervals();

    QueryMetrics queryMetrics = metricsMap.get(name);
    if (queryMetrics != null) {
        return queryMetrics;
    }

    synchronized (QueryMetricsFacade.class) {
        queryMetrics = metricsMap.get(name);
        if (queryMetrics != null) {
            return queryMetrics;
        }

        try {
            queryMetrics = new QueryMetrics(intervals).registerWith(name);
            metricsMap.put(name, queryMetrics);
            return queryMetrics;
        } catch (MetricsException e) {
            logger.warn(name + " register error: ", e);
        }
    }
    return queryMetrics;
}
 
Example #8
Source File: InvertedIndexMapper.java    From Kylin with Apache License 2.0 6 votes vote down vote up
@Override
protected void setup(Context context) throws IOException {
    super.publishConfiguration(context.getConfiguration());

    Configuration conf = context.getConfiguration();

    KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf);
    IIManager mgr = IIManager.getInstance(config);
    IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME));
    IISegment seg = ii.getSegment(conf.get(BatchConstants.CFG_II_SEGMENT_NAME), SegmentStatusEnum.NEW);
    this.info = new TableRecordInfo(seg);
    this.rec = this.info.createTableRecord();

    outputKey = new LongWritable();
    outputValue = new ImmutableBytesWritable(rec.getBytes());

    schema = HCatInputFormat.getTableSchema(context.getConfiguration());
    
    fields = schema.getFields();
}
 
Example #9
Source File: BadQueryHistoryManagerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
@Test
public void testUpdateEntryToProject() throws IOException {
    KylinConfig kylinConfig = getTestConfig();
    BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig);

    String queryId = RandomUtil.randomUUID().toString();
    manager.upsertEntryToProject(
            new BadQueryEntry("sql", "adj", 1459362239000L, 100, "server", "t-0", "user", queryId, "cube"),
            "default");
    BadQueryHistory history = manager.upsertEntryToProject(
            new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user", queryId, "cube"), "default");

    NavigableSet<BadQueryEntry> entries = history.getEntries();
    BadQueryEntry newEntry = entries
            .floor(new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user", queryId, "cube"));
    System.out.println(newEntry);
    assertEquals("adj2", newEntry.getAdj());
    assertEquals("server2", newEntry.getServer());
    assertEquals("t-1", newEntry.getThread());
    assertEquals("user", newEntry.getUser());
    assertEquals(120, (int) newEntry.getRunningSec());
}
 
Example #10
Source File: HBaseLookupMRSteps.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig);
    TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject());
    IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId());
    try {
        ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot(
                sourceTable.getSignature(), tableName);
        if (latestSnapshot != null) {
            logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName);
            context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath());
            return;
        }
    } catch (IOException ioException) {
        throw new RuntimeException(ioException);
    }
    logger.info("add build snapshot steps for table:{}", tableName);
    String snapshotID = genLookupSnapshotID();
    context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID));
    addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID);
    addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID);
    if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) {
        addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID);
    }
}
 
Example #11
Source File: BasicMeasureType.java    From kylin with Apache License 2.0 6 votes vote down vote up
@Override
public void validate(FunctionDesc functionDesc) throws IllegalArgumentException {
    DataType rtype = dataType;

    if (funcName.equals(FunctionDesc.FUNC_SUM)) {
        if (rtype.isNumberFamily() == false) {
            throw new IllegalArgumentException("Return type for function " + funcName + " must be one of " + DataType.NUMBER_FAMILY);
        }
    } else if (funcName.equals(FunctionDesc.FUNC_COUNT)) {
        if (rtype.isIntegerFamily() == false) {
            throw new IllegalArgumentException("Return type for function " + funcName + " must be one of " + DataType.INTEGER_FAMILY);
        }
    } else if (funcName.equals(FunctionDesc.FUNC_MAX) || funcName.equals(FunctionDesc.FUNC_MIN)) {
        if (rtype.isNumberFamily() == false) {
            throw new IllegalArgumentException("Return type for function " + funcName + " must be one of " + DataType.NUMBER_FAMILY);
        }
    } else {
        KylinConfig config = KylinConfig.getInstanceFromEnv();
        if (config.isQueryIgnoreUnknownFunction() == false)
            throw new IllegalArgumentException("Unrecognized function: [" + funcName + "]");
    }
}
 
Example #12
Source File: CubeDescManager.java    From kylin with Apache License 2.0 6 votes vote down vote up
private CubeDescManager(KylinConfig cfg) throws IOException {
    logger.info("Initializing CubeDescManager with config {}", cfg);
    this.config = cfg;
    this.cubeDescMap = new CaseInsensitiveStringCache<>(config, "cube_desc");
    this.crud = new CachedCrudAssist<CubeDesc>(getStore(), ResourceStore.CUBE_DESC_RESOURCE_ROOT, CubeDesc.class,
            cubeDescMap) {
        @Override
        protected CubeDesc initEntityAfterReload(CubeDesc cubeDesc, String resourceName) {
            if (cubeDesc.isDraft())
                throw new IllegalArgumentException(String.format(Locale.ROOT, CUBE_SHOULD_NOT_BE_DRAFT_MSG, cubeDesc.getName()));

            try {
                cubeDesc.init(config);
            } catch (Exception e) {
                logger.warn(String.format(Locale.ROOT, BROKEN_CUBE_MSG, cubeDesc.resourceName()), e);
                cubeDesc.addError(e.toString());
            }
            return cubeDesc;
        }
    };

    // touch lower level metadata before registering my listener
    crud.reloadAll();
    Broadcaster.getInstance(config).registerListener(new CubeDescSyncListener(), "cube_desc");
}
 
Example #13
Source File: SparkExecutable.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private void updateSparkDimensionDicMetadata(KylinConfig config, CubeInstance cube, String segmentId)
        throws IOException {
    KylinConfig hdfsConfig = AbstractHadoopJob
            .loadKylinConfigFromHdfs(this.getParam(SparkBuildDictionary.OPTION_META_URL.getOpt()));
    CubeInstance cubeInstance = CubeManager.getInstance(hdfsConfig).reloadCube(cube.getName());
    CubeSegment segment = cubeInstance.getSegmentById(segmentId);

    CubeSegment oldSeg = cube.getSegmentById(segmentId);
    oldSeg.setDictionaries((ConcurrentHashMap<String, String>) segment.getDictionaries());
    oldSeg.setSnapshots((ConcurrentHashMap) segment.getSnapshots());
    oldSeg.getRowkeyStats().addAll(segment.getRowkeyStats());
    CubeInstance cubeCopy = cube.latestCopyForWrite();
    CubeUpdate update = new CubeUpdate(cubeCopy);
    update.setToUpdateSegs(oldSeg);
    CubeManager.getInstance(config).updateCube(update);

    Set<String> dumpList = new LinkedHashSet<>();
    dumpList.addAll(segment.getDictionaryPaths());
    dumpList.addAll(segment.getSnapshotPaths());

    JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segment.getConfig(),
            config.getMetadataUrl().toString());
}
 
Example #14
Source File: AbstractHadoopJob.java    From kylin with Apache License 2.0 5 votes vote down vote up
public static KylinConfig loadKylinPropsAndMetadata() throws IOException {
    File metaDir = new File("meta");
    if (!metaDir.getAbsolutePath().equals(System.getProperty(KylinConfig.KYLIN_CONF))) {
        System.setProperty(KylinConfig.KYLIN_CONF, metaDir.getAbsolutePath());
        logger.info("The absolute path for meta dir is " + metaDir.getAbsolutePath());
        KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
        Map<String, String> paramsMap = new HashMap<>();
        paramsMap.put("path", metaDir.getAbsolutePath());
        StorageURL storageURL = new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "ifile", paramsMap);
        kylinConfig.setMetadataUrl(storageURL.toString());
        return kylinConfig;
    } else {
        return KylinConfig.getInstanceFromEnv();
    }
}
 
Example #15
Source File: LocalFileMetadataTestCase.java    From kylin with Apache License 2.0 5 votes vote down vote up
protected String getLocalWorkingDirectory() {
    String dir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
    if (dir.startsWith("file://"))
        dir = dir.substring("file://".length());
    try {
        return new File(dir).getCanonicalPath();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example #16
Source File: CubeMetaIngester.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private void injest(File metaRoot) throws IOException {
    KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath());
    TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig);
    DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig);
    HybridManager srcHybridManager = HybridManager.getInstance(srcConfig);
    CubeManager srcCubeManager = CubeManager.getInstance(srcConfig);
    CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig);

    checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager);
    new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources));

    // clear the cache
    Broadcaster.getInstance(kylinConfig).notifyClearAll();
    
    ProjectManager projectManager = ProjectManager.getInstance(kylinConfig);
    for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) {
        logger.info("add " + tableDesc + " to " + targetProjectName);
        projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName);
    }

    for (CubeInstance cube : srcCubeManager.listAllCubes()) {
        logger.info("add " + cube + " to " + targetProjectName);
        projectManager.addModelToProject(cube.getModel().getName(), targetProjectName);
        projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null);
    }

}
 
Example #17
Source File: SqoopCmdStep.java    From kylin with Apache License 2.0 5 votes vote down vote up
protected String generateSqoopConfigArgString() {
    KylinConfig kylinConfig = getConfig();
    Map<String, String> config = Maps.newHashMap();
    config.put("mapreduce.job.queuename", getSqoopJobQueueName(kylinConfig)); // override job queue from mapreduce config
    config.putAll(SourceConfigurationUtil.loadSqoopConfiguration());
    config.putAll(kylinConfig.getSqoopConfigOverride());

    StringBuilder args = new StringBuilder();
    for (Map.Entry<String, String> entry : config.entrySet()) {
        args.append(" -D" + entry.getKey() + "=" + entry.getValue() + " ");
    }
    return args.toString();
}
 
Example #18
Source File: SAMLSimpleUserDetailsService.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Override
public Object loadUserBySAML(SAMLCredential samlCredential) throws UsernameNotFoundException {
    final String userEmail = samlCredential.getAttributeAsString("email");
    logger.debug("samlCredential.email:" + userEmail);
    final String userName = userEmail.substring(0, userEmail.indexOf("@"));

    KylinUserManager userManager = KylinUserManager.getInstance(KylinConfig.getInstanceFromEnv());
    ManagedUser existUser = userManager.get(userName);
    // create if not exists
    if (existUser == null) {
        ManagedUser user = new ManagedUser(userName, NO_EXISTENCE_PASSWORD, true, defaultAuthorities);
        userManager.update(user);
    }
    return userManager.get(userName);
}
 
Example #19
Source File: ITInMemCubeBuilderTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testSSBCubeMore() throws Exception {
    testBuild("ssb", //
            LOCALMETA_TEST_DATA + "/data/" + KylinConfig.getInstanceFromEnv().getHiveIntermediateTablePrefix()
                    + "ssb_19920101000000_19920201000000.csv",
            7000, 4);
}
 
Example #20
Source File: AssignmentsCache.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private AssignmentsCache() {
    this.metadataStore = StreamMetadataStoreFactory.getStreamMetaDataStore();
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    cubeAssignmentCache = CacheBuilder.newBuilder()
            .removalListener((RemovalNotification<String, List<ReplicaSet>> notification) -> logger
                    .debug("{} is removed because {} ", notification.getKey(), notification.getCause()))
            .expireAfterWrite(300, TimeUnit.SECONDS)
            .build();
     Broadcaster.getInstance(config).registerListener(new AssignCacheSyncListener(), ASSIGNMENT_ENTITY);
}
 
Example #21
Source File: QueryWithTableACLTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Test
public void testFailQuery() throws SQLException, IOException {
    QueryACLTestUtil.setUser(MODELER);
    QueryACLTestUtil.mockQuery(PROJECT, "select * from TEST_KYLIN_FACT");

    QueryACLTestUtil.setUser(ADMIN);
    TableACLManager.getInstance(KylinConfig.getInstanceFromEnv()).addTableACL(PROJECT, "ADMIN", QUERY_TABLE, MetadataConstants.TYPE_USER);
    thrown.expectCause(CoreMatchers.isA(AccessDeniedException.class));
    thrown.expectMessage(CoreMatchers.containsString("Query failed.Access table:DEFAULT.TEST_KYLIN_FACT denied"));
    QueryACLTestUtil.mockQuery(PROJECT, "select * from TEST_KYLIN_FACT");
}
 
Example #22
Source File: HBaseResourceStore.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public HBaseResourceStore(KylinConfig kylinConfig) throws IOException {
    super(kylinConfig);
    metadataUrl = buildMetadataUrl(kylinConfig);
    tableName = metadataUrl.getIdentifier();
    createHTableIfNeeded(tableName);

    kvSizeLimit = Integer
            .parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760"));
}
 
Example #23
Source File: ZookeeperAclBuilder.java    From kylin with Apache License 2.0 5 votes vote down vote up
public static List<ZKUtil.ZKAuthInfo> getZKAuths() throws Exception {
    // Parse Auths from configuration.
    String zkAuthConf = KylinConfig.getInstanceFromEnv().getZKAuths();
    try {
        zkAuthConf = ZKUtil.resolveConfIndirection(zkAuthConf);
        if (zkAuthConf != null) {
            return ZKUtil.parseAuth(zkAuthConf);
        } else {
            return Collections.emptyList();
        }
    } catch (Exception e) {
        logger.error("Couldn't read Auth based on 'kylin.env.zookeeper.zk-auth' in kylin.properties");
        throw e;
    }
}
 
Example #24
Source File: StreamingCubeDataSearcherPerfTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
public StreamingCubeDataSearcherPerfTest() throws Exception {
    this.createTestMetadata();
    this.baseStorePath = KylinConfig.getInstanceFromEnv().getStreamingIndexPath();
    this.cubeInstance = CubeManager.getInstance(getTestConfig()).reloadCubeQuietly(cubeName);
    this.parsedStreamingCubeInfo = new ParsedStreamingCubeInfo(cubeInstance);
    setupCubeConfig();
    this.testHelper = new TestHelper(cubeInstance);
    this.streamingSegmentManager = new StreamingSegmentManager(baseStorePath, cubeInstance,
            new MockPositionHandler(), null);
    this.searcher = streamingSegmentManager.getSearcher();
}
 
Example #25
Source File: SparkBuildDictionary.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private void init() {
    try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig
            .setAndUnsetThreadLocalConfig(config)) {
        cubeManager = CubeManager.getInstance(config);
        cubeSegment = cubeManager.getCube(cubeName).getSegmentById(segmentId);
    }
    initialized = true;
}
 
Example #26
Source File: ZKUtil.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
/**
 * Get zookeeper connection string from kylin.properties
 */
public static String getZKConnectString(KylinConfig config) {
    String zkString = config.getZookeeperConnectString();
    if (zkString == null) {
        zkString = getZKConnectStringFromHBase();
        if (zkString == null) {
            throw new RuntimeException("Please set 'kylin.env.zookeeper-connect-string' in kylin.properties");
        }
    }

    return zkString;
}
 
Example #27
Source File: SnapshotManagerTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public void runTestCase() throws IOException, InterruptedException {
    kylinConfig = KylinConfig.getInstanceFromEnv();
    snapshotManager = SnapshotManager.getInstance(kylinConfig);
    SnapshotTable origin = snapshotManager.buildSnapshot(genTable("./origin", expect), genTableDesc("TEST_TABLE"),
            kylinConfig);

    // sleep 1 second to avoid file resource store precision lost
    Thread.sleep(1000);
    SnapshotTable dup = snapshotManager.buildSnapshot(genTable("./dup", expect), genTableDesc("TEST_TABLE"),
            kylinConfig);
    // assert same snapshot file
    Assert.assertEquals(origin.getUuid(), dup.getUuid());
    Assert.assertEquals(origin.getResourcePath(), dup.getResourcePath());

    // assert the file has been updated
    long originLastModified = origin.getLastModified();
    long dupLastModified = dup.getLastModified();
    Assert.assertTrue(dupLastModified > originLastModified);

    SnapshotTable actual = snapshotManager.getSnapshotTable(origin.getResourcePath());
    IReadableTable.TableReader reader = actual.getReader();
    Assert.assertEquals(expect.size(), actual.getRowCount());
    int i = 0;
    while (reader.next()) {
        Assert.assertEquals(stringJoin(expect.get(i++)), stringJoin(reader.getRow()));
    }

    SnapshotTable difTable = snapshotManager.buildSnapshot(genTable("./dif", dif), genTableDesc("TEST_TABLE"),
            kylinConfig);
    Assert.assertNotEquals(origin.getUuid(), difTable.getUuid());
}
 
Example #28
Source File: ITStorageTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
    this.createTestMetadata();

    CubeManager cubeMgr = CubeManager.getInstance(getTestConfig());
    cube = cubeMgr.getCube("test_kylin_cube_without_slr_left_join_empty");
    Assert.assertNotNull(cube);
    storageEngine = StorageFactory.createQuery(cube);
    context = new StorageContext();
    context.setConnUrl(KylinConfig.getInstanceFromEnv().getStorageUrl());
    mockup = new StorageMockUtils(cube.getModel());
}
 
Example #29
Source File: JdbcTableReader.java    From kylin with Apache License 2.0 5 votes vote down vote up
/**
 * Constructor for reading whole jdbc table
 * @param dbName
 * @param tableName
 * @throws IOException
 */
public JdbcTableReader(String dbName, String tableName) throws IOException {
    this.dbName = dbName;
    this.tableName = tableName;
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String connectionUrl = config.getJdbcSourceConnectionUrl();
    String driverClass = config.getJdbcSourceDriver();
    String jdbcUser = config.getJdbcSourceUser();
    String jdbcPass = config.getJdbcSourcePass();
    dbconf = new DBConnConf(driverClass, connectionUrl, jdbcUser, jdbcPass);
    jdbcCon = SqlUtil.getConnection(dbconf);
    IJdbcMetadata meta = JdbcMetadataFactory
            .getJdbcMetadata(SourceDialect.getDialect(config.getJdbcSourceDialect()), dbconf);

    Map<String, String> metadataCache = new TreeMap<>();
    JdbcHiveInputBase.JdbcBaseBatchCubingInputSide.calCachedJdbcMeta(metadataCache, dbconf, meta);
    String database = JdbcHiveInputBase.getSchemaQuoted(metadataCache, dbName, meta, true);
    String table = JdbcHiveInputBase.getTableIdentityQuoted(dbName, tableName, metadataCache, meta, true);

    String sql = String.format(Locale.ROOT, "select * from %s.%s", database, table);
    try {
        statement = jdbcCon.createStatement();
        rs = statement.executeQuery(sql);
        colCount = rs.getMetaData().getColumnCount();
    } catch (SQLException e) {
        throw new IOException(String.format(Locale.ROOT, "error while exec %s", sql), e);
    }
}
 
Example #30
Source File: QueryCli.java    From kylin with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

        Options options = new Options();
        options.addOption(OPTION_METADATA);
        options.addOption(OPTION_SQL);

        CommandLineParser parser = new GnuParser();
        CommandLine commandLine = parser.parse(options, args);
        KylinConfig config = KylinConfig.createInstanceFromUri(commandLine.getOptionValue(OPTION_METADATA.getOpt()));
        String sql = commandLine.getOptionValue(OPTION_SQL.getOpt());

        Connection conn = null;
        Statement stmt = null;
        ResultSet rs = null;
        try {
            conn = QueryConnection.getConnection(null);

            stmt = conn.createStatement();
            rs = stmt.executeQuery(sql);
            int n = 0;
            ResultSetMetaData meta = rs.getMetaData();
            while (rs.next()) {
                n++;
                for (int i = 1; i <= meta.getColumnCount(); i++) {
                    System.out.println(n + " - " + meta.getColumnLabel(i) + ":\t" + rs.getObject(i));
                }
            }
        } finally {
            DBUtils.closeQuietly(rs);
            DBUtils.closeQuietly(stmt);
            DBUtils.closeQuietly(conn);
        }

    }