org.apache.kylin.common.KylinConfig Java Examples
The following examples show how to use
org.apache.kylin.common.KylinConfig.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BuildLayoutWithUpdate.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
public void updateLayout(SegmentInfo seg, KylinConfig config) { for (int i = 0; i < currentLayoutsNum; i++) { try { logger.info("Wait to take job result."); JobResult result = completionService.take().get(); logger.info("Take job result successful."); if (result.isFailed()) { shutDownPool(); throw new RuntimeException(result.getThrowable()); } seg.updateLayout(result.layout); } catch (InterruptedException | ExecutionException e) { shutDownPool(); throw new RuntimeException(e); } } currentLayoutsNum = 0; }
Example #2
Source File: CubeStatsReader.java From kylin with Apache License 2.0 | 6 votes |
/** * @param cuboidScheduler if it's null, part of it's functions will not be supported */ public CubeStatsReader(CubeSegment cubeSegment, CuboidScheduler cuboidScheduler, KylinConfig kylinConfig) throws IOException { ResourceStore store = ResourceStore.getStore(kylinConfig); String statsKey = cubeSegment.getStatisticsResourcePath(); RawResource resource = store.getResource(statsKey); if (resource == null) throw new IllegalStateException("Missing resource at " + statsKey); File tmpSeqFile = writeTmpSeqFile(resource.content()); Path path = new Path(HadoopUtil.fixWindowsPath("file://" + tmpSeqFile.getAbsolutePath())); CubeStatsResult cubeStatsResult = new CubeStatsResult(path, kylinConfig.getCubeStatsHLLPrecision()); tmpSeqFile.delete(); this.seg = cubeSegment; this.cuboidScheduler = cuboidScheduler; this.samplingPercentage = cubeStatsResult.getPercentage(); this.mapperNumberOfFirstBuild = cubeStatsResult.getMapperNumber(); this.mapperOverlapRatioOfFirstBuild = cubeStatsResult.getMapperOverlapRatio(); this.cuboidRowEstimatesHLL = cubeStatsResult.getCounterMap(); this.sourceRowCount = cubeStatsResult.getSourceRecordCount(); }
Example #3
Source File: BaseCuboidBuilder.java From kylin with Apache License 2.0 | 6 votes |
public BaseCuboidBuilder(KylinConfig kylinConfig, CubeDesc cubeDesc, CubeSegment cubeSegment, CubeJoinedFlatTableEnrich intermediateTableDesc, Map<TblColRef, Dictionary<String>> dictionaryMap) { this.kylinConfig = kylinConfig; this.cubeDesc = cubeDesc; this.cubeSegment = cubeSegment; this.intermediateTableDesc = intermediateTableDesc; this.dictionaryMap = dictionaryMap; Cuboid baseCuboid = Cuboid.getBaseCuboid(cubeDesc); rowKeyEncoder = AbstractRowKeyEncoder.createInstance(cubeSegment, baseCuboid); measureDescList = cubeDesc.getMeasures(); aggrIngesters = MeasureIngester.create(measureDescList); measureCodec = new BufferedMeasureCodec(measureDescList); kvBuilder = new KeyValueBuilder(intermediateTableDesc); checkHiveGlobalDictionaryColumn(); }
Example #4
Source File: AbstractInfoExtractor.java From kylin with Apache License 2.0 | 6 votes |
private void dumpBasicDiagInfo() throws IOException { try { for (String commitSHA1File : COMMIT_SHA1_FILES) { File commitFile = new File(KylinConfig.getKylinHome(), commitSHA1File); if (commitFile.exists()) { FileUtils.copyFileToDirectory(commitFile, exportDir); } } } catch (IOException e) { logger.warn("Failed to copy commit_SHA1 file.", e); } String output = KylinVersion.getKylinClientInformation() + "\n"; FileUtils.writeStringToFile(new File(exportDir, "kylin_env"), output, Charset.defaultCharset()); StringBuilder basicSb = new StringBuilder(); basicSb.append("MetaStoreID: ").append(ToolUtil.getMetaStoreId()).append("\n"); basicSb.append("PackageType: ").append(packageType.toUpperCase(Locale.ROOT)).append("\n"); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z", Locale.ROOT); basicSb.append("PackageTimestamp: ").append(format.format(new Date())).append("\n"); basicSb.append("Host: ").append(ToolUtil.getHostName()).append("\n"); FileUtils.writeStringToFile(new File(exportDir, "info"), basicSb.toString(), Charset.defaultCharset()); }
Example #5
Source File: QueryConnection.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
public static Connection getConnection(String project) throws SQLException { if (!isRegister) { try { Class<?> aClass = Thread.currentThread().getContextClassLoader() .loadClass("org.apache.calcite.jdbc.Driver"); Driver o = (Driver) aClass.getDeclaredConstructor().newInstance(); DriverManager.registerDriver(o); } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) { e.printStackTrace(); } isRegister = true; } File olapTmp = OLAPSchemaFactory.createTempOLAPJson(project, KylinConfig.getInstanceFromEnv()); Properties info = new Properties(); info.putAll(KylinConfig.getInstanceFromEnv().getCalciteExtrasProperties()); // Import calcite props from jdbc client(override the kylin.properties) info.putAll(BackdoorToggles.getJdbcDriverClientCalciteProps()); info.put("model", olapTmp.getAbsolutePath()); info.put("typeSystem", "org.apache.kylin.query.calcite.KylinRelDataTypeSystem"); return DriverManager.getConnection("jdbc:calcite:", info); }
Example #6
Source File: RealizationSignature.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
static HybridSignature getHybridSignature(KylinConfig config, String realizationName) { HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(realizationName); if (hybridInstance == null) { return null; } IRealization[] realizations = hybridInstance.getRealizations(); Set<RealizationSignature> realizationSignatureSet = Sets.newHashSetWithExpectedSize(realizations.length); for (IRealization realization : realizations) { RealizationSignature realizationSignature = null; if (realization.getType() == RealizationType.CUBE) { realizationSignature = CubeSignature.getCubeSignature(config, realization.getName()); } else if (realization.getType() == RealizationType.HYBRID) { realizationSignature = getHybridSignature(config, realization.getName()); } if (realizationSignature != null) { realizationSignatureSet.add(realizationSignature); } } return new HybridSignature(realizationName, realizationSignatureSet); }
Example #7
Source File: QueryMetricsFacade.java From kylin with Apache License 2.0 | 6 votes |
private static QueryMetrics getQueryMetrics(String name) { KylinConfig config = KylinConfig.getInstanceFromEnv(); int[] intervals = config.getQueryMetricsPercentilesIntervals(); QueryMetrics queryMetrics = metricsMap.get(name); if (queryMetrics != null) { return queryMetrics; } synchronized (QueryMetricsFacade.class) { queryMetrics = metricsMap.get(name); if (queryMetrics != null) { return queryMetrics; } try { queryMetrics = new QueryMetrics(intervals).registerWith(name); metricsMap.put(name, queryMetrics); return queryMetrics; } catch (MetricsException e) { logger.warn(name + " register error: ", e); } } return queryMetrics; }
Example #8
Source File: InvertedIndexMapper.java From Kylin with Apache License 2.0 | 6 votes |
@Override protected void setup(Context context) throws IOException { super.publishConfiguration(context.getConfiguration()); Configuration conf = context.getConfiguration(); KylinConfig config = AbstractHadoopJob.loadKylinPropsAndMetadata(conf); IIManager mgr = IIManager.getInstance(config); IIInstance ii = mgr.getII(conf.get(BatchConstants.CFG_II_NAME)); IISegment seg = ii.getSegment(conf.get(BatchConstants.CFG_II_SEGMENT_NAME), SegmentStatusEnum.NEW); this.info = new TableRecordInfo(seg); this.rec = this.info.createTableRecord(); outputKey = new LongWritable(); outputValue = new ImmutableBytesWritable(rec.getBytes()); schema = HCatInputFormat.getTableSchema(context.getConfiguration()); fields = schema.getFields(); }
Example #9
Source File: BadQueryHistoryManagerTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
@Test public void testUpdateEntryToProject() throws IOException { KylinConfig kylinConfig = getTestConfig(); BadQueryHistoryManager manager = BadQueryHistoryManager.getInstance(kylinConfig); String queryId = RandomUtil.randomUUID().toString(); manager.upsertEntryToProject( new BadQueryEntry("sql", "adj", 1459362239000L, 100, "server", "t-0", "user", queryId, "cube"), "default"); BadQueryHistory history = manager.upsertEntryToProject( new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user", queryId, "cube"), "default"); NavigableSet<BadQueryEntry> entries = history.getEntries(); BadQueryEntry newEntry = entries .floor(new BadQueryEntry("sql", "adj2", 1459362239000L, 120, "server2", "t-1", "user", queryId, "cube")); System.out.println(newEntry); assertEquals("adj2", newEntry.getAdj()); assertEquals("server2", newEntry.getServer()); assertEquals("t-1", newEntry.getThread()); assertEquals("user", newEntry.getUser()); assertEquals(120, (int) newEntry.getRunningSec()); }
Example #10
Source File: HBaseLookupMRSteps.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) { KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig); TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject()); IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId()); try { ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot( sourceTable.getSignature(), tableName); if (latestSnapshot != null) { logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName); context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath()); return; } } catch (IOException ioException) { throw new RuntimeException(ioException); } logger.info("add build snapshot steps for table:{}", tableName); String snapshotID = genLookupSnapshotID(); context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID)); addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID); addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID); if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) { addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID); } }
Example #11
Source File: BasicMeasureType.java From kylin with Apache License 2.0 | 6 votes |
@Override public void validate(FunctionDesc functionDesc) throws IllegalArgumentException { DataType rtype = dataType; if (funcName.equals(FunctionDesc.FUNC_SUM)) { if (rtype.isNumberFamily() == false) { throw new IllegalArgumentException("Return type for function " + funcName + " must be one of " + DataType.NUMBER_FAMILY); } } else if (funcName.equals(FunctionDesc.FUNC_COUNT)) { if (rtype.isIntegerFamily() == false) { throw new IllegalArgumentException("Return type for function " + funcName + " must be one of " + DataType.INTEGER_FAMILY); } } else if (funcName.equals(FunctionDesc.FUNC_MAX) || funcName.equals(FunctionDesc.FUNC_MIN)) { if (rtype.isNumberFamily() == false) { throw new IllegalArgumentException("Return type for function " + funcName + " must be one of " + DataType.NUMBER_FAMILY); } } else { KylinConfig config = KylinConfig.getInstanceFromEnv(); if (config.isQueryIgnoreUnknownFunction() == false) throw new IllegalArgumentException("Unrecognized function: [" + funcName + "]"); } }
Example #12
Source File: CubeDescManager.java From kylin with Apache License 2.0 | 6 votes |
private CubeDescManager(KylinConfig cfg) throws IOException { logger.info("Initializing CubeDescManager with config {}", cfg); this.config = cfg; this.cubeDescMap = new CaseInsensitiveStringCache<>(config, "cube_desc"); this.crud = new CachedCrudAssist<CubeDesc>(getStore(), ResourceStore.CUBE_DESC_RESOURCE_ROOT, CubeDesc.class, cubeDescMap) { @Override protected CubeDesc initEntityAfterReload(CubeDesc cubeDesc, String resourceName) { if (cubeDesc.isDraft()) throw new IllegalArgumentException(String.format(Locale.ROOT, CUBE_SHOULD_NOT_BE_DRAFT_MSG, cubeDesc.getName())); try { cubeDesc.init(config); } catch (Exception e) { logger.warn(String.format(Locale.ROOT, BROKEN_CUBE_MSG, cubeDesc.resourceName()), e); cubeDesc.addError(e.toString()); } return cubeDesc; } }; // touch lower level metadata before registering my listener crud.reloadAll(); Broadcaster.getInstance(config).registerListener(new CubeDescSyncListener(), "cube_desc"); }
Example #13
Source File: SparkExecutable.java From kylin-on-parquet-v2 with Apache License 2.0 | 6 votes |
private void updateSparkDimensionDicMetadata(KylinConfig config, CubeInstance cube, String segmentId) throws IOException { KylinConfig hdfsConfig = AbstractHadoopJob .loadKylinConfigFromHdfs(this.getParam(SparkBuildDictionary.OPTION_META_URL.getOpt())); CubeInstance cubeInstance = CubeManager.getInstance(hdfsConfig).reloadCube(cube.getName()); CubeSegment segment = cubeInstance.getSegmentById(segmentId); CubeSegment oldSeg = cube.getSegmentById(segmentId); oldSeg.setDictionaries((ConcurrentHashMap<String, String>) segment.getDictionaries()); oldSeg.setSnapshots((ConcurrentHashMap) segment.getSnapshots()); oldSeg.getRowkeyStats().addAll(segment.getRowkeyStats()); CubeInstance cubeCopy = cube.latestCopyForWrite(); CubeUpdate update = new CubeUpdate(cubeCopy); update.setToUpdateSegs(oldSeg); CubeManager.getInstance(config).updateCube(update); Set<String> dumpList = new LinkedHashSet<>(); dumpList.addAll(segment.getDictionaryPaths()); dumpList.addAll(segment.getSnapshotPaths()); JobRelatedMetaUtil.dumpAndUploadKylinPropsAndMetadata(dumpList, (KylinConfigExt) segment.getConfig(), config.getMetadataUrl().toString()); }
Example #14
Source File: AbstractHadoopJob.java From kylin with Apache License 2.0 | 5 votes |
public static KylinConfig loadKylinPropsAndMetadata() throws IOException { File metaDir = new File("meta"); if (!metaDir.getAbsolutePath().equals(System.getProperty(KylinConfig.KYLIN_CONF))) { System.setProperty(KylinConfig.KYLIN_CONF, metaDir.getAbsolutePath()); logger.info("The absolute path for meta dir is " + metaDir.getAbsolutePath()); KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); Map<String, String> paramsMap = new HashMap<>(); paramsMap.put("path", metaDir.getAbsolutePath()); StorageURL storageURL = new StorageURL(kylinConfig.getMetadataUrl().getIdentifier(), "ifile", paramsMap); kylinConfig.setMetadataUrl(storageURL.toString()); return kylinConfig; } else { return KylinConfig.getInstanceFromEnv(); } }
Example #15
Source File: LocalFileMetadataTestCase.java From kylin with Apache License 2.0 | 5 votes |
protected String getLocalWorkingDirectory() { String dir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory(); if (dir.startsWith("file://")) dir = dir.substring("file://".length()); try { return new File(dir).getCanonicalPath(); } catch (IOException e) { throw new RuntimeException(e); } }
Example #16
Source File: CubeMetaIngester.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
private void injest(File metaRoot) throws IOException { KylinConfig srcConfig = KylinConfig.createInstanceFromUri(metaRoot.getAbsolutePath()); TableMetadataManager srcMetadataManager = TableMetadataManager.getInstance(srcConfig); DataModelManager srcModelManager = DataModelManager.getInstance(srcConfig); HybridManager srcHybridManager = HybridManager.getInstance(srcConfig); CubeManager srcCubeManager = CubeManager.getInstance(srcConfig); CubeDescManager srcCubeDescManager = CubeDescManager.getInstance(srcConfig); checkAndMark(srcMetadataManager, srcModelManager, srcHybridManager, srcCubeManager, srcCubeDescManager); new ResourceTool().copy(srcConfig, kylinConfig, Lists.newArrayList(requiredResources)); // clear the cache Broadcaster.getInstance(kylinConfig).notifyClearAll(); ProjectManager projectManager = ProjectManager.getInstance(kylinConfig); for (TableDesc tableDesc : srcMetadataManager.listAllTables(null)) { logger.info("add " + tableDesc + " to " + targetProjectName); projectManager.addTableDescToProject(Lists.newArrayList(tableDesc.getIdentity()).toArray(new String[0]), targetProjectName); } for (CubeInstance cube : srcCubeManager.listAllCubes()) { logger.info("add " + cube + " to " + targetProjectName); projectManager.addModelToProject(cube.getModel().getName(), targetProjectName); projectManager.moveRealizationToProject(RealizationType.CUBE, cube.getName(), targetProjectName, null); } }
Example #17
Source File: SqoopCmdStep.java From kylin with Apache License 2.0 | 5 votes |
protected String generateSqoopConfigArgString() { KylinConfig kylinConfig = getConfig(); Map<String, String> config = Maps.newHashMap(); config.put("mapreduce.job.queuename", getSqoopJobQueueName(kylinConfig)); // override job queue from mapreduce config config.putAll(SourceConfigurationUtil.loadSqoopConfiguration()); config.putAll(kylinConfig.getSqoopConfigOverride()); StringBuilder args = new StringBuilder(); for (Map.Entry<String, String> entry : config.entrySet()) { args.append(" -D" + entry.getKey() + "=" + entry.getValue() + " "); } return args.toString(); }
Example #18
Source File: SAMLSimpleUserDetailsService.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Override public Object loadUserBySAML(SAMLCredential samlCredential) throws UsernameNotFoundException { final String userEmail = samlCredential.getAttributeAsString("email"); logger.debug("samlCredential.email:" + userEmail); final String userName = userEmail.substring(0, userEmail.indexOf("@")); KylinUserManager userManager = KylinUserManager.getInstance(KylinConfig.getInstanceFromEnv()); ManagedUser existUser = userManager.get(userName); // create if not exists if (existUser == null) { ManagedUser user = new ManagedUser(userName, NO_EXISTENCE_PASSWORD, true, defaultAuthorities); userManager.update(user); } return userManager.get(userName); }
Example #19
Source File: ITInMemCubeBuilderTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Test public void testSSBCubeMore() throws Exception { testBuild("ssb", // LOCALMETA_TEST_DATA + "/data/" + KylinConfig.getInstanceFromEnv().getHiveIntermediateTablePrefix() + "ssb_19920101000000_19920201000000.csv", 7000, 4); }
Example #20
Source File: AssignmentsCache.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
private AssignmentsCache() { this.metadataStore = StreamMetadataStoreFactory.getStreamMetaDataStore(); KylinConfig config = KylinConfig.getInstanceFromEnv(); cubeAssignmentCache = CacheBuilder.newBuilder() .removalListener((RemovalNotification<String, List<ReplicaSet>> notification) -> logger .debug("{} is removed because {} ", notification.getKey(), notification.getCause())) .expireAfterWrite(300, TimeUnit.SECONDS) .build(); Broadcaster.getInstance(config).registerListener(new AssignCacheSyncListener(), ASSIGNMENT_ENTITY); }
Example #21
Source File: QueryWithTableACLTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Test public void testFailQuery() throws SQLException, IOException { QueryACLTestUtil.setUser(MODELER); QueryACLTestUtil.mockQuery(PROJECT, "select * from TEST_KYLIN_FACT"); QueryACLTestUtil.setUser(ADMIN); TableACLManager.getInstance(KylinConfig.getInstanceFromEnv()).addTableACL(PROJECT, "ADMIN", QUERY_TABLE, MetadataConstants.TYPE_USER); thrown.expectCause(CoreMatchers.isA(AccessDeniedException.class)); thrown.expectMessage(CoreMatchers.containsString("Query failed.Access table:DEFAULT.TEST_KYLIN_FACT denied")); QueryACLTestUtil.mockQuery(PROJECT, "select * from TEST_KYLIN_FACT"); }
Example #22
Source File: HBaseResourceStore.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
public HBaseResourceStore(KylinConfig kylinConfig) throws IOException { super(kylinConfig); metadataUrl = buildMetadataUrl(kylinConfig); tableName = metadataUrl.getIdentifier(); createHTableIfNeeded(tableName); kvSizeLimit = Integer .parseInt(getConnection().getConfiguration().get("hbase.client.keyvalue.maxsize", "10485760")); }
Example #23
Source File: ZookeeperAclBuilder.java From kylin with Apache License 2.0 | 5 votes |
public static List<ZKUtil.ZKAuthInfo> getZKAuths() throws Exception { // Parse Auths from configuration. String zkAuthConf = KylinConfig.getInstanceFromEnv().getZKAuths(); try { zkAuthConf = ZKUtil.resolveConfIndirection(zkAuthConf); if (zkAuthConf != null) { return ZKUtil.parseAuth(zkAuthConf); } else { return Collections.emptyList(); } } catch (Exception e) { logger.error("Couldn't read Auth based on 'kylin.env.zookeeper.zk-auth' in kylin.properties"); throw e; } }
Example #24
Source File: StreamingCubeDataSearcherPerfTest.java From kylin with Apache License 2.0 | 5 votes |
public StreamingCubeDataSearcherPerfTest() throws Exception { this.createTestMetadata(); this.baseStorePath = KylinConfig.getInstanceFromEnv().getStreamingIndexPath(); this.cubeInstance = CubeManager.getInstance(getTestConfig()).reloadCubeQuietly(cubeName); this.parsedStreamingCubeInfo = new ParsedStreamingCubeInfo(cubeInstance); setupCubeConfig(); this.testHelper = new TestHelper(cubeInstance); this.streamingSegmentManager = new StreamingSegmentManager(baseStorePath, cubeInstance, new MockPositionHandler(), null); this.searcher = streamingSegmentManager.getSearcher(); }
Example #25
Source File: SparkBuildDictionary.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
private void init() { try (KylinConfig.SetAndUnsetThreadLocalConfig autoUnset = KylinConfig .setAndUnsetThreadLocalConfig(config)) { cubeManager = CubeManager.getInstance(config); cubeSegment = cubeManager.getCube(cubeName).getSegmentById(segmentId); } initialized = true; }
Example #26
Source File: ZKUtil.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
/** * Get zookeeper connection string from kylin.properties */ public static String getZKConnectString(KylinConfig config) { String zkString = config.getZookeeperConnectString(); if (zkString == null) { zkString = getZKConnectStringFromHBase(); if (zkString == null) { throw new RuntimeException("Please set 'kylin.env.zookeeper-connect-string' in kylin.properties"); } } return zkString; }
Example #27
Source File: SnapshotManagerTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
public void runTestCase() throws IOException, InterruptedException { kylinConfig = KylinConfig.getInstanceFromEnv(); snapshotManager = SnapshotManager.getInstance(kylinConfig); SnapshotTable origin = snapshotManager.buildSnapshot(genTable("./origin", expect), genTableDesc("TEST_TABLE"), kylinConfig); // sleep 1 second to avoid file resource store precision lost Thread.sleep(1000); SnapshotTable dup = snapshotManager.buildSnapshot(genTable("./dup", expect), genTableDesc("TEST_TABLE"), kylinConfig); // assert same snapshot file Assert.assertEquals(origin.getUuid(), dup.getUuid()); Assert.assertEquals(origin.getResourcePath(), dup.getResourcePath()); // assert the file has been updated long originLastModified = origin.getLastModified(); long dupLastModified = dup.getLastModified(); Assert.assertTrue(dupLastModified > originLastModified); SnapshotTable actual = snapshotManager.getSnapshotTable(origin.getResourcePath()); IReadableTable.TableReader reader = actual.getReader(); Assert.assertEquals(expect.size(), actual.getRowCount()); int i = 0; while (reader.next()) { Assert.assertEquals(stringJoin(expect.get(i++)), stringJoin(reader.getRow())); } SnapshotTable difTable = snapshotManager.buildSnapshot(genTable("./dif", dif), genTableDesc("TEST_TABLE"), kylinConfig); Assert.assertNotEquals(origin.getUuid(), difTable.getUuid()); }
Example #28
Source File: ITStorageTest.java From kylin-on-parquet-v2 with Apache License 2.0 | 5 votes |
@Before public void setUp() throws Exception { this.createTestMetadata(); CubeManager cubeMgr = CubeManager.getInstance(getTestConfig()); cube = cubeMgr.getCube("test_kylin_cube_without_slr_left_join_empty"); Assert.assertNotNull(cube); storageEngine = StorageFactory.createQuery(cube); context = new StorageContext(); context.setConnUrl(KylinConfig.getInstanceFromEnv().getStorageUrl()); mockup = new StorageMockUtils(cube.getModel()); }
Example #29
Source File: JdbcTableReader.java From kylin with Apache License 2.0 | 5 votes |
/** * Constructor for reading whole jdbc table * @param dbName * @param tableName * @throws IOException */ public JdbcTableReader(String dbName, String tableName) throws IOException { this.dbName = dbName; this.tableName = tableName; KylinConfig config = KylinConfig.getInstanceFromEnv(); String connectionUrl = config.getJdbcSourceConnectionUrl(); String driverClass = config.getJdbcSourceDriver(); String jdbcUser = config.getJdbcSourceUser(); String jdbcPass = config.getJdbcSourcePass(); dbconf = new DBConnConf(driverClass, connectionUrl, jdbcUser, jdbcPass); jdbcCon = SqlUtil.getConnection(dbconf); IJdbcMetadata meta = JdbcMetadataFactory .getJdbcMetadata(SourceDialect.getDialect(config.getJdbcSourceDialect()), dbconf); Map<String, String> metadataCache = new TreeMap<>(); JdbcHiveInputBase.JdbcBaseBatchCubingInputSide.calCachedJdbcMeta(metadataCache, dbconf, meta); String database = JdbcHiveInputBase.getSchemaQuoted(metadataCache, dbName, meta, true); String table = JdbcHiveInputBase.getTableIdentityQuoted(dbName, tableName, metadataCache, meta, true); String sql = String.format(Locale.ROOT, "select * from %s.%s", database, table); try { statement = jdbcCon.createStatement(); rs = statement.executeQuery(sql); colCount = rs.getMetaData().getColumnCount(); } catch (SQLException e) { throw new IOException(String.format(Locale.ROOT, "error while exec %s", sql), e); } }
Example #30
Source File: QueryCli.java From kylin with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { Options options = new Options(); options.addOption(OPTION_METADATA); options.addOption(OPTION_SQL); CommandLineParser parser = new GnuParser(); CommandLine commandLine = parser.parse(options, args); KylinConfig config = KylinConfig.createInstanceFromUri(commandLine.getOptionValue(OPTION_METADATA.getOpt())); String sql = commandLine.getOptionValue(OPTION_SQL.getOpt()); Connection conn = null; Statement stmt = null; ResultSet rs = null; try { conn = QueryConnection.getConnection(null); stmt = conn.createStatement(); rs = stmt.executeQuery(sql); int n = 0; ResultSetMetaData meta = rs.getMetaData(); while (rs.next()) { n++; for (int i = 1; i <= meta.getColumnCount(); i++) { System.out.println(n + " - " + meta.getColumnLabel(i) + ":\t" + rs.getObject(i)); } } } finally { DBUtils.closeQuietly(rs); DBUtils.closeQuietly(stmt); DBUtils.closeQuietly(conn); } }