Java Code Examples for org.apache.kylin.common.KylinConfig#getInstanceFromEnv()

@Test
public void testOverwriteJobConf() throws Exception {
    MapReduceExecutable executable = new MapReduceExecutable();
    KylinConfig config = KylinConfig.getInstanceFromEnv();

    Method method = MapReduceExecutable.class.getDeclaredMethod("overwriteJobConf", Configuration.class,
            KylinConfig.class, new String[] {}.getClass());
    method.setAccessible(true);
    Configuration conf = new Configuration();
    conf.set("mapreduce.job.is-mem-hungry", "true");
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("mem-test1", conf.get("test1"));
    Assert.assertEquals("mem-test2", conf.get("test2"));

    conf.clear();
    method.invoke(executable, conf, config, new String[] { "-cubename", "ci_inner_join_cube" });
    Assert.assertEquals("test1", conf.get("test1"));
    Assert.assertEquals("test2", conf.get("test2"));
}
 

public static void staticCreateTestMetadata(boolean useTestMeta, MetadataTestCaseHook hook) {
    try {
        KylinConfig.destroyInstance();

        FileUtils.deleteDirectory(new File(LOCALMETA_TEMP_DATA));
        if (useTestMeta) {
            FileUtils.copyDirectory(new File(LOCALMETA_TEST_DATA), new File(LOCALMETA_TEMP_DATA));
        }

        if (System.getProperty(KylinConfig.KYLIN_CONF) == null && System.getenv(KylinConfig.KYLIN_CONF) == null) {
            System.setProperty(KylinConfig.KYLIN_CONF, LOCALMETA_TEMP_DATA);
        }

        if (hook != null) {
            hook.hook();
        }
        KylinConfig config = KylinConfig.getInstanceFromEnv();
        config.setMetadataUrl(LOCALMETA_TEMP_DATA);
        config.setProperty("kylin.env.hdfs-working-dir", "file:///tmp/kylin");
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

public static long estimatedCubeSize(String cubeName, long[] cardinality) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    CubeManager cubeManager = CubeManager.getInstance(config);
    CubeInstance cubeInstance = cubeManager.getCube(cubeName);
    CubeDesc cubeDesc = cubeInstance.getDescriptor();

    CuboidScheduler scheduler = new CuboidScheduler(cubeDesc);
    long baseCuboid = Cuboid.getBaseCuboidId(cubeDesc);
    LinkedList<Long> cuboidQueue = new LinkedList<Long>();
    cuboidQueue.push(baseCuboid);

    long totalSpace = 0;

    while (!cuboidQueue.isEmpty()) {
        long cuboidID = cuboidQueue.pop();
        Collection<Long> spanningCuboid = scheduler.getSpanningCuboid(cuboidID);
        for (Long sc : spanningCuboid) {
            cuboidQueue.push(sc);
        }

        totalSpace += estimateCuboidSpace(cuboidID, cardinality, cubeDesc);
    }
    return totalSpace;
}
 

@Override
public void init(DictionaryInfo dictInfo, int baseId, String hdfsDir) throws IOException {
    sourceColumn = dictInfo.getSourceTable() + "." + dictInfo.getSourceColumn();

    KylinConfig config = KylinConfig.getInstanceFromEnv();
    int maxEntriesPerSlice = config.getAppendDictEntrySize();
    if (hdfsDir == null) {
        //build in Kylin job server
        hdfsDir = KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory();
    }

    //use UUID to make each segment dict in different HDFS dir and support concurrent build
    //use timestamp to make the segment dict easily to delete
    String baseDir = hdfsDir + "resources/SegmentDict" + dictInfo.getResourceDir() + "/"
            + RandomUtil.randomUUID().toString() + "_" + System.currentTimeMillis() + "/";

    this.builder = new AppendTrieDictionaryBuilder(baseDir, maxEntriesPerSlice, false);
    this.baseId = baseId;
}
 

public void addMaterializeLookupTableSteps(LookupMaterializeContext context, String tableName, SnapshotTableDesc snapshotTableDesc) {
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig);
    TableDesc tableDesc = TableMetadataManager.getInstance(kylinConfig).getTableDesc(tableName, cube.getProject());
    IReadableTable sourceTable = SourceManager.createReadableTable(tableDesc, context.getJobFlow().getId());
    try {
        ExtTableSnapshotInfo latestSnapshot = extTableSnapshotInfoManager.getLatestSnapshot(
                sourceTable.getSignature(), tableName);
        if (latestSnapshot != null) {
            logger.info("there is latest snapshot exist for table:{}, skip build snapshot step.", tableName);
            context.addLookupSnapshotPath(tableName, latestSnapshot.getResourcePath());
            return;
        }
    } catch (IOException ioException) {
        throw new RuntimeException(ioException);
    }
    logger.info("add build snapshot steps for table:{}", tableName);
    String snapshotID = genLookupSnapshotID();
    context.addLookupSnapshotPath(tableName, ExtTableSnapshotInfo.getResourcePath(tableName, snapshotID));
    addLookupTableConvertToHFilesStep(context.getJobFlow(), tableName, snapshotID);
    addLookupTableHFilesBulkLoadStep(context.getJobFlow(), tableName, snapshotID);
    if (snapshotTableDesc !=null && snapshotTableDesc.isEnableLocalCache()) {
        addUpdateSnapshotQueryCacheStep(context.getJobFlow(), tableName, snapshotID);
    }
}
 

protected void doAccessControl(List<OLAPContext> contexts) {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    String controllerCls = config.getQueryAccessController();
    if (null != controllerCls && !controllerCls.isEmpty()) {
        OLAPContext.IAccessController accessController = (OLAPContext.IAccessController) ClassUtil.newInstance(controllerCls);
        accessController.check(contexts, config);
    }
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    try {
        sqoopFlatHiveTable(config);
        return new ExecuteResult(ExecuteResult.State.SUCCEED, stepLogger.getBufferedLog());

    } catch (Exception e) {
        logger.error("job:" + getId() + " execute finished with exception", e);
        return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog(), e);
    }
}
 

@Override
protected ExecuteResult doWork(ExecutableContext context) throws ExecuteException {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    try {
        createFlatHiveTable(config);
        return new ExecuteResult(ExecuteResult.State.SUCCEED, stepLogger.getBufferedLog());

    } catch (Exception e) {
        logger.error("job:" + getId() + " execute finished with exception", e);
        return new ExecuteResult(ExecuteResult.State.ERROR, stepLogger.getBufferedLog(), e);
    }
}
 

public HiveCmdBuilder(String jobName) {
    kylinConfig = KylinConfig.getInstanceFromEnv();
    hiveConfProps = SourceConfigurationUtil.loadHiveConfiguration();
    hiveConfProps.putAll(kylinConfig.getHiveConfigOverride());
    if (StringUtils.isNotEmpty(jobName)) {
        addStatement("set mapred.job.name='" + jobName + "';");
    }
}
 

protected void attachKylinPropsAndMetadata(IIInstance ii, Configuration conf) throws IOException {
    File tmp = File.createTempFile("kylin_job_meta", "");
    tmp.delete(); // we need a directory, so delete the file first

    File metaDir = new File(tmp, "meta");
    metaDir.mkdirs();
    metaDir.getParentFile().deleteOnExit();

    // write kylin.properties
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    File kylinPropsFile = new File(metaDir, "kylin.properties");
    kylinConfig.writeProperties(kylinPropsFile);

    // write II / model_desc / II_desc / dict / table
    ArrayList<String> dumpList = new ArrayList<String>();
    dumpList.add(ii.getResourcePath());
    dumpList.add(ii.getDescriptor().getModel().getResourcePath());
    dumpList.add(ii.getDescriptor().getResourcePath());

    for (String tableName : ii.getDescriptor().getModel().getAllTables()) {
        TableDesc table = MetadataManager.getInstance(kylinConfig).getTableDesc(tableName);
        dumpList.add(table.getResourcePath());
    }

    for (IISegment segment : ii.getSegments()) {
        dumpList.addAll(segment.getDictionaryPaths());
    }

    dumpResources(kylinConfig, metaDir, dumpList);

    // hadoop distributed cache
    conf.set("tmpfiles", "file:///" + OptionsHelper.convertToFileURL(metaDir.getAbsolutePath()));
}
 

@Override
public IStreamingConnector createStreamingConnector(String cubeName, List<Partition> assignedPartitions,
        ConsumerStartProtocol startProtocol, StreamingSegmentManager streamingSegmentManager) {
    logger.info("Create StreamingConnector for Cube {}, assignedPartitions {}, startProtocol {}", cubeName,
            assignedPartitions, startProtocol);
    try {
        KylinConfig kylinConf = KylinConfig.getInstanceFromEnv();
        CubeInstance cubeInstance = CubeManager.getInstance(kylinConf).getCube(cubeName);
        IStreamingSource streamingSource = StreamingSourceFactory.getStreamingSource(cubeInstance);
        String streamingName = cubeInstance.getRootFactTable();
        StreamingSourceConfig streamingSourceConfig = StreamingSourceConfigManager.getInstance(kylinConf)
                .getConfig(streamingName);
        String topic = getTopicName(streamingSourceConfig.getProperties());
        Map<String, Object> conf = getKafkaConf(streamingSourceConfig.getProperties(), cubeInstance.getConfig());

        Class<?> clazz = getStreamingMessageParserClass(streamingSourceConfig.getProperties());
        Constructor<?> constructor = clazz.getConstructor(CubeDesc.class, MessageParserInfo.class);
        IStreamingMessageParser<?> parser = (IStreamingMessageParser<?>) constructor
                .newInstance(cubeInstance.getDescriptor(), streamingSourceConfig.getParserInfo());
        KafkaConnector connector = new KafkaConnector(conf, topic, parser, this);
        if (startProtocol != null) {
            if (startProtocol.getStartPosition() != null && startProtocol.getStartPosition().length() > 0) {
                KafkaPosition position = (KafkaPosition) streamingSource.getSourcePositionHandler().parsePosition(startProtocol.getStartPosition());
                connector.setStartPartition(assignedPartitions, startProtocol.getStartMode(),
                        position.getPartitionOffsets());
                streamingSegmentManager.restoreConsumerStates(position);
            } else {
                connector.setStartPartition(assignedPartitions, startProtocol.getStartMode(),
                        null);
            }
            streamingSegmentManager.checkpoint();
        } else if (streamingSegmentManager != null) {
            setupConnectorFromCheckpoint(connector, assignedPartitions, streamingSource, streamingSegmentManager);
        }

        return connector;
    } catch (Exception e) {
        throw new StreamingException("streaming connector create fail, cube:" + cubeName, e);
    }
}
 

public ExampleServer(String address) throws Exception {
    this.address = address;

    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    KylinConfig kylinConfig1 = KylinConfig.createKylinConfig(kylinConfig);
    kylinConfig1.setProperty("kylin.server.host-address", address);

    CuratorFramework client = ZKUtil.newZookeeperClient(kylinConfig1);
    scheduler = new CuratorScheduler(client);
    scheduler.init(new JobEngineConfig(kylinConfig1), new MockJobLock());
    if (!scheduler.hasStarted()) {
        throw new RuntimeException("scheduler has not been started");
    }
}
 

private LinkedList<String> createRow(TreeMap<String, String> factTableCol2LookupCol, TreeSet<String> usedCols, TreeSet<String> defaultColumns) throws Exception {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    LinkedList<String> columnValues = new LinkedList<String>();

    for (ColumnDesc cDesc : MetadataManager.getInstance(config).getTableDesc(factTableName).getColumns()) {

        String colName = cDesc.getName();

        if (factTableCol2LookupCol.containsKey(colName)) {

            // if the current column is a fk column in fact table
            ArrayList<String> candidates = this.feasibleValues.get(factTableCol2LookupCol.get(colName));

            columnValues.add(candidates.get(r.nextInt(candidates.size())));
        } else if (usedCols.contains(colName)) {

            // if the current column is a metric column in fact table
            columnValues.add(createCell(cDesc));
        } else {

            // otherwise this column is not useful in OLAP
            columnValues.add(createDefaultsCell(cDesc.getTypeName()));
            defaultColumns.add(colName);
        }
    }

    return columnValues;
}
 

static JdbcPushDownConnectionManager getConnectionManager(String id) throws ClassNotFoundException {
    if (manager == null || id != null) {
        synchronized (JdbcPushDownConnectionManager.class) {
            if (manager == null || id != null) {
                manager = new JdbcPushDownConnectionManager(KylinConfig.getInstanceFromEnv(),
                        id);
            }
        }
    }
    return manager;
}
 

static JdbcPushDownConnectionManager getConnectionManager(String id) throws ClassNotFoundException {
    if (manager == null) {
        synchronized (JdbcPushDownConnectionManager.class) {
            if (manager == null) {
                manager = new JdbcPushDownConnectionManager(KylinConfig.getInstanceFromEnv(),
                        id);
            }
        }
    }
    return manager;
}
 

@BeforeClass
public static void setupClass() throws SQLException {
    staticCreateTestMetadata();
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    kylinConfig.setProperty("kylin.source.hive.quote-enabled", "true");
}
 

@Override
public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(OPTION_INPUT_PATH);
    options.addOption(OPTION_TABLE_NAME);
    options.addOption(OPTION_CUBING_JOB_ID);
    options.addOption(OPTION_LOOKUP_SNAPSHOT_ID);
    parseOptions(options, args);

    String tableName = getOptionValue(OPTION_TABLE_NAME);
    String cubingJobID = getOptionValue(OPTION_CUBING_JOB_ID);
    String snapshotID = getOptionValue(OPTION_LOOKUP_SNAPSHOT_ID);

    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    ExecutableManager execMgr = ExecutableManager.getInstance(kylinConfig);
    DefaultChainedExecutable job = (DefaultChainedExecutable) execMgr.getJob(cubingJobID);

    ExtTableSnapshotInfoManager extTableSnapshotInfoManager = ExtTableSnapshotInfoManager.getInstance(kylinConfig);
    ExtTableSnapshotInfo snapshot = extTableSnapshotInfoManager.getSnapshot(tableName, snapshotID);
    long srcTableRowCnt = Long.parseLong(job.findExtraInfoBackward(BatchConstants.LOOKUP_EXT_SNAPSHOT_SRC_RECORD_CNT_PFX + tableName, "-1"));
    logger.info("update table:{} snapshot row count:{}", tableName, srcTableRowCnt);
    snapshot.setRowCnt(srcTableRowCnt);
    snapshot.setLastBuildTime(System.currentTimeMillis());
    extTableSnapshotInfoManager.updateSnapshot(snapshot);

    String hTableName = snapshot.getStorageLocationIdentifier();
    // e.g
    // /tmp/kylin-3f150b00-3332-41ca-9d3d-652f67f044d7/test_kylin_cube_with_slr_ready_2_segments/hfile/
    // end with "/"
    String input = getOptionValue(OPTION_INPUT_PATH);

    Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
    FsShell shell = new FsShell(conf);

    int exitCode = -1;
    int retryCount = 10;
    while (exitCode != 0 && retryCount >= 1) {
        exitCode = shell.run(new String[] { "-chmod", "-R", "777", input });
        retryCount--;
        Thread.sleep(5000);
    }

    if (exitCode != 0) {
        logger.error("Failed to change the file permissions: {}", input);
        throw new IOException("Failed to change the file permissions: " + input);
    }

    String[] newArgs = new String[2];
    newArgs[0] = input;
    newArgs[1] = hTableName;

    logger.debug("Start to run LoadIncrementalHFiles");
    int ret = MRUtil.runMRJob(new LoadIncrementalHFiles(conf), newArgs);
    logger.debug("End to run LoadIncrementalHFiles");
    return ret;
}
 

@Before
public void setup() throws Exception {
    this.createTestMetadata();
    kylinConfig = KylinConfig.getInstanceFromEnv();
}
 

public static StreamMetadataStore getStreamMetaDataStore() {
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    return getStreamMetaDataStore(config.getStreamingMetadataStoreType());
}
 

@Before
public void setup() throws Exception {
    createTestMetadata();
    config = KylinConfig.getInstanceFromEnv();
    lookupTable = initLookupTable();
}