backtype.storm.spout.Scheme Java Examples

private MultiScheme createMultiScheme(Map conf, String topic, String schemeClsName) throws Exception {
    Object scheme = SchemeBuilder.buildFromClsName(schemeClsName, topic, conf);
    if (scheme instanceof MultiScheme) {
        return (MultiScheme) scheme;
    } else if (scheme instanceof Scheme) {
        return new SchemeAsMultiScheme((Scheme) scheme);
    } else {
        LOG.error("create spout scheme failed.");
        throw new IllegalArgumentException("create spout scheme failed.");
    }
}
 

@Override
public KafkaStreamSourceConfig getSourceConfig(String streamId, Config config) {
    KafkaStreamSourceConfig sourceConfig = new KafkaStreamSourceConfig();

    sourceConfig.setTopicId(getSourceTopicName(streamId,config));
    sourceConfig.setBrokerZkQuorum(config.getString("dataSourceConfig.zkConnection"));

    if (hasNonBlankConfigPath(config, "dataSourceConfig.fetchSize")) {
        sourceConfig.setFetchSize(config.getInt("dataSourceConfig.fetchSize"));
    }
    if (hasNonBlankConfigPath(config, "dataSourceConfig.transactionZKRoot")) {
        sourceConfig.setTransactionZKRoot(config.getString("dataSourceConfig.transactionZKRoot"));
    }
    if (hasNonBlankConfigPath(config, "dataSourceConfig.consumerGroupId")) {
        sourceConfig.setConsumerGroupId(config.getString("dataSourceConfig.consumerGroupId"));
    }
    if (hasNonBlankConfigPath(config, "dataSourceConfig.brokerZkPath")) {
        sourceConfig.setBrokerZkPath(config.getString("dataSourceConfig.brokerZkPath"));
    }
    if (hasNonBlankConfigPath(config, "dataSourceConfig.txZkServers")) {
        sourceConfig.setTransactionZkServers(config.getString("dataSourceConfig.txZkServers"));
    }
    if (hasNonBlankConfigPath(config, "dataSourceConfig.transactionStateUpdateMS")) {
        sourceConfig.setTransactionStateUpdateMS(config.getLong("dataSourceConfig.transactionStateUpdateMS"));
    }
    if (hasNonBlankConfigPath(config, "dataSourceConfig.startOffsetTime")) {
        sourceConfig.setStartOffsetTime(config.getInt("dataSourceConfig.startOffsetTime"));
    }
    if (hasNonBlankConfigPath(config, "dataSourceConfig.forceFromStart")) {
        sourceConfig.setForceFromStart(config.getBoolean("dataSourceConfig.forceFromStart"));
    }
    String schemeCls = getSourceSchemeCls(streamId, config);
    if (schemeCls != null && StringUtils.isNotBlank(schemeCls)) {
        try {
            sourceConfig.setSchemaClass((Class<? extends Scheme>) Class.forName(schemeCls));
        } catch (ClassNotFoundException e) {
            LOG.error("Class not found error, dataSourceConfig.schemeCls = {}", schemeCls, e);
        }
    }
    return sourceConfig;
}
 

private static KafkaSpout createKafkaSpout(KafkaStreamSourceConfig config) {

        // the following is for fetching data from one topic
        // Kafka topic
        String topic = config.getTopicId();
        // Kafka broker zk connection
        String zkConnString = config.getBrokerZkQuorum();
        // Kafka fetch size
        int fetchSize = config.getFetchSize();
        LOG.info(String.format("Use topic : %s, zkQuorum : %s , fetchSize : %d", topic, zkConnString, fetchSize));

        /*
         the following is for recording offset for processing the data
         the zk path to store current offset is comprised of the following
         offset zkPath = zkRoot + "/" + topic + "/" + consumerGroupId + "/" + partition_Id

         consumerGroupId is for differentiating different consumers which consume the same topic
        */
        // transaction zkRoot
        String zkRoot = config.getTransactionZKRoot();
        // Kafka consumer group id
        String groupId = config.getConsumerGroupId();
        String brokerZkPath = config.getBrokerZkPath();

        BrokerHosts hosts;
        if (StringUtils.isNotBlank(brokerZkPath)) {
            hosts = new ZkHosts(zkConnString);
        } else {
            hosts = new ZkHosts(zkConnString, brokerZkPath);
        }

        SpoutConfig spoutConfig = new SpoutConfig(hosts,
            topic,
            zkRoot + "/" + topic,
            groupId);

        // transaction zkServers to store kafka consumer offset. Default to use storm zookeeper
        if (StringUtils.isNotBlank(config.getTransactionZkServers())) {
            String[] txZkServers = config.getTransactionZkServers().split(",");
            spoutConfig.zkServers = Arrays.stream(txZkServers).map(server -> server.split(":")[0]).collect(Collectors.toList());
            spoutConfig.zkPort = Integer.parseInt(txZkServers[0].split(":")[1]);
            LOG.info("txZkServers:" + spoutConfig.zkServers + ", zkPort:" + spoutConfig.zkPort);
        }

        // transaction update interval
        spoutConfig.stateUpdateIntervalMs = config.getTransactionStateUpdateMS();
        // Kafka fetch size
        spoutConfig.fetchSizeBytes = fetchSize;
        spoutConfig.startOffsetTime = kafka.api.OffsetRequest.LatestTime();

        // "startOffsetTime" is for test usage, prod should not use this
        if (config.getStartOffsetTime() >= 0) {
            spoutConfig.startOffsetTime = config.getStartOffsetTime();
        }
        // "forceFromStart" is for test usage, prod should not use this
        if (config.isForceFromStart()) {
            spoutConfig.startOffsetTime = kafka.api.OffsetRequest.EarliestTime();
        }

        Preconditions.checkNotNull(config.getSchemaClass(), "schemaClass is null");
        try {
            Scheme s = config.getSchemaClass().newInstance();
            spoutConfig.scheme = new SchemeAsMultiScheme(s);
        } catch (Exception ex) {
            LOG.error("Error instantiating scheme object");
            throw new IllegalStateException(ex);
        }
        return new KafkaSpout(spoutConfig);
    }
 

Scheme getScheme(); 

void setScheme(Scheme scheme);