package com.imooc.bigdata.integration.jdbc; import com.google.common.collect.Maps; import org.apache.storm.Config; import org.apache.storm.LocalCluster; import org.apache.storm.jdbc.bolt.JdbcInsertBolt; import org.apache.storm.jdbc.common.ConnectionProvider; import org.apache.storm.jdbc.common.HikariCPConnectionProvider; import org.apache.storm.jdbc.mapper.JdbcMapper; import org.apache.storm.jdbc.mapper.SimpleJdbcMapper; import org.apache.storm.redis.bolt.RedisStoreBolt; import org.apache.storm.redis.common.config.JedisPoolConfig; import org.apache.storm.redis.common.mapper.RedisDataTypeDescription; import org.apache.storm.redis.common.mapper.RedisStoreMapper; import org.apache.storm.spout.SpoutOutputCollector; import org.apache.storm.task.OutputCollector; import org.apache.storm.task.TopologyContext; import org.apache.storm.topology.OutputFieldsDeclarer; import org.apache.storm.topology.TopologyBuilder; import org.apache.storm.topology.base.BaseRichBolt; import org.apache.storm.topology.base.BaseRichSpout; import org.apache.storm.tuple.Fields; import org.apache.storm.tuple.ITuple; import org.apache.storm.tuple.Tuple; import org.apache.storm.tuple.Values; import org.apache.storm.utils.Utils; import java.util.HashMap; import java.util.Map; import java.util.Random; public class LocalWordCountJDBCStormTopology { /** * 使用 Strom 完成词频统计功能 */ public static class DataSourceSpout extends BaseRichSpout { private SpoutOutputCollector collector; public static final String[] words = new String[]{"apple", "orange", "pineapple", "banana", "watermelon"}; @Override public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { this.collector = collector; } /** * 1.读取指定目录下的文件数据 * 2.把读取到的每一行数据发送出去 */ @Override public void nextTuple() { Random random = new Random(); String word = words[random.nextInt(words.length)]; this.collector.emit(new Values(word)); System.out.println("emit: " + word); Utils.sleep(1000); } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("line")); } } /** * 对数据进行分割 */ public static class SplitBolt extends BaseRichBolt { private OutputCollector collector; @Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { this.collector = collector; } /** * 对line按照逗号进行分割 * * @param input */ @Override public void execute(Tuple input) { String word = input.getStringByField("line"); this.collector.emit(new Values(word)); } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("word")); } } /** * 词频汇总Bolt */ public static class CountBolt extends BaseRichBolt { private OutputCollector collector; @Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { this.collector = collector; } Map<String, Integer> map = new HashMap<String, Integer>(); /** * 1.获取每个单词 * 2.对所有单词进行汇总 * 3.输出 * * @param input */ @Override public void execute(Tuple input) { //获取每个单词 String word = input.getStringByField("word"); //对所有单词汇总 Integer count = map.get(word); if (count == null) { count = 0; } count++; map.put(word, count); //输出 this.collector.emit(new Values(word, map.get(word))); } @Override public void declareOutputFields(OutputFieldsDeclarer declarer) { declarer.declare(new Fields("word", "word_count")); } } public static void main(String[] args) { //根据Spout和Bolt构建TopologyBuilder TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("DataSourceSpout", new DataSourceSpout()); builder.setBolt("SplitBolt", new SplitBolt()).shuffleGrouping("DataSourceSpout"); builder.setBolt("CountBolt", new CountBolt()).shuffleGrouping("SplitBolt"); Map hikariConfigMap = Maps.newHashMap(); hikariConfigMap.put("dataSourceClassName","com.mysql.jdbc.jdbc2.optional.MysqlDataSource"); hikariConfigMap.put("dataSource.url", "jdbc:mysql://192.168.60.11/storm"); hikariConfigMap.put("dataSource.user","root"); hikariConfigMap.put("dataSource.password","123"); ConnectionProvider connectionProvider = new HikariCPConnectionProvider(hikariConfigMap); String tableName = "wc"; JdbcMapper simpleJdbcMapper = new SimpleJdbcMapper(tableName, connectionProvider); JdbcInsertBolt userPersistanceBolt = new JdbcInsertBolt(connectionProvider, simpleJdbcMapper) .withTableName(tableName) .withQueryTimeoutSecs(30); builder.setBolt("JdbcInsertBolt", userPersistanceBolt).shuffleGrouping("CountBolt"); //创建本地集群 LocalCluster cluster = new LocalCluster(); cluster.submitTopology("LocalWordCountRedisStormTopology", new Config(), builder.createTopology()); } }