org.apache.storm.LocalCluster Java Examples
The following examples show how to use
org.apache.storm.LocalCluster.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TopologyStarter.java From breeze with Apache License 2.0 | 6 votes |
@Override public void run() { properties.put(Config.TOPOLOGY_NAME, ID); Config config = stormConfig(properties); ApplicationContext spring = SingletonApplicationContext.loadXml(config, MAIN_CONTEXT); try { StormTopology topology = spring.getBean(ID, StormTopology.class); Properties systemProperties = System.getProperties(); if (systemProperties.containsKey(LOCAL_RUN_PARAM)) { String timeout = systemProperties.getProperty(LOCAL_RUN_PARAM); if (! hasText(timeout)) timeout = LOCAL_RUN_DEFAULT_TIMEOUT; long ms = 1000L * Integer.parseInt(timeout); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(ID, config, topology); sleep(ms); cluster.shutdown(); } else StormSubmitter.submitTopology(ID, config, topology); } catch (Exception e) { e.printStackTrace(); exit(255); } }
Example #2
Source File: NginxStorm.java From storm-nginx-log with MIT License | 6 votes |
public static void main(String[] argv) throws InterruptedException { Config config = new Config(); config.setDebug(true); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("LogSpout", new LogSpout(), 1); builder.setBolt("SpliteBolt", new SpliteBolt(), 1).shuffleGrouping("LogSpout"); builder.setBolt("CounterBolt", new CounterBolt(), 1) .fieldsGrouping("SpliteBolt", new Fields("item")); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("NginxLog", config, builder.createTopology()); // Thread.sleep(10000); // // cluster.killTopology("NginxLog"); // cluster.shutdown(); }
Example #3
Source File: SinkTopology.java From DBus with Apache License 2.0 | 6 votes |
private void start(StormTopology topology, boolean runAsLocal) throws Exception { Config conf = new Config(); conf.put(Constants.ZOOKEEPER_SERVERS, zkConnect); conf.put(Constants.TOPOLOGY_ID, topologyId); conf.put(Constants.SINK_TYPE, sinkType); conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, sinkerConf.getProperty(SinkerConstants.TOPOLOGY_WORKER_CHILDOPTS)); conf.setMessageTimeoutSecs(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_MESSAGE_TIMEOUT))); conf.setMaxSpoutPending(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_MAX_SPOUT_PENDING))); conf.setDebug(true); conf.setNumWorkers(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_NUM_WORKERS))); if (runAsLocal) { conf.setMaxTaskParallelism(10); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(topologyName, conf, topology); } else { StormSubmitter.submitTopology(topologyName, conf, topology); } }
Example #4
Source File: DBusLogProcessorTopology.java From DBus with Apache License 2.0 | 6 votes |
private void start(StormTopology topology, boolean runAsLocal) throws Exception { Config conf = new Config(); conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zkConnect); conf.put(Constants.TOPOLOGY_ID, topologyId); conf.setMessageTimeoutSecs(Integer.parseInt(properties.getProperty(Constants.LOG_MESSAGE_TIMEOUT))); //conf.setMaxSpoutPending(30); conf.setDebug(true); conf.setNumWorkers(Integer.parseInt(properties.getProperty(Constants.LOG_NUMWORKERS))); if (runAsLocal) { conf.setMaxTaskParallelism(10); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(topologyName, conf, topology); } else { StormSubmitter.submitTopology(topologyName, conf, topology); } }
Example #5
Source File: FullPullerTopology.java From DBus with Apache License 2.0 | 6 votes |
private void start(StormTopology topology, boolean runAsLocal) throws Exception { Config conf = new Config(); conf.put(FullPullConstants.FULL_SPLITTER_TOPOLOGY_ID, fullSplitterTopologyId); conf.put(FullPullConstants.FULL_PULLER_TOPOLOGY_ID, fullPullerTopologyId); conf.put(FullPullConstants.DS_NAME, topologyId); conf.put(FullPullConstants.ZKCONNECT, zkConnect); conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, commonConfSplit.getProperty(FullPullConstants.TOPOLOGY_WORKER_CHILDOPTS)); //设置message超时时间为,保证每个分片都能在该内拉完数据 conf.setMessageTimeoutSecs(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_MESSAGE_TIMEOUT))); conf.setMaxSpoutPending(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_MAX_SPOUT_PENDING))); conf.setNumWorkers(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_NUM_WORKERS))); conf.setDebug(true); if (runAsLocal) { conf.setMaxTaskParallelism(3); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(topologyName, conf, topology); } else { StormSubmitter.submitTopology(topologyName, conf, topology); } }
Example #6
Source File: MultiSpoutExclamationTopology.java From incubator-heron with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("word0", new TestWordSpout(), 2); builder.setSpout("word1", new TestWordSpout(), 2); builder.setSpout("word2", new TestWordSpout(), 2); builder.setBolt("exclaim1", new ExclamationBolt(), 2) .shuffleGrouping("word0") .shuffleGrouping("word1") .shuffleGrouping("word2"); Config conf = new Config(); conf.setDebug(true); conf.setMaxSpoutPending(10); conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError"); if (args != null && args.length > 0) { conf.setNumWorkers(3); StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); } else { LocalCluster cluster = new LocalCluster(); cluster.submitTopology("test", conf, builder.createTopology()); Utils.sleep(10000); cluster.killTopology("test"); cluster.shutdown(); } }
Example #7
Source File: ScottyDemoTopology.java From scotty-window-processor with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { LocalCluster cluster = new LocalCluster(); TopologyBuilder builder = new TopologyBuilder(); Config conf = new Config(); conf.setDebug(false); conf.setNumWorkers(1); conf.setMaxTaskParallelism(1); //Disable Acking conf.setNumAckers(0); KeyedScottyWindowOperator scottyBolt = new KeyedScottyWindowOperator<Integer, Integer>(new Sum(), 0); scottyBolt.addWindow(new TumblingWindow(WindowMeasure.Time, 1000)); scottyBolt.addWindow(new SlidingWindow(WindowMeasure.Time, 1000, 250)); scottyBolt.addWindow(new SessionWindow(WindowMeasure.Time, 1000)); builder.setSpout("spout", new DataGeneratorSpout()); builder.setBolt("scottyWindow", scottyBolt).fieldsGrouping("spout", new Fields("key")); builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("scottyWindow"); cluster.submitTopology("testTopology", conf, builder.createTopology()); //cluster.killTopology("testTopology"); //cluster.shutdown(); }
Example #8
Source File: StormRangerAuthorizerTest.java From ranger with Apache License 2.0 | 6 votes |
@org.junit.BeforeClass public static void setup() throws Exception { cluster = new LocalCluster(); final Config conf = new Config(); conf.setDebug(true); final TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("words", new WordSpout()); builder.setBolt("counter", new WordCounterBolt()).shuffleGrouping("words"); // bob can create a new topology final Subject subject = new Subject(); subject.getPrincipals().add(new SimplePrincipal("bob")); Subject.doAs(subject, new PrivilegedExceptionAction<Void>() { public Void run() throws Exception { cluster.submitTopology("word-count", conf, builder.createTopology()); return null; } }); }
Example #9
Source File: StormMain.java From chuidiang-ejemplos with GNU Lesser General Public License v3.0 | 6 votes |
public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException, AuthorizationException { TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(DATA_GENERATOR, new ASpout()); builder.setBolt(DATA_CALCULATOR, new ABolt()).shuffleGrouping(DATA_GENERATOR); builder.setBolt(DATA_PRINTER, new DataPrinter()).shuffleGrouping(DATA_CALCULATOR).shuffleGrouping(DATA_GENERATOR); Config config = new Config(); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology()); Thread.sleep(100000); cluster.killTopology(TOPOLOGY_NAME); cluster.shutdown(); }
Example #10
Source File: LocalWordCountRedisStormTopology.java From 163-bigdate-note with GNU General Public License v3.0 | 6 votes |
public static void main(String[] args) { //根据Spout和Bolt构建TopologyBuilder TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("DataSourceSpout", new DataSourceSpout()); builder.setBolt("SplitBolt", new SplitBolt()).shuffleGrouping("DataSourceSpout"); builder.setBolt("CountBolt", new CountBolt()).shuffleGrouping("SplitBolt"); JedisPoolConfig poolConfig = new JedisPoolConfig.Builder() .setHost("192.168.60.11").setPort(6379).build(); RedisStoreMapper storeMapper = new WordCountStoreMapper(); RedisStoreBolt storeBolt = new RedisStoreBolt(poolConfig, storeMapper); builder.setBolt("RedisStoreBolt", storeBolt).shuffleGrouping("CountBolt"); //创建本地集群 LocalCluster cluster = new LocalCluster(); cluster.submitTopology("LocalWordCountRedisStormTopology", new Config(), builder.createTopology()); }
Example #11
Source File: StatisticTopology.java From storm-statistic with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); /** * 设置spout和bolt的dag(有向无环图) */ KafkaSpout kafkaSpout = createKafkaSpout(); builder.setSpout("id_kafka_spout", kafkaSpout); builder.setBolt("id_convertIp_bolt", new ConvertIPBolt()).shuffleGrouping("id_kafka_spout"); // 通过不同的数据流转方式,来指定数据的上游组件 builder.setBolt("id_statistic_bolt", new StatisticBolt()).shuffleGrouping("id_convertIp_bolt"); // 通过不同的数据流转方式,来指定数据的上游组件 // 使用builder构建topology StormTopology topology = builder.createTopology(); String topologyName = KafkaStormTopology.class.getSimpleName(); // 拓扑的名称 Config config = new Config(); // Config()对象继承自HashMap,但本身封装了一些基本的配置 // 启动topology,本地启动使用LocalCluster,集群启动使用StormSubmitter if (args == null || args.length < 1) { // 没有参数时使用本地模式,有参数时使用集群模式 LocalCluster localCluster = new LocalCluster(); // 本地开发模式,创建的对象为LocalCluster localCluster.submitTopology(topologyName, config, topology); } else { StormSubmitter.submitTopology(topologyName, config, topology); } }
Example #12
Source File: TopologyRunner.java From tutorials with MIT License | 6 votes |
public static void runTopology() { String filePath = "./src/main/resources/operations.txt"; TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("randomNumberSpout", new RandomNumberSpout()); builder.setBolt("filteringBolt", new FilteringBolt()).shuffleGrouping("randomNumberSpout"); builder.setBolt("aggregatingBolt", new AggregatingBolt() .withTimestampField("timestamp") .withLag(BaseWindowedBolt.Duration.seconds(1)) .withWindow(BaseWindowedBolt.Duration.seconds(5))).shuffleGrouping("filteringBolt"); builder.setBolt("fileBolt", new FileWritingBolt(filePath)).shuffleGrouping("aggregatingBolt"); Config config = new Config(); config.setDebug(false); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("Test", config, builder.createTopology()); }
Example #13
Source File: ConfigurableTopology.java From storm-crawler with Apache License 2.0 | 6 votes |
/** Submits the topology under a specific name **/ protected int submit(String name, Config conf, TopologyBuilder builder) { // register Metadata for serialization with FieldsSerializer Config.registerSerialization(conf, Metadata.class); if (isLocal) { LocalCluster cluster = new LocalCluster(); cluster.submitTopology(name, conf, builder.createTopology()); if (ttl != -1) { Utils.sleep(ttl * 1000); cluster.shutdown(); } } else { try { StormSubmitter.submitTopology(name, conf, builder.createTopology()); } catch (Exception e) { e.printStackTrace(); return -1; } } return 0; }
Example #14
Source File: ParserBoltTest.java From logparser with Apache License 2.0 | 6 votes |
@Test public void runRest() throws InterruptedException, NoSuchMethodException { TopologyBuilder builder = new TopologyBuilder(); // ---------- builder.setSpout("Spout", new TestApacheLogsSpout()); // ---------- HttpdLoglineParserBolt parserBolt = new HttpdLoglineParserBolt(TestCase.getLogFormat(), INPUT_FIELD_NAME, OUTPUT_FIELD_NAME); builder.setBolt("Parser", parserBolt, 1).shuffleGrouping("Spout"); // ---------- builder.setBolt("Printer", new ValidateOutput(), 1).shuffleGrouping("Parser"); // ---------- StormTopology topology = builder.createTopology(); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("Unit test", new HashMap<String, String>(), topology); Thread.sleep(10000L); // Run for 10 seconds cluster.killTopology("Unit test"); cluster.shutdown(); }
Example #15
Source File: NiFiStormTopology.java From nifi with Apache License 2.0 | 5 votes |
public static void main( String[] args ) { // Build a Site-To-Site client config for pulling data final SiteToSiteClientConfig inputConfig = new SiteToSiteClient.Builder() .url("http://localhost:8080/nifi") .portName("Data for Storm") .buildConfig(); // Build a Site-To-Site client config for pushing data final SiteToSiteClientConfig outputConfig = new SiteToSiteClient.Builder() .url("http://localhost:8080/nifi") .portName("Data from Storm") .buildConfig(); final int tickFrequencySeconds = 5; final NiFiDataPacketBuilder niFiDataPacketBuilder = new SimpleNiFiDataPacketBuilder(); final NiFiBolt niFiBolt = new NiFiBolt(outputConfig, niFiDataPacketBuilder, tickFrequencySeconds) //.withBatchSize(1) ; TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("nifiInput", new NiFiSpout(inputConfig)); builder.setBolt("nifiOutput", niFiBolt).shuffleGrouping("nifiInput"); // Submit the topology running in local mode Config conf = new Config(); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("test", conf, builder.createTopology()); Utils.sleep(90000); cluster.shutdown(); }
Example #16
Source File: ThresholdingEngine.java From monasca-thresh with Apache License 2.0 | 5 votes |
protected void run() throws Exception { Config config = Injector.getInstance(Config.class); StormTopology topology = Injector.getInstance(StormTopology.class); config.registerSerialization(monasca.thresh.domain.model.SubAlarm.class); config.registerSerialization(monasca.thresh.domain.model.SubExpression.class); if (local) { logger.info("submitting topology {} to local storm cluster", topologyName); new LocalCluster().submitTopology(topologyName, config, topology); } else { logger.info("submitting topology {} to non-local storm cluster", topologyName); StormSubmitter.submitTopology(topologyName, config, topology); } }
Example #17
Source File: ParserTopologyComponent.java From metron with Apache License 2.0 | 5 votes |
@Override public void start() throws UnableToStartException { try { final Map<String, Object> stormConf = new HashMap<>(); stormConf.put(Config.TOPOLOGY_DEBUG, true); ParserTopologyBuilder.ParserTopology topologyBuilder = ParserTopologyBuilder.build ( topologyProperties.getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY), Optional.ofNullable(brokerUrl), sensorTypes, (x,y) -> Collections.nCopies(sensorTypes.size(), 1), (x,y) -> Collections.nCopies(sensorTypes.size(), 1), (x,y) -> 1, (x,y) -> 1, (x,y) -> 1, (x,y) -> 1, (x,y) -> Collections.nCopies(sensorTypes.size(), new HashMap<>()), (x,y) -> null, (x,y) -> outputTopic, (x,y) -> errorTopic, (x,y) -> { Config c = new Config(); c.putAll(stormConf); return c; } ); stormCluster = new LocalCluster(); stormCluster.submitTopology(getTopologyName(), stormConf, topologyBuilder.getBuilder().createTopology()); } catch (Exception e) { throw new UnableToStartException("Unable to start parser topology for sensorTypes: " + sensorTypes, e); } }
Example #18
Source File: ParserTopologyCLI.java From metron with Apache License 2.0 | 5 votes |
public static void main(String[] args) { try { Options options = new Options(); final CommandLine cmd = parse(options, args); if (cmd.hasOption("h")) { final HelpFormatter usageFormatter = new HelpFormatter(); usageFormatter.printHelp("ParserTopologyCLI", null, options, null, true); System.exit(0); } ParserTopologyCLI cli = new ParserTopologyCLI(); ParserTopologyBuilder.ParserTopology topology = cli.createParserTopology(cmd); String sensorTypes = ParserOptions.SENSOR_TYPES.get(cmd); String topologyName = sensorTypes.replaceAll(TOPOLOGY_OPTION_SEPARATOR, STORM_JOB_SEPARATOR); if (ParserOptions.TEST.has(cmd)) { topology.getTopologyConfig().put(Config.TOPOLOGY_DEBUG, true); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(topologyName, topology.getTopologyConfig(), topology.getBuilder().createTopology()); Utils.sleep(300000); cluster.shutdown(); } else { StormSubmitter.submitTopology(topologyName, topology.getTopologyConfig(), topology.getBuilder().createTopology()); } } catch (Exception e) { e.printStackTrace(); System.exit(-1); } }
Example #19
Source File: SplitJoinTopologyTest.java From streamline with Apache License 2.0 | 5 votes |
protected void submitTopology() throws Exception { final Config config = getConfig(); final String topologyName = "SplitJoinTopologyTest"; final StormTopology topology = createTopology(); log.info("Created topology with name: [{}] and topology: [{}]", topologyName, topology); ILocalCluster localCluster = new LocalCluster(); log.info("Submitting topology: [{}]", topologyName); localCluster.submitTopology(topologyName, config, topology); }
Example #20
Source File: NormalizationTopologyTest.java From streamline with Apache License 2.0 | 5 votes |
public void testNormalizationTopology(NormalizationProcessor normalizationProcessor) throws Exception { final Config config = new Config(); config.setDebug(true); final String topologyName = "SplitJoinTopologyTest"; final StormTopology topology = createTopology(normalizationProcessor); log.info("Created topology with name: [{}] and topology: [{}]", topologyName, topology); ILocalCluster localCluster = new LocalCluster(); log.info("Submitting topology: [{}]", topologyName); localCluster.submitTopology(topologyName, config, topology); Thread.sleep(2000); localCluster.shutdown(); }
Example #21
Source File: AbstractStormSuite.java From elasticsearch-hadoop with Apache License 2.0 | 5 votes |
@Override protected void before() throws Throwable { copyPropertiesIntoCfg(cfg); String stormMode = TestSettings.TESTING_PROPS.getProperty("storm", "local"); isLocal = "local".equals(stormMode); //cfg.setDebug(true); cfg.setNumWorkers(Integer.parseInt(TestSettings.TESTING_PROPS.getProperty("storm.numworkers", "2"))); //cfg.setMaxTaskParallelism(2); cfg.put(Config.TOPOLOGY_MIN_REPLICATION_COUNT, 0); cfg.put(Config.TOPOLOGY_MAX_REPLICATION_WAIT_TIME_SEC, 0); stormCluster = new LocalCluster(); }
Example #22
Source File: ExclamationTopology.java From incubator-heron with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { TopologyBuilder builder = new TopologyBuilder(); int parallelism = 2; int spouts = parallelism; builder.setSpout("word", new TestWordSpout(Duration.ofMillis(50)), spouts); int bolts = 2 * parallelism; builder.setBolt("exclaim1", new ExclamationBolt(), bolts) .shuffleGrouping("word"); Config conf = new Config(); conf.setDebug(true); conf.setMaxSpoutPending(10); conf.setMessageTimeoutSecs(600); conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError"); if (args != null && args.length > 0) { conf.setNumWorkers(parallelism); StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); } else { System.out.println("Topology name not provided as an argument, running in simulator mode."); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("test", conf, builder.createTopology()); Utils.sleep(10000); cluster.killTopology("test"); cluster.shutdown(); } }
Example #23
Source File: StormKafkaProcess.java From BigData with GNU General Public License v3.0 | 5 votes |
public static void main(String[] args) throws InterruptedException, InvalidTopologyException, AuthorizationException, AlreadyAliveException { String topologyName = "TSAS";// 元组名 // Zookeeper主机地址,会自动选取其中一个 ZkHosts zkHosts = new ZkHosts("192.168.230.128:2181,192.168.230.129:2181,192.168.230.131:2181"); String topic = "trademx"; String zkRoot = "/storm";// storm在Zookeeper上的根路径 String id = "tsaPro"; // 创建SpoutConfig对象 SpoutConfig spontConfig = new SpoutConfig(zkHosts, topic, zkRoot, id); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("kafka", new KafkaSpout(spontConfig), 2); builder.setBolt("AccBolt", new AccBolt()).shuffleGrouping("kafka"); builder.setBolt("ToDbBolt", new ToDbBolt()).shuffleGrouping("AccBolt"); Config config = new Config(); config.setDebug(false); if (args.length == 0) { // 本地运行,用于测试 LocalCluster localCluster = new LocalCluster(); localCluster.submitTopology(topologyName, config, builder.createTopology()); Thread.sleep(1000 * 3600); localCluster.killTopology(topologyName); localCluster.shutdown(); } else { // 提交至集群运行 StormSubmitter.submitTopology(topologyName, config, builder.createTopology()); } }
Example #24
Source File: AppMain.java From storm_spring_boot_demo with MIT License | 5 votes |
/** * 用于debug * @param name * @param builder * @throws InterruptedException */ private static void localSubmit(String name,TopologyBuilder builder, Config conf) throws InterruptedException { conf.setDebug(true); conf.setMaxTaskParallelism(3); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(name, conf, builder.createTopology()); Thread.sleep(100000); cluster.shutdown(); }
Example #25
Source File: KafkaStormWordCountTopology.java From Building-Data-Streaming-Applications-with-Apache-Kafka with MIT License | 5 votes |
public static void main(String[] args) throws Exception { String zkConnString = "localhost:2181"; String topic = "words"; BrokerHosts hosts = new ZkHosts(zkConnString); SpoutConfig kafkaSpoutConfig = new SpoutConfig(hosts, topic, "/" + topic, "wordcountID"); kafkaSpoutConfig.startOffsetTime = kafka.api.OffsetRequest.EarliestTime(); kafkaSpoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme()); TopologyBuilder topologyBuilder = new TopologyBuilder(); topologyBuilder.setSpout("kafkaspout", new KafkaSpout(kafkaSpoutConfig)); topologyBuilder.setBolt("stringsplit", new StringToWordsSpliterBolt()).shuffleGrouping("kafkaspout"); topologyBuilder.setBolt("counter", new WordCountCalculatorBolt()).shuffleGrouping("stringsplit"); Config config = new Config(); config.setDebug(true); if (args != null && args.length > 1) { config.setNumWorkers(3); StormSubmitter.submitTopology(args[1], config, topologyBuilder.createTopology()); } else { // Cap the maximum number of executors that can be spawned // for a component to 3 config.setMaxTaskParallelism(3); // LocalCluster is used to run locally LocalCluster cluster = new LocalCluster(); cluster.submitTopology("KafkaLocal", config, topologyBuilder.createTopology()); // sleep try { Thread.sleep(10000); } catch (InterruptedException e) { // TODO Auto-generated catch block cluster.killTopology("KafkaToplogy"); cluster.shutdown(); } cluster.shutdown(); } }
Example #26
Source File: DBusRouterTopology.java From DBus with Apache License 2.0 | 5 votes |
private void start(StormTopology topology, boolean runAsLocal) throws Exception { Config conf = new Config(); conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zkConnect); conf.put(Constants.TOPOLOGY_ID, topologyId); conf.put(Constants.TOPOLOGY_ALIAS, alias); conf.put(Constants.ROUTER_PROJECT_NAME, projectName); String workerChildOpts = routerConf.getProperty(DBusRouterConstants.STORM_TOPOLOGY_WORKER_CHILDOPTS, "-Xmx2g"); conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, workerChildOpts); int msgTimeout = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_MESSAGE_TIMEOUT, "10")); conf.setMessageTimeoutSecs(msgTimeout); int maxSpoutPending = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_MAX_SPOUT_PENDING, "100")); conf.setMaxSpoutPending(maxSpoutPending); conf.setDebug(true); int numWorks = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_NUM_WORKS, "1")); conf.setNumWorkers(numWorks); if (runAsLocal) { conf.setMaxTaskParallelism(10); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(topologyName, conf, topology); } else { StormSubmitter.submitTopology(topologyName, conf, topology); } }
Example #27
Source File: NiFiStormTopology.java From localization_nifi with Apache License 2.0 | 5 votes |
public static void main( String[] args ) { // Build a Site-To-Site client config for pulling data final SiteToSiteClientConfig inputConfig = new SiteToSiteClient.Builder() .url("http://localhost:8080/nifi") .portName("Data for Storm") .buildConfig(); // Build a Site-To-Site client config for pushing data final SiteToSiteClientConfig outputConfig = new SiteToSiteClient.Builder() .url("http://localhost:8080/nifi") .portName("Data from Storm") .buildConfig(); final int tickFrequencySeconds = 5; final NiFiDataPacketBuilder niFiDataPacketBuilder = new SimpleNiFiDataPacketBuilder(); final NiFiBolt niFiBolt = new NiFiBolt(outputConfig, niFiDataPacketBuilder, tickFrequencySeconds) //.withBatchSize(1) ; TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("nifiInput", new NiFiSpout(inputConfig)); builder.setBolt("nifiOutput", niFiBolt).shuffleGrouping("nifiInput"); // Submit the topology running in local mode Config conf = new Config(); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("test", conf, builder.createTopology()); Utils.sleep(90000); cluster.shutdown(); }
Example #28
Source File: App.java From springBoot-study with Apache License 2.0 | 5 votes |
public static void main(String[] args) { //定义一个拓扑 TopologyBuilder builder=new TopologyBuilder(); //设置1个Executeor(线程),默认一个 builder.setSpout(test_spout, new TestSpout(),1); //shuffleGrouping:表示是随机分组 //设置1个Executeor(线程),和两个task builder.setBolt(test_bolt, new TestBolt(),1).setNumTasks(1).shuffleGrouping(test_spout); //fieldsGrouping:表示是按字段分组 //设置1个Executeor(线程),和1个task builder.setBolt(test2_bolt, new Test2Bolt(),1).setNumTasks(1).fieldsGrouping(test_bolt, new Fields("count")); Config conf = new Config(); conf.put("test", "test"); try{ //运行拓扑 if(args !=null&&args.length>0){ //有参数时,表示向集群提交作业,并把第一个参数当做topology名称 System.out.println("运行远程模式"); StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); } else{//没有参数时,本地提交 //启动本地模式 System.out.println("运行本地模式"); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("Word-counts" ,conf, builder.createTopology() ); Thread.sleep(20000); // //关闭本地集群 cluster.shutdown(); } }catch (Exception e){ e.printStackTrace(); } }
Example #29
Source File: App.java From springBoot-study with Apache License 2.0 | 5 votes |
public static void main(String[] args) { // TODO Auto-generated method stub //定义一个拓扑 TopologyBuilder builder=new TopologyBuilder(); builder.setSpout(str1, new TestSpout()); builder.setBolt(str2, new TestBolt()).shuffleGrouping(str1); Config conf = new Config(); conf.put("test", "test"); try{ //运行拓扑 if(args !=null&&args.length>0){ //有参数时,表示向集群提交作业,并把第一个参数当做topology名称 System.out.println("远程模式"); StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); } else{//没有参数时,本地提交 //启动本地模式 System.out.println("本地模式"); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("111" ,conf, builder.createTopology() ); // Thread.sleep(2000); // //关闭本地集群 // cluster.shutdown(); } }catch (Exception e){ e.printStackTrace(); } }
Example #30
Source File: TopologyApp.java From springBoot-study with Apache License 2.0 | 5 votes |
public void runStorm(String[] args) { // 定义一个拓扑 TopologyBuilder builder = new TopologyBuilder(); // 设置1个Executeor(线程),默认一个 builder.setSpout(Constants.KAFKA_SPOUT, new KafkaInsertDataSpout(), 1); // shuffleGrouping:表示是随机分组 // 设置1个Executeor(线程),和两个task builder.setBolt(Constants.INSERT_BOLT, new InsertBolt(), 1).setNumTasks(1).shuffleGrouping(Constants.KAFKA_SPOUT); Config conf = new Config(); //设置一个应答者 conf.setNumAckers(1); //设置一个work conf.setNumWorkers(3); try { // 有参数时,表示向集群提交作业,并把第一个参数当做topology名称 // 没有参数时,本地提交 if (args != null && args.length > 0) { logger.info("运行远程模式"); StormSubmitter.submitTopology(args[0], conf, builder.createTopology()); } else { // 启动本地模式 logger.info("运行本地模式"); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("TopologyApp", conf, builder.createTopology()); } } catch (Exception e) { logger.error("storm启动失败!程序退出!",e); System.exit(1); } logger.info("storm启动成功..."); }