Java Code Examples for org.apache.storm.LocalCluster

The following examples show how to use org.apache.storm.LocalCluster. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: storm-nginx-log   Source File: NginxStorm.java    License: MIT License 6 votes vote down vote up
public static void main(String[] argv) throws InterruptedException {

        Config config = new Config();
        config.setDebug(true);

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("LogSpout", new LogSpout(), 1);
        builder.setBolt("SpliteBolt", new SpliteBolt(), 1).shuffleGrouping("LogSpout");
        builder.setBolt("CounterBolt", new CounterBolt(), 1)
                .fieldsGrouping("SpliteBolt", new Fields("item"));

        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("NginxLog", config, builder.createTopology());
//        Thread.sleep(10000);
//
//        cluster.killTopology("NginxLog");
//        cluster.shutdown();
    }
 
Example 2
Source Project: storm-statistic   Source File: StatisticTopology.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    /**
     * 设置spout和bolt的dag(有向无环图)
     */
    KafkaSpout kafkaSpout = createKafkaSpout();
    builder.setSpout("id_kafka_spout", kafkaSpout);
    builder.setBolt("id_convertIp_bolt", new ConvertIPBolt()).shuffleGrouping("id_kafka_spout"); // 通过不同的数据流转方式,来指定数据的上游组件
    builder.setBolt("id_statistic_bolt", new StatisticBolt()).shuffleGrouping("id_convertIp_bolt"); // 通过不同的数据流转方式,来指定数据的上游组件
    // 使用builder构建topology
    StormTopology topology = builder.createTopology();
    String topologyName = KafkaStormTopology.class.getSimpleName();  // 拓扑的名称
    Config config = new Config();   // Config()对象继承自HashMap,但本身封装了一些基本的配置

    // 启动topology,本地启动使用LocalCluster,集群启动使用StormSubmitter
    if (args == null || args.length < 1) {  // 没有参数时使用本地模式,有参数时使用集群模式
        LocalCluster localCluster = new LocalCluster(); // 本地开发模式,创建的对象为LocalCluster
        localCluster.submitTopology(topologyName, config, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, config, topology);
    }
}
 
Example 3
public static void main(String[] args) {
    //根据Spout和Bolt构建TopologyBuilder
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("DataSourceSpout", new DataSourceSpout());
    builder.setBolt("SplitBolt", new SplitBolt()).shuffleGrouping("DataSourceSpout");
    builder.setBolt("CountBolt", new CountBolt()).shuffleGrouping("SplitBolt");

    JedisPoolConfig poolConfig = new JedisPoolConfig.Builder()
            .setHost("192.168.60.11").setPort(6379).build();
    RedisStoreMapper storeMapper = new WordCountStoreMapper();
    RedisStoreBolt storeBolt = new RedisStoreBolt(poolConfig, storeMapper);

    builder.setBolt("RedisStoreBolt", storeBolt).shuffleGrouping("CountBolt");

    //创建本地集群
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("LocalWordCountRedisStormTopology", new Config(), builder.createTopology());

}
 
Example 4
public static void main(String[] args) throws Exception {
    LocalCluster cluster = new LocalCluster();
    TopologyBuilder builder = new TopologyBuilder();

    Config conf = new Config();
    conf.setDebug(false);
    conf.setNumWorkers(1);
    conf.setMaxTaskParallelism(1);
    //Disable Acking
    conf.setNumAckers(0);

    KeyedScottyWindowOperator scottyBolt = new KeyedScottyWindowOperator<Integer, Integer>(new Sum(), 0);
    scottyBolt.addWindow(new TumblingWindow(WindowMeasure.Time, 1000));
    scottyBolt.addWindow(new SlidingWindow(WindowMeasure.Time, 1000, 250));
    scottyBolt.addWindow(new SessionWindow(WindowMeasure.Time, 1000));

    builder.setSpout("spout", new DataGeneratorSpout());
    builder.setBolt("scottyWindow", scottyBolt).fieldsGrouping("spout", new Fields("key"));
    builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("scottyWindow");

    cluster.submitTopology("testTopology", conf, builder.createTopology());
    //cluster.killTopology("testTopology");
    //cluster.shutdown();
}
 
Example 5
Source Project: DBus   Source File: FullPullerTopology.java    License: Apache License 2.0 6 votes vote down vote up
private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(FullPullConstants.FULL_SPLITTER_TOPOLOGY_ID, fullSplitterTopologyId);
    conf.put(FullPullConstants.FULL_PULLER_TOPOLOGY_ID, fullPullerTopologyId);
    conf.put(FullPullConstants.DS_NAME, topologyId);
    conf.put(FullPullConstants.ZKCONNECT, zkConnect);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, commonConfSplit.getProperty(FullPullConstants.TOPOLOGY_WORKER_CHILDOPTS));
    //设置message超时时间为,保证每个分片都能在该内拉完数据
    conf.setMessageTimeoutSecs(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_MESSAGE_TIMEOUT)));
    conf.setMaxSpoutPending(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_MAX_SPOUT_PENDING)));
    conf.setNumWorkers(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_NUM_WORKERS)));
    conf.setDebug(true);

    if (runAsLocal) {
        conf.setMaxTaskParallelism(3);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}
 
Example 6
Source Project: DBus   Source File: DBusLogProcessorTopology.java    License: Apache License 2.0 6 votes vote down vote up
private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zkConnect);
    conf.put(Constants.TOPOLOGY_ID, topologyId);
    conf.setMessageTimeoutSecs(Integer.parseInt(properties.getProperty(Constants.LOG_MESSAGE_TIMEOUT)));
    //conf.setMaxSpoutPending(30);
    conf.setDebug(true);
    conf.setNumWorkers(Integer.parseInt(properties.getProperty(Constants.LOG_NUMWORKERS)));

    if (runAsLocal) {
        conf.setMaxTaskParallelism(10);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}
 
Example 7
Source Project: DBus   Source File: SinkTopology.java    License: Apache License 2.0 6 votes vote down vote up
private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(Constants.ZOOKEEPER_SERVERS, zkConnect);
    conf.put(Constants.TOPOLOGY_ID, topologyId);
    conf.put(Constants.SINK_TYPE, sinkType);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, sinkerConf.getProperty(SinkerConstants.TOPOLOGY_WORKER_CHILDOPTS));

    conf.setMessageTimeoutSecs(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_MESSAGE_TIMEOUT)));
    conf.setMaxSpoutPending(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_MAX_SPOUT_PENDING)));
    conf.setDebug(true);
    conf.setNumWorkers(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_NUM_WORKERS)));
    if (runAsLocal) {
        conf.setMaxTaskParallelism(10);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}
 
Example 8
public static void main(String[] args) throws Exception {
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("word0", new TestWordSpout(), 2);
  builder.setSpout("word1", new TestWordSpout(), 2);
  builder.setSpout("word2", new TestWordSpout(), 2);
  builder.setBolt("exclaim1", new ExclamationBolt(), 2)
      .shuffleGrouping("word0")
      .shuffleGrouping("word1")
      .shuffleGrouping("word2");

  Config conf = new Config();
  conf.setDebug(true);
  conf.setMaxSpoutPending(10);
  conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
  if (args != null && args.length > 0) {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
  } else {
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    cluster.killTopology("test");
    cluster.shutdown();
  }
}
 
Example 9
Source Project: ranger   Source File: StormRangerAuthorizerTest.java    License: Apache License 2.0 6 votes vote down vote up
@org.junit.BeforeClass
public static void setup() throws Exception {
    cluster = new LocalCluster();

    final Config conf = new Config();
    conf.setDebug(true);

    final TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("words", new WordSpout());
    builder.setBolt("counter", new WordCounterBolt()).shuffleGrouping("words");

    // bob can create a new topology
    final Subject subject = new Subject();
    subject.getPrincipals().add(new SimplePrincipal("bob"));
    Subject.doAs(subject, new PrivilegedExceptionAction<Void>() {
        public Void run() throws Exception {
            cluster.submitTopology("word-count", conf, builder.createTopology());
            return null;
        }
    });

}
 
Example 10
public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException, AuthorizationException {
   TopologyBuilder builder = new TopologyBuilder();
   builder.setSpout(DATA_GENERATOR, new ASpout());
   builder.setBolt(DATA_CALCULATOR, new ABolt()).shuffleGrouping(DATA_GENERATOR);
   builder.setBolt(DATA_PRINTER, new DataPrinter()).shuffleGrouping(DATA_CALCULATOR).shuffleGrouping(DATA_GENERATOR);

   Config config = new Config();

   LocalCluster cluster = new LocalCluster();
   cluster.submitTopology(TOPOLOGY_NAME, config,
         builder.createTopology());

   Thread.sleep(100000);
   cluster.killTopology(TOPOLOGY_NAME);
   cluster.shutdown();
}
 
Example 11
Source Project: tutorials   Source File: TopologyRunner.java    License: MIT License 6 votes vote down vote up
public static void runTopology() {
    String filePath = "./src/main/resources/operations.txt";
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("randomNumberSpout", new RandomNumberSpout());
    builder.setBolt("filteringBolt", new FilteringBolt()).shuffleGrouping("randomNumberSpout");
    builder.setBolt("aggregatingBolt", new AggregatingBolt()
      .withTimestampField("timestamp")
      .withLag(BaseWindowedBolt.Duration.seconds(1))
      .withWindow(BaseWindowedBolt.Duration.seconds(5))).shuffleGrouping("filteringBolt");
    builder.setBolt("fileBolt", new FileWritingBolt(filePath)).shuffleGrouping("aggregatingBolt");

    Config config = new Config();
    config.setDebug(false);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("Test", config, builder.createTopology());
}
 
Example 12
Source Project: storm-crawler   Source File: ConfigurableTopology.java    License: Apache License 2.0 6 votes vote down vote up
/** Submits the topology under a specific name **/
protected int submit(String name, Config conf, TopologyBuilder builder) {

    // register Metadata for serialization with FieldsSerializer
    Config.registerSerialization(conf, Metadata.class);

    if (isLocal) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(name, conf, builder.createTopology());
        if (ttl != -1) {
            Utils.sleep(ttl * 1000);
            cluster.shutdown();
        }
    }

    else {
        try {
            StormSubmitter.submitTopology(name, conf,
                    builder.createTopology());
        } catch (Exception e) {
            e.printStackTrace();
            return -1;
        }
    }
    return 0;
}
 
Example 13
Source Project: logparser   Source File: ParserBoltTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void runRest() throws InterruptedException, NoSuchMethodException {
    TopologyBuilder builder = new TopologyBuilder();

    // ----------
    builder.setSpout("Spout", new TestApacheLogsSpout());
    // ----------
    HttpdLoglineParserBolt parserBolt = new HttpdLoglineParserBolt(TestCase.getLogFormat(), INPUT_FIELD_NAME, OUTPUT_FIELD_NAME);

    builder.setBolt("Parser", parserBolt, 1).shuffleGrouping("Spout");
    // ----------
    builder.setBolt("Printer", new ValidateOutput(), 1).shuffleGrouping("Parser");
    // ----------

    StormTopology topology = builder.createTopology();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("Unit test", new HashMap<String, String>(), topology);
    Thread.sleep(10000L); // Run for 10 seconds
    cluster.killTopology("Unit test");
    cluster.shutdown();

}
 
Example 14
Source Project: breeze   Source File: TopologyStarter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
	properties.put(Config.TOPOLOGY_NAME, ID);
	Config config = stormConfig(properties);
	ApplicationContext spring = SingletonApplicationContext.loadXml(config, MAIN_CONTEXT);
	try {
		StormTopology topology = spring.getBean(ID, StormTopology.class);

		Properties systemProperties = System.getProperties();
		if (systemProperties.containsKey(LOCAL_RUN_PARAM)) {
			String timeout = systemProperties.getProperty(LOCAL_RUN_PARAM);
			if (! hasText(timeout))
				timeout = LOCAL_RUN_DEFAULT_TIMEOUT;
			long ms = 1000L * Integer.parseInt(timeout);

			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology(ID, config, topology);
			sleep(ms);
			cluster.shutdown();
		} else
			StormSubmitter.submitTopology(ID, config, topology);
	} catch (Exception e) {
		e.printStackTrace();
		exit(255);
	}
}
 
Example 15
Source Project: localization_nifi   Source File: NiFiStormTopology.java    License: Apache License 2.0 5 votes vote down vote up
public static void main( String[] args ) {
    // Build a Site-To-Site client config for pulling data
    final SiteToSiteClientConfig inputConfig = new SiteToSiteClient.Builder()
            .url("http://localhost:8080/nifi")
            .portName("Data for Storm")
            .buildConfig();

    // Build a Site-To-Site client config for pushing data
    final SiteToSiteClientConfig outputConfig = new SiteToSiteClient.Builder()
            .url("http://localhost:8080/nifi")
            .portName("Data from Storm")
            .buildConfig();

    final int tickFrequencySeconds = 5;
    final NiFiDataPacketBuilder niFiDataPacketBuilder = new SimpleNiFiDataPacketBuilder();
    final NiFiBolt niFiBolt = new NiFiBolt(outputConfig, niFiDataPacketBuilder, tickFrequencySeconds)
            //.withBatchSize(1)
            ;

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("nifiInput", new NiFiSpout(inputConfig));
    builder.setBolt("nifiOutput", niFiBolt).shuffleGrouping("nifiInput");

    // Submit the topology running in local mode
    Config conf = new Config();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());

    Utils.sleep(90000);
    cluster.shutdown();
}
 
Example 16
Source Project: springBoot-study   Source File: App.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args)  {
		//定义一个拓扑
		TopologyBuilder builder=new TopologyBuilder();
		//设置1个Executeor(线程),默认一个
		builder.setSpout(test_spout, new TestSpout(),1);
		//shuffleGrouping:表示是随机分组
		//设置1个Executeor(线程),和两个task
		builder.setBolt(test_bolt, new TestBolt(),1).setNumTasks(1).shuffleGrouping(test_spout);
		//fieldsGrouping:表示是按字段分组
		//设置1个Executeor(线程),和1个task
		builder.setBolt(test2_bolt, new Test2Bolt(),1).setNumTasks(1).fieldsGrouping(test_bolt, new Fields("count"));
		Config conf = new Config();
		conf.put("test", "test");
		try{
		  //运行拓扑
	       if(args !=null&&args.length>0){ //有参数时,表示向集群提交作业,并把第一个参数当做topology名称
	       	 System.out.println("运行远程模式");
			 StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
	      } else{//没有参数时,本地提交
	        //启动本地模式
	     	System.out.println("运行本地模式");
	        LocalCluster cluster = new LocalCluster();
	        cluster.submitTopology("Word-counts" ,conf,  builder.createTopology() );
	        Thread.sleep(20000);
//	        //关闭本地集群
	        cluster.shutdown();
	      }
		}catch (Exception e){
			e.printStackTrace();
		}
	}
 
Example 17
Source Project: springBoot-study   Source File: App.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args)  {
		// TODO Auto-generated method stub
		//定义一个拓扑
		TopologyBuilder builder=new TopologyBuilder();
		builder.setSpout(str1, new TestSpout());
		builder.setBolt(str2, new TestBolt()).shuffleGrouping(str1);
		Config conf = new Config();
		conf.put("test", "test");
		try{
		  //运行拓扑
	       if(args !=null&&args.length>0){ //有参数时,表示向集群提交作业,并把第一个参数当做topology名称
	       	 System.out.println("远程模式");
			 StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
	      } else{//没有参数时,本地提交
	        //启动本地模式
	     	System.out.println("本地模式");
	        LocalCluster cluster = new LocalCluster();
	        cluster.submitTopology("111" ,conf,  builder.createTopology() );
//	        Thread.sleep(2000);
//	        //关闭本地集群
//	        cluster.shutdown();
	      }
		}catch (Exception e){
			e.printStackTrace();
		}
		
	}
 
Example 18
Source Project: springBoot-study   Source File: TopologyApp.java    License: Apache License 2.0 5 votes vote down vote up
public  void runStorm(String[] args) {
	// 定义一个拓扑
	TopologyBuilder builder = new TopologyBuilder();
	// 设置1个Executeor(线程),默认一个
	builder.setSpout(Constants.KAFKA_SPOUT, new KafkaInsertDataSpout(), 1);
	// shuffleGrouping:表示是随机分组
	// 设置1个Executeor(线程),和两个task
	builder.setBolt(Constants.INSERT_BOLT, new InsertBolt(), 1).setNumTasks(1).shuffleGrouping(Constants.KAFKA_SPOUT);
	Config conf = new Config();
	//设置一个应答者
	conf.setNumAckers(1);
	//设置一个work
	conf.setNumWorkers(3);
	try {
		// 有参数时,表示向集群提交作业,并把第一个参数当做topology名称
		// 没有参数时,本地提交
		if (args != null && args.length > 0) { 
			logger.info("运行远程模式");
			StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
		} else {
			// 启动本地模式
			logger.info("运行本地模式");
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology("TopologyApp", conf, builder.createTopology());
		}
	} catch (Exception e) {
		logger.error("storm启动失败!程序退出!",e);
		System.exit(1);
	}

	logger.info("storm启动成功...");

}
 
Example 19
public static void main(String[] args) {
    //根据Spout和Bolt构建TopologyBuilder
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("DataSourceSpout", new DataSourceSpout());
    builder.setBolt("SplitBolt", new SplitBolt()).shuffleGrouping("DataSourceSpout");
    builder.setBolt("CountBolt", new CountBolt()).shuffleGrouping("SplitBolt");

    Map hikariConfigMap = Maps.newHashMap();
    hikariConfigMap.put("dataSourceClassName","com.mysql.jdbc.jdbc2.optional.MysqlDataSource");
    hikariConfigMap.put("dataSource.url", "jdbc:mysql://192.168.60.11/storm");
    hikariConfigMap.put("dataSource.user","root");
    hikariConfigMap.put("dataSource.password","123");
    ConnectionProvider connectionProvider = new HikariCPConnectionProvider(hikariConfigMap);

    String tableName = "wc";
    JdbcMapper simpleJdbcMapper = new SimpleJdbcMapper(tableName, connectionProvider);

    JdbcInsertBolt userPersistanceBolt = new JdbcInsertBolt(connectionProvider, simpleJdbcMapper)
            .withTableName(tableName)
            .withQueryTimeoutSecs(30);
    builder.setBolt("JdbcInsertBolt", userPersistanceBolt).shuffleGrouping("CountBolt");

    //创建本地集群
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("LocalWordCountRedisStormTopology", new Config(), builder.createTopology());

}
 
Example 20
public static void main(String[] args) {
    LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("addUser");
    builder.addBolt(new MyBolt());

    LocalCluster localCluster = new LocalCluster();
    LocalDRPC drpc = new LocalDRPC();
    localCluster.submitTopology("local-drpc", new Config(), builder.createLocalTopology(drpc));

    String result = drpc.execute("addUser", "zhangsan");
    System.out.println("From client: " + result);

    localCluster.shutdown();
    drpc.shutdown();
}
 
Example 21
public static void main(String[] args) {
    //根据Spout和Bolt构建TopologyBuilder
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("DataSourceSpout", new DataSourceSpout());
    builder.setBolt("SplitBolt", new SplitBolt()).shuffleGrouping("DataSourceSpout");
    builder.setBolt("CountBolt", new CountBolt()).shuffleGrouping("SplitBolt");

    //创建本地集群
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("LocalWordCountStormTopology", new Config(), builder.createTopology());

}
 
Example 22
Source Project: java-study   Source File: WordCountApp.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException {
 	//定义拓扑
     TopologyBuilder builder = new TopologyBuilder();
     builder.setSpout("word-reader" , new WordReader());
     builder.setBolt("word-normalizer" , new WordNormalizer()).shuffleGrouping("word-reader" );
     builder.setBolt("word-counter" , new WordCounter()).fieldsGrouping("word-normalizer" , new Fields("word"));
     StormTopology topology = builder.createTopology();
     //配置
     
     Config conf = new Config();
     String fileName ="words.txt" ;
     conf.put("fileName" , fileName );
     conf.setDebug(false);
 
      //运行拓扑
      System.out.println("开始...");
      if(args !=null&&args.length>0){ //有参数时,表示向集群提交作业,并把第一个参数当做topology名称
     	 System.out.println("远程模式");
          try {
	StormSubmitter.submitTopology(args[0], conf, topology);
} catch (AuthorizationException e) {
	e.printStackTrace();
}
    } else{//没有参数时,本地提交
      //启动本地模式
 	 System.out.println("本地模式");
      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("Getting-Started-Topologie" , conf , topology );
      Thread.sleep(5000);
      //关闭本地集群
      cluster.shutdown();
    }
      System.out.println("结束");
    
 }
 
Example 23
Source Project: java-study   Source File: App.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args)  {
		//定义一个拓扑
		TopologyBuilder builder=new TopologyBuilder();
		//设置两个Executeor(线程),默认一个
		builder.setSpout(test_spout, new TestSpout(),2);
		//shuffleGrouping:表示是随机分组
		//设置两个Executeor(线程),和两个task
		builder.setBolt(test_bolt, new TestBolt(),2).setNumTasks(2).shuffleGrouping(test_spout);
		//fieldsGrouping:表示是按字段分组
		//设置两个Executeor(线程),和两个task
		builder.setBolt(test2_bolt, new Test2Bolt(),2).setNumTasks(2).fieldsGrouping(test_bolt, new Fields("count"));
		Config conf = new Config();
		conf.put("test", "test");
		try{
		  //运行拓扑
	       if(args !=null&&args.length>0){ //有参数时,表示向集群提交作业,并把第一个参数当做topology名称
	       	 System.out.println("运行远程模式");
			 StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
	      } else{//没有参数时,本地提交
	        //启动本地模式
	     	System.out.println("运行本地模式");
	        LocalCluster cluster = new LocalCluster();
	        cluster.submitTopology("Word-counts" ,conf,  builder.createTopology() );
	        Thread.sleep(20000);
//	        //关闭本地集群
	        cluster.shutdown();
	      }
		}catch (Exception e){
			e.printStackTrace();
		}
	}
 
Example 24
Source Project: java-study   Source File: WordCountApp.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException {
 	//定义拓扑
     TopologyBuilder builder = new TopologyBuilder();
     builder.setSpout("word-reader" , new WordReader());
     builder.setBolt("word-normalizer" , new WordNormalizer()).shuffleGrouping("word-reader" );
     builder.setBolt("word-counter" , new WordCounter()).fieldsGrouping("word-normalizer" , new Fields("word"));
     StormTopology topology = builder.createTopology();
     //配置
     
     Config conf = new Config();
     String fileName ="words.txt" ;
     conf.put("fileName" , fileName );
     conf.setDebug(false);
 
      //运行拓扑
      System.out.println("开始...");
      if(args !=null&&args.length>0){ //有参数时,表示向集群提交作业,并把第一个参数当做topology名称
     	 System.out.println("远程模式");
          try {
	StormSubmitter.submitTopology(args[0], conf, topology);
} catch (AuthorizationException e) {
	e.printStackTrace();
}
    } else{//没有参数时,本地提交
      //启动本地模式
 	 System.out.println("本地模式");
      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("Getting-Started-Topologie" , conf , topology );
      Thread.sleep(5000);
      //关闭本地集群
      cluster.shutdown();
    }
      System.out.println("结束");
    
 }
 
Example 25
Source Project: java-study   Source File: App.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args)  {
		// TODO Auto-generated method stub
		//定义一个拓扑
		TopologyBuilder builder=new TopologyBuilder();
		builder.setSpout(str1, new TestSpout());
		builder.setBolt(str2, new TestBolt()).shuffleGrouping(str1);
		Config conf = new Config();
		conf.put("test", "test");
		try{
		  //运行拓扑
	       if(args !=null&&args.length>0){ //有参数时,表示向集群提交作业,并把第一个参数当做topology名称
	       	 System.out.println("远程模式");
			 StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
	      } else{//没有参数时,本地提交
	        //启动本地模式
	     	System.out.println("本地模式");
	        LocalCluster cluster = new LocalCluster();
	        cluster.submitTopology("111" ,conf,  builder.createTopology() );
//	        Thread.sleep(2000);
//	        //关闭本地集群
//	        cluster.shutdown();
	      }
		}catch (Exception e){
			e.printStackTrace();
		}
		
	}
 
Example 26
private static void submitTopology(TopologyBuilder builder) {
    try {
        LocalCluster cluster = new LocalCluster();
        config.put(Config.STORM_CLUSTER_MODE, "local");
        cluster.submitTopology(Configuration.getConfig().getString("topology.name"), config, builder.createTopology());
    } catch (Exception e) {
        LOG.error(e.getMessage(), e.getCause());
    }
}
 
Example 27
Source Project: DBus   Source File: MysqlExtractorTopology.java    License: Apache License 2.0 5 votes vote down vote up
public void buildTopology(String[] args) {
    //TODO
    if (parseCommandArgs(args) != 0) {
        return;
    }
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("CanalClientSpout", new CanalClientSpout(), 1);
    builder.setBolt("KafkaProducerBolt", new KafkaProducerBolt(), 1).shuffleGrouping("CanalClientSpout");

    Config conf = new Config();
    conf.put(Constants.ZOOKEEPER_SERVERS, zkServers);
    conf.put(Constants.EXTRACTOR_TOPOLOGY_ID, extractorTopologyId);
    logger.info(Constants.ZOOKEEPER_SERVERS + "=" + zkServers);
    logger.info(Constants.EXTRACTOR_TOPOLOGY_ID + "=" + extractorTopologyId);
    conf.setNumWorkers(1);
    conf.setMaxSpoutPending(50);
    conf.setMessageTimeoutSecs(120);
    if (!runAsLocal) {
        conf.setDebug(false);
        try {
            //StormSubmitter.submitTopology("extractorTopologyId", conf, builder.createTopology());
            StormSubmitter.submitTopology(extractorTopologyId, conf, builder.createTopology());
        } catch (Exception e) {
            e.printStackTrace();
        }
    } else {
        conf.setDebug(false);
        LocalCluster cluster = new LocalCluster();
        //cluster.submitTopology("extractorTopologyId", conf, builder.createTopology());
        cluster.submitTopology(extractorTopologyId, conf, builder.createTopology());
    }
}
 
Example 28
Source Project: DBus   Source File: DBusRouterTopology.java    License: Apache License 2.0 5 votes vote down vote up
private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zkConnect);
    conf.put(Constants.TOPOLOGY_ID, topologyId);
    conf.put(Constants.TOPOLOGY_ALIAS, alias);
    conf.put(Constants.ROUTER_PROJECT_NAME, projectName);

    String workerChildOpts = routerConf.getProperty(DBusRouterConstants.STORM_TOPOLOGY_WORKER_CHILDOPTS, "-Xmx2g");
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, workerChildOpts);

    int msgTimeout = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_MESSAGE_TIMEOUT, "10"));
    conf.setMessageTimeoutSecs(msgTimeout);

    int maxSpoutPending = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_MAX_SPOUT_PENDING, "100"));
    conf.setMaxSpoutPending(maxSpoutPending);
    conf.setDebug(true);

    int numWorks = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_NUM_WORKS, "1"));
    conf.setNumWorkers(numWorks);

    if (runAsLocal) {
        conf.setMaxTaskParallelism(10);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}
 
Example 29
public static void main(String[] args) throws Exception {

        String zkConnString = "localhost:2181";
        String topic = "words";
        BrokerHosts hosts = new ZkHosts(zkConnString);

        SpoutConfig kafkaSpoutConfig = new SpoutConfig(hosts, topic, "/" + topic,
                "wordcountID");
        kafkaSpoutConfig.startOffsetTime = kafka.api.OffsetRequest.EarliestTime();
        kafkaSpoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());

        TopologyBuilder topologyBuilder = new TopologyBuilder();
        topologyBuilder.setSpout("kafkaspout", new KafkaSpout(kafkaSpoutConfig));
        topologyBuilder.setBolt("stringsplit", new StringToWordsSpliterBolt()).shuffleGrouping("kafkaspout");
        topologyBuilder.setBolt("counter", new WordCountCalculatorBolt()).shuffleGrouping("stringsplit");

        Config config = new Config();
        config.setDebug(true);
        if (args != null && args.length > 1) {
            config.setNumWorkers(3);
            StormSubmitter.submitTopology(args[1], config, topologyBuilder.createTopology());
        } else {
            // Cap the maximum number of executors that can be spawned
            // for a component to 3
            config.setMaxTaskParallelism(3);
            // LocalCluster is used to run locally
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("KafkaLocal", config, topologyBuilder.createTopology());
            // sleep
            try {
                Thread.sleep(10000);
            } catch (InterruptedException e) {
                // TODO Auto-generated catch block
                cluster.killTopology("KafkaToplogy");
                cluster.shutdown();
            }

            cluster.shutdown();
        }
    }
 
Example 30
Source Project: storm_spring_boot_demo   Source File: AppMain.java    License: MIT License 5 votes vote down vote up
/**
 * 用于debug
 * @param name
 * @param builder
 * @throws InterruptedException
 */
private static void localSubmit(String name,TopologyBuilder builder, Config conf)
        throws InterruptedException {
    conf.setDebug(true);
    conf.setMaxTaskParallelism(3);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology(name, conf, builder.createTopology());
    Thread.sleep(100000);
    cluster.shutdown();
}