Java Code Examples for backtype.storm.Config.setMaxTaskParallelism()

The following are Jave code examples for showing how to use setMaxTaskParallelism() of the backtype.storm.Config class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
Example 1
Project: LearnStorm   File: ReachTopology.java   Source Code and License Vote up 7 votes
public static void main(String[] args) throws Exception {
  LinearDRPCTopologyBuilder builder = construct();


  Config conf = new Config();

  if (args == null || args.length == 0) {
    conf.setMaxTaskParallelism(3);
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("reach-drpc", conf, builder.createLocalTopology(drpc));

    String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
    for (String url : urlsToTry) {
      System.out.println("Reach of " + url + ": " + drpc.execute("reach", url));
    }

    cluster.shutdown();
    drpc.shutdown();
  }
  else {
    conf.setNumWorkers(6);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createRemoteTopology());
  }
}
 
Example 2
Project: learningJava   File: TopologyMain.java   Source Code and License Vote up 6 votes
public static void main(String[] args) throws InterruptedException {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word-reader", new WordReader());
    builder.setBolt("word-normalizer", new WordNormalizer()).shuffleGrouping("word-reader");
    builder.setBolt("word-counter", new WordCounter(), 2).fieldsGrouping("word-normalizer", new Fields("word"));

    Config conf = new Config();
    conf.setMaxTaskParallelism(3);
    conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 3);
    conf.setDebug(false);

    //Topology run
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("storm-wordcount", conf, builder.createTopology());
    Thread.sleep(30000);
    cluster.shutdown();
}
 
Example 3
Project: Infrastructure   File: TestTopology.java   Source Code and License Vote up 6 votes
/**
 * Creates a standalone topology.
 * 
 * @param args the topology arguments
 * @throws Exception in case of creation problems
 */
public static void main(String[] args) throws Exception {
    Config config = new Config();
    Naming.setDefaultInitializeAlgorithms(config, defaultInitAlgorithms);
    config.setMessageTimeoutSecs(100);
    PipelineOptions options = new PipelineOptions(args);
    RecordingTopologyBuilder b = new RecordingTopologyBuilder(options);
    createTopology(b);
    b.close(args[0], config);
    
    // main topology: int numWorkers = options.getNumberOfWorkers(2);
    options.toConf(config);
    
    if (args != null && args.length > 0) {
        config.setNumWorkers(2);
        StormSubmitter.submitTopology(args[0], config, b.createTopology());
    } else {
        config.setMaxTaskParallelism(2);
        final LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(PIP_NAME, config, b.createTopology());
    }
}
 
Example 4
Project: rb-bi   File: TopologyFunctionTest.java   Source Code and License Vote up 5 votes
@Test
public void macVendorTest() throws FileNotFoundException {

    File fileFlow = new File(Thread.currentThread().getContextClassLoader().getResource("inputData/flows.json").getPath());
    File checkFlow = new File(Thread.currentThread().getContextClassLoader().getResource("dataCheck/macVendorFlows.json").getPath());

    Scanner flows = new Scanner(fileFlow);
    Scanner checkFlows = new Scanner(checkFlow);

    MacVendorFunction._ouiFilePath = Thread.currentThread().getContextClassLoader().getResource("db/oui-vendors").getPath();

    List<String> fieldsFlow = new ArrayList<String>();

    fieldsFlow.add("flows");
    fieldsFlow.add("macVendor");

    LocalDRPC drpc = new LocalDRPC();

    TridentTopology topology = new TridentTopology();
    topology.newDRPCStream("test", drpc)
            .each(new Fields("args"), new MapperFunction("rb_test"), new Fields("flows"))
            .each(new Fields("flows"), new MacVendorFunction(), new Fields("macVendor"))
            .each(new Fields(fieldsFlow), new MergeMapsFunction(), new Fields("finalMap"))
            .project(new Fields("finalMap"))
            .each(new Fields("finalMap"), new MapToJSONFunction(), new Fields("jsonString"));

    Config conf = new Config();
    conf.put("rbDebug", true);
    conf.setMaxTaskParallelism(1);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("testing-topology", conf, topology.build());

    while (flows.hasNextLine()) {
        String stormFlow = drpc.execute("test", flows.nextLine());
        stormFlow = stormFlow.substring(stormFlow.indexOf("{"), stormFlow.indexOf("}") + 1);
        Assert.assertEquals(checkFlows.nextLine(), stormFlow);
    }
}
 
Example 5
Project: rb-bi   File: TopologyFunctionTest.java   Source Code and License Vote up 5 votes
@Test
public void nonTimestampTest() throws FileNotFoundException {

    File fileFlow = new File(Thread.currentThread().getContextClassLoader().getResource("inputData/nonTimestampFlows.json").getPath());

    Scanner flows = new Scanner(fileFlow);


    List<String> fieldsFlow = new ArrayList<String>();

    fieldsFlow.add("flows");

    LocalDRPC drpc = new LocalDRPC();

    TridentTopology topology = new TridentTopology();
    topology.newDRPCStream("test", drpc)
            .each(new Fields("args"), new MapperFunction("rb_test"), new Fields("flows"))
            .each(new Fields("flows"), new CheckTimestampFunction(), new Fields("finalMap"))
            .project(new Fields("finalMap"))
            .each(new Fields("finalMap"), new MapToJSONFunction(), new Fields("jsonString"));

    Config conf = new Config();
    conf.put("rbDebug", true);
    conf.setMaxTaskParallelism(1);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("testing-topology", conf, topology.build());

    while (flows.hasNextLine()) {
        String stormFlow = drpc.execute("test", flows.nextLine());
        stormFlow = stormFlow.substring(stormFlow.indexOf("{"), stormFlow.indexOf("}") + 1);
        Assert.assertEquals(true, stormFlow.contains("timestamp"));
    }
}
 
Example 6
Project: rb-bi   File: TopologyFunctionTest.java   Source Code and License Vote up 5 votes
@Test
public void analizeHttpUrlTest() throws FileNotFoundException {

    File fileFlow = new File(Thread.currentThread().getContextClassLoader().getResource("inputData/httpFlows.json").getPath());
    File checkFlow = new File(Thread.currentThread().getContextClassLoader().getResource("dataCheck/httpFlows.json").getPath());

    Scanner flows = new Scanner(fileFlow);
    Scanner checkFlows = new Scanner(checkFlow);

    List<String> fieldsFlow = new ArrayList<String>();

    fieldsFlow.add("flows");
    fieldsFlow.add("httpUrlMap");

    LocalDRPC drpc = new LocalDRPC();

    TridentTopology topology = new TridentTopology();
    topology.newDRPCStream("test", drpc)
            .each(new Fields("args"), new MapperFunction("rb_test"), new Fields("flows"))
            .each(new Fields("flows"), new AnalizeHttpUrlFunction(), new Fields("httpUrlMap"))
            .each(new Fields(fieldsFlow), new MergeMapsFunction(), new Fields("finalMap"))
            .project(new Fields("finalMap"))
            .each(new Fields("finalMap"), new MapToJSONFunction(), new Fields("jsonString"));

    Config conf = new Config();
    conf.put("rbDebug", true);
    conf.setMaxTaskParallelism(1);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("testing-topology", conf, topology.build());

    while (flows.hasNextLine()) {
        String stormFlow = drpc.execute("test", flows.nextLine());
        stormFlow = stormFlow.substring(stormFlow.indexOf("{"), stormFlow.indexOf("}") + 1);
        Assert.assertEquals(checkFlows.nextLine(), stormFlow);
    }
}
 
Example 7
Project: storm-kafka-examples   File: CounterTopology.java   Source Code and License Vote up 5 votes
/**
 * @param args
 * http://www.programcreek.com/java-api-examples/index.php?api=storm.kafka.KafkaSpout
 */
public static void main(String[] args) {
	try{
		//设置喷发节点并分配并发数,该并发数将会控制该对象在集群中的线程数(6个)
		String zkhost = "wxb-1:2181,wxb-2:2181,wxb-3:2181";
		String topic = "order";
		String groupId = "id";
		int spoutNum = 3;
		int boltNum = 1;
		ZkHosts zkHosts = new ZkHosts(zkhost);//kafaka所在的zookeeper
		SpoutConfig spoutConfig = new SpoutConfig(zkHosts, topic, "/order", groupId);  // create /order /id
		spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
		KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("spout", kafkaSpout, spoutNum);
		builder.setBolt("check", new CheckOrderBolt(), boltNum).shuffleGrouping("spout");
        builder.setBolt("counter", new CounterBolt(),boltNum).shuffleGrouping("check");

        Config config = new Config();
        config.setDebug(true);
        
        if(args!=null && args.length > 0) {
            config.setNumWorkers(2);
            StormSubmitter.submitTopology(args[0], config, builder.createTopology());
        } else {        
            config.setMaxTaskParallelism(2);

            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("Wordcount-Topology", config, builder.createTopology());

            Thread.sleep(500000);

            cluster.shutdown();
        }
	}catch (Exception e) {
		e.printStackTrace();
	}
}
 
Example 8
Project: miner   File: TopologyMain.java   Source Code and License Vote up 5 votes
public static void main(String[] args) {
	try{
		TopologyBuilder topologyBuilder = new TopologyBuilder();
		topologyBuilder.setSpout("Spout", new EmitMessageSpout(), 1);

		topologyBuilder.setBolt("generate", new ParseLoopBolt(), 1)
				.shuffleGrouping("Spout");



		topologyBuilder.setBolt("Store", new PrintBolt(), 1)
				.shuffleGrouping("generate");
		
		Config config = new Config();
		config.setDebug(false);
		
		if(args != null && args.length>0){
			config.setNumWorkers(4);
			StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology());
		}else{
			config.setMaxTaskParallelism(2);
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology("test", config, topologyBuilder.createTopology());
		}
		
	}catch(Exception e){
		e.printStackTrace();
	}
}
 
Example 9
Project: miner   File: TopologyMain.java   Source Code and License Vote up 5 votes
public static void main(String[] args) {
	try{
		TopologyBuilder topologyBuilder = new TopologyBuilder();
		topologyBuilder.setSpout("spout-number", new ProduceRecordSpout(Type.NUMBER, new String[]{"111 222 333", "80966 31"}), 1);
		topologyBuilder.setSpout("spout-string", new ProduceRecordSpout(Type.STRING, new String[]{"abc ddd fasko", "hello the world"}), 1);
		topologyBuilder.setSpout("spout-sign", new ProduceRecordSpout(Type.SIGN, new String[]{"++ -*% *** @@", "{+-} ^#######"}), 1);

		topologyBuilder.setBolt("bolt-splitter", new SplitRecordBolt(), 2)
				.shuffleGrouping("spout-number")
				.shuffleGrouping("spout-string")
				.shuffleGrouping("spout-sign");

		topologyBuilder.setBolt("bolt-distributor", new DistributeWordByTypeBolt(), 1)
				.fieldsGrouping("bolt-splitter", new Fields("type"));

		topologyBuilder.setBolt("bolt-number-saver", new SaveDataBolt(Type.NUMBER), 1)
				.shuffleGrouping("bolt-distributor", "stream-number-saver");
		topologyBuilder.setBolt("bolt-string-saver", new SaveDataBolt(Type.STRING), 1)
				.shuffleGrouping("bolt-distributor", "stream-string-saver");
		topologyBuilder.setBolt("bolt-sign-saver", new SaveDataBolt(Type.SIGN), 1)
				.shuffleGrouping("bolt-distributor", "stream-sign-saver");

		Config config = new Config();
		config.setDebug(false);
		
		if(args != null && args.length>0){
			config.setNumWorkers(4);
			StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology());
		}else{
			config.setMaxTaskParallelism(2);
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology("test", config, topologyBuilder.createTopology());
		}
		
	}catch(Exception e){
		e.printStackTrace();
	}
}
 
Example 10
Project: miner   File: TopologyMain.java   Source Code and License Vote up 5 votes
public static void main(String[] args) {
		try{
			TopologyBuilder topologyBuilder = new TopologyBuilder();
			topologyBuilder.setSpout("Spout", new EmitMessageSpout(), 1);

			topologyBuilder.setBolt("generate", new GenerateUrlBolt(), 1)
					.shuffleGrouping("Spout");
			topologyBuilder.setBolt("generate_loop", new GenerateUrlBolt(), 1)
					.shuffleGrouping("Parse", "loop");

//			topologyBuilder.setBolt("Parse", new ParseTestBolt(), 1).shuffleGrouping("Spout");
			topologyBuilder.setBolt("Parse", new ParseLoopBolt(), 1)
					.shuffleGrouping("generate")
					.shuffleGrouping("generate_loop");

			topologyBuilder.setBolt("Store", new StoreTestBolt(), 1)
					.shuffleGrouping("Parse", "store");
			
			Config config = new Config();
			config.setDebug(false);
			
			if(args != null && args.length>0){
				config.setNumWorkers(4);
				StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology());
			}else{
				config.setMaxTaskParallelism(2);
				LocalCluster cluster = new LocalCluster();
				cluster.submitTopology("test", config, topologyBuilder.createTopology());
			}
			
		}catch(Exception e){
			e.printStackTrace();
		}
	}
 
Example 11
Project: erad2016-streamprocessing   File: SentimentAnalysisTopology.java   Source Code and License Vote up 5 votes
private static Config createConfig(boolean local) {
    int workers = Properties.getInt("sa.storm.workers");
    Config conf = new Config();
    conf.setDebug(true);
    if (local)
        conf.setMaxTaskParallelism(workers);
    else
        conf.setNumWorkers(workers);
    return conf;
}
 
Example 12
Project: learn_jstorm   File: MasterBatchCoordinator.java   Source Code and License Vote up 5 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    ret.registerSerialization(TransactionAttempt.class);
    return ret;
}
 
Example 13
Project: jstrom   File: MasterBatchCoordinator.java   Source Code and License Vote up 5 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    ret.registerSerialization(TransactionAttempt.class);
    return ret;
}
 
Example 14
Project: cloud-computing-specialization   File: TopWordFinderTopologyPartC.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    Config config = new Config();
    config.setDebug(true);

/*
----------------------TODO-----------------------
Task: wire up the topology

NOTE:make sure when connecting components together, using the functions setBolt(name,…) and setSpout(name,…),
you use the following names for each component:

FileReaderSpout -> "spout"
SplitSentenceBolt -> "split"
WordCountBolt -> "count"
NormalizerBolt -> "normalize"
------------------------------------------------- */
    String spoutId = "spout";
    String splitId = "split";
    String countId = "count";
    String normalizeId = "normalize";

    builder.setSpout(spoutId, new FileReaderSpout(args[0]), 5);
    builder.setBolt(splitId, new SplitSentenceBolt(), 8).shuffleGrouping(spoutId);
    builder.setBolt(normalizeId, new NormalizerBolt(), 12).fieldsGrouping(splitId, new Fields("word"));
    builder.setBolt(countId, new WordCountBolt(), 12).fieldsGrouping(normalizeId, new Fields("word"));

    config.setMaxTaskParallelism(3);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("word-count", config, builder.createTopology());

    //wait for 2 minutes then kill the job
    Thread.sleep(2 * 60 * 1000);

    cluster.shutdown();
}
 
Example 15
Project: Tstream   File: MasterBatchCoordinator.java   Source Code and License Vote up 5 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    ret.registerSerialization(TransactionAttempt.class);
    return ret;
}
 
Example 16
Project: rb-bi   File: TopologyFunctionTest.java   Source Code and License Vote up 4 votes
@Test
public void geoIpTest() throws FileNotFoundException {

    File fileFlow = new File(Thread.currentThread().getContextClassLoader().getResource("inputData/flows.json").getPath());
    File checkFlow = new File(Thread.currentThread().getContextClassLoader().getResource("dataCheck/geoIpFlows.json").getPath());

    Scanner flows = new Scanner(fileFlow);
    Scanner checkFlows = new Scanner(checkFlow);

    GeoIpFunction.CITY_DB_PATH = Thread.currentThread().getContextClassLoader().getResource("db/city.dat").getPath();
    GeoIpFunction.CITY_V6_DB_PATH = Thread.currentThread().getContextClassLoader().getResource("db/cityv6.dat").getPath();
    GeoIpFunction.ASN_DB_PATH = Thread.currentThread().getContextClassLoader().getResource("db/asn.dat").getPath();
    GeoIpFunction.ASN_V6_DB_PATH = Thread.currentThread().getContextClassLoader().getResource("db/asnv6.dat").getPath();

    List<String> fieldsFlow = new ArrayList<String>();

    fieldsFlow.add("flows");
    fieldsFlow.add("geoIPMap");

    LocalDRPC drpc = new LocalDRPC();

    TridentTopology topology = new TridentTopology();
    topology.newDRPCStream("test", drpc)
            .each(new Fields("args"), new MapperFunction("rb_test"), new Fields("flows"))
            .each(new Fields("flows"), new GeoIpFunction(), new Fields("geoIPMap"))
            .each(new Fields(fieldsFlow), new MergeMapsFunction(), new Fields("finalMap"))
            .project(new Fields("finalMap"))
            .each(new Fields("finalMap"), new MapToJSONFunction(), new Fields("jsonString"));

    Config conf = new Config();
    conf.put("rbDebug", true);
    conf.setMaxTaskParallelism(1);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("testing-topology", conf, topology.build());

    while (flows.hasNextLine()) {
        String stormFlow = drpc.execute("test", flows.nextLine());
        stormFlow = stormFlow.substring(stormFlow.indexOf("{"), stormFlow.indexOf("}") + 1);
        //System.out.println(stormFlow);
        Assert.assertEquals(checkFlows.nextLine(), stormFlow);
    }
}
 
Example 17
Project: storm-kafka-examples   File: HdfsTopology.java   Source Code and License Vote up 4 votes
public static void main(String[] args) {
    try{
        String zkhost = "wxb-1:2181,wxb-2:2181,wxb-3:2181";
        String topic = "order";
        String groupId = "id";
        int spoutNum = 3;
        int boltNum = 1;
        ZkHosts zkHosts = new ZkHosts(zkhost);//kafaka所在的zookeeper
        SpoutConfig spoutConfig = new SpoutConfig(zkHosts, topic, "/order", groupId);  // create /order /id
        spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
        KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);

        // HDFS bolt
        // use "|" instead of "," for field delimiter
        RecordFormat format = new DelimitedRecordFormat()
                .withFieldDelimiter("|");

        // sync the filesystem after every 1k tuples
        SyncPolicy syncPolicy = new CountSyncPolicy(1000);

        // rotate files when they reach 5MB
        FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
        // FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);

        FileNameFormat fileNameFormat = new DefaultFileNameFormat()
                .withPath("/tmp/").withPrefix("order_").withExtension(".log");

        HdfsBolt hdfsBolt = new HdfsBolt()
                .withFsUrl("hdfs://wxb-1:8020")
                .withFileNameFormat(fileNameFormat)
                .withRecordFormat(format)
                .withRotationPolicy(rotationPolicy)
                .withSyncPolicy(syncPolicy);

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("spout", kafkaSpout, spoutNum);
        builder.setBolt("check", new CheckOrderBolt(), boltNum).shuffleGrouping("spout");
        builder.setBolt("counter", new CounterBolt(),boltNum).shuffleGrouping("check");
        builder.setBolt("hdfs", hdfsBolt,boltNum).shuffleGrouping("counter");

        Config config = new Config();
        config.setDebug(true);

        if(args!=null && args.length > 0) {
            config.setNumWorkers(2);
            StormSubmitter.submitTopology(args[0], config, builder.createTopology());
        } else {
            config.setMaxTaskParallelism(2);

            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("Wordcount-Topology", config, builder.createTopology());

            Thread.sleep(500000);

            cluster.shutdown();
        }
    }catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 18
Project: miner   File: TopologyMain.java   Source Code and License Vote up 4 votes
public static void main(String[] args) {
	try{
		TopologyBuilder topologyBuilder = new TopologyBuilder();

		topologyBuilder.setSpout("beginspout", new BeginSpout(), PlatformParas.begin_spout_num).setMaxSpoutPending(200);//1,500
		topologyBuilder.setSpout("loopspout", new LoopSpout(), PlatformParas.loop_spout_num).setMaxSpoutPending(200);

		topologyBuilder.setBolt("generateurl", new GenerateUrlBolt(), PlatformParas.generateurl_bolt_num)//2
				.shuffleGrouping("beginspout")
				.shuffleGrouping("loopspout");
		topologyBuilder.setBolt("generateurl-loop-bolt", new GenerateUrlBolt(), PlatformParas.generateurl_bolt_num)
				.shuffleGrouping("parse", "generate-loop");

		topologyBuilder.setBolt("proxy", new ProxyBolt(), PlatformParas.proxy_bolt_num)
				.shuffleGrouping("generateurl")
				.shuffleGrouping("generateurl-loop-bolt");

		topologyBuilder.setBolt("fetch", new FetchBolt(), PlatformParas.fetch_bolt_num)
				.shuffleGrouping("proxy");

		topologyBuilder.setBolt("parse", new ParseBolt(), PlatformParas.parse_bolt_num)
				.shuffleGrouping("fetch");

		topologyBuilder.setBolt("store", new StoreBolt(), PlatformParas.store_bolt_num)
				.shuffleGrouping("parse", "store");
		
		Config config = new Config();
		config.setDebug(false);
		//default:30s
		config.setMessageTimeoutSecs(PlatformParas.message_timeout_secs);
		//config.setMaxSpoutPending(2000);
		
		if(args != null && args.length>0){
			config.setNumWorkers(PlatformParas.work_num);
			StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology());
		}else{
			config.setMaxTaskParallelism(2);
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology("test", config, topologyBuilder.createTopology());
		}

	}catch(Exception e){
		e.printStackTrace();
	}
}
 
Example 19
Project: erad2016-streamprocessing   File: TwitterSpout.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    return ret;
}
 
Example 20
Project: Get-ENVS   File: WordCountTopology.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws Exception {
//Used to build the topology
  TopologyBuilder builder = new TopologyBuilder();
  //Add the spout, with a name of 'spout'
  //and parallelism hint of 5 executors
  builder.setSpout("spout", new RandomSentenceSpout(), 5);
  //Add the SplitSentence bolt, with a name of 'split'
  //and parallelism hint of 8 executors
  //shufflegrouping subscribes to the spout, and equally distributes
  //tuples (sentences) across instances of the SplitSentence bolt
  builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
  //Add the counter, with a name of 'count'
  //and parallelism hint of 12 executors
  //fieldsgrouping subscribes to the split bolt, and
  //ensures that the same word is sent to the same instance (group by field 'word')
  builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));

  //new configuration
  Config conf = new Config();
  conf.setDebug(true);

  //If there are arguments, we are running on a cluster
  if (args != null && args.length > 0) {
    //parallelism hint to set the number of workers
    conf.setNumWorkers(3);
    //submit the topology
    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
  }
  //Otherwise, we are running locally
  else {
    //Cap the maximum number of executors that can be spawned
    //for a component to 3
    conf.setMaxTaskParallelism(3);
    //LocalCluster is used to run locally
    LocalCluster cluster = new LocalCluster();
    //submit the topology
    cluster.submitTopology("word-count", conf, builder.createTopology());
    //sleep
    Thread.sleep(10000);
    //shut down the cluster
    cluster.shutdown();
  }
}
 
Example 21
Project: ignite-book-code-samples   File: SpeedViolationTopology.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws Exception {
    if (getProperties() == null || getProperties().isEmpty()) {
        System.out.println("Property file <ignite-storm.property> is not found or empty");
        return;
    }
    // Ignite Stream Ibolt
    final StormStreamer<String, String> stormStreamer = new StormStreamer<>();

    stormStreamer.setAutoFlushFrequency(10L);
    stormStreamer.setAllowOverwrite(true);
    stormStreamer.setCacheName(getProperties().getProperty("cache.name"));

    stormStreamer.setIgniteTupleField(getProperties().getProperty("tuple.name"));
    stormStreamer.setIgniteConfigFile(getProperties().getProperty("ignite.spring.xml"));


    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new FileSourceSpout(), 1);
    builder.setBolt("limit", new SpeedLimitBolt(), 1).fieldsGrouping("spout", new Fields("trafficLog"));
    // set ignite bolt
    builder.setBolt("ignite-bolt", stormStreamer, STORM_EXECUTORS).shuffleGrouping("limit");

    Config conf = new Config();
    conf.setDebug(false);

    conf.setMaxTaskParallelism(1);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("speed-violation", conf, builder.createTopology());
    Thread.sleep(10000);
    cluster.shutdown();

}
 
Example 22
Project: ignite-book-code-samples   File: WordCountTopology.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws Exception {
    // Ignite Stream Ibolt
    final StormStreamer<String, String> stormStreamer = new StormStreamer<>();

    stormStreamer.setAutoFlushFrequency(10L);
    stormStreamer.setAllowOverwrite(true);
    stormStreamer.setCacheName("testCache");
    stormStreamer.setIgniteTupleField("ignite");
    stormStreamer.setIgniteConfigFile("/Users/shamim/Development/workshop/assembla/ignite-book/chapters/chapter-cep-storm/src/main/resources/example-ignite.xml");

    //Used to build the topology
    TopologyBuilder builder = new TopologyBuilder();
    //Add the spout, with a name of 'spout'
    //and parallelism hint of 5 executors
    builder.setSpout("spout", new RandomSentenceSpout(), 5);
    //Add the SplitSentence bolt, with a name of 'split'
    //and parallelism hint of 8 executors
    //shufflegrouping subscribes to the spout, and equally distributes
    //tuples (sentences) across instances of the SplitSentence bolt
    builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
    //Add the counter, with a name of 'count'
    //and parallelism hint of 12 executors
    //fieldsgrouping subscribes to the split bolt, and
    //ensures that the same word is sent to the same instance (group by field 'word')
    builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));
    // set ignite bolt
    builder.setBolt("ignite-bolt", stormStreamer,STORM_EXECUTORS).shuffleGrouping("count");

    //new configuration
    Config conf = new Config();
    //Set to false to disable debug information
    // when running in production mode.
    conf.setDebug(false);

    //If there are arguments, we are running on a cluster
    if (args != null && args.length > 0) {
        //parallelism hint to set the number of workers
        conf.setNumWorkers(3);
        //submit the topology
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    }
    //Otherwise, we are running locally
    else {
        //Cap the maximum number of executors that can be spawned
        //for a component to 3
        conf.setMaxTaskParallelism(3);
        //LocalCluster is used to run locally
        LocalCluster cluster = new LocalCluster();
        //submit the topology
        cluster.submitTopology("word-count", conf, builder.createTopology());
        //sleep
        Thread.sleep(10000);
        //shut down the cluster
        cluster.shutdown();
    }
}
 
Example 23
Project: java   File: TradeProcessingTopology.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
  TopologyBuilder builder = new TopologyBuilder();

  LOGGER.info("Building Trade Processing Topology..");

  builder.setSpout(TRD_COLLECTOR_SPOUT, new TradeCollectorSpout(),
      CONFIG.getNumber("TRD_COLLECTOR_SPOUT_PARALLELISM"));

  builder
      .setBolt(TRD_ELIGIBILITY_BOLT, new TradeEligibilityBolt(),
          CONFIG.getNumber("TRD_ELIGIBILITY_BOLT_PARALLELISM"))
      .shuffleGrouping(TRD_COLLECTOR_SPOUT);

  builder
      .setBolt(TRD_REPORTING_BOLT, new TradeReportPersistenceBolt(),
          CONFIG.getNumber("TRD_REPORTING_BOLT_PARALLELISM"))
      .shuffleGrouping(TRD_ELIGIBILITY_BOLT, REPORT_STREAM);

  builder
      .setBolt(TRD_EXCLUSION_BOLT, new TradeExclusionPersistenceBolt(),
          CONFIG.getNumber("TRD_EXCLUSION_BOLT_PARALLELISM"))
      .shuffleGrouping(TRD_ELIGIBILITY_BOLT, EXCLUDE_STREAM);

  Config conf = new Config();
  conf.setDebug(CONFIG.is("DEBUG_FLAG"));
  conf.setNumWorkers(CONFIG.getInt("NUMBER_OF_WORKERS"));
  conf.setMaxTaskParallelism(CONFIG.getInt("MAX_TASK_PARALLELISM"));
  conf.setMaxSpoutPending(CONFIG.getInt("MAX_SPOUT_PENDING"));
  conf.put(Config.TOPOLOGY_SLEEP_SPOUT_WAIT_STRATEGY_TIME_MS,
      CONFIG.getInt("MAX_SPOUT_PENDING_WAIT_MS"));
  conf.put(Config.TOPOLOGY_SPOUT_WAIT_STRATEGY, CONFIG.get("TOPOLOGY_WAIT_STRATEGY"));
  conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, CONFIG.getInt("TOPOLOGY_MESSAGE_TIMEOUT_SECS"));
  conf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS,
      CONFIG.is("TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS"));
  LOGGER.info("Submitting Trade Processing Topology..");
  if (args != null && args.length > 0) {
    conf.put(Config.NIMBUS_HOST, CONFIG.get("LOCAL_NIMBUS_HOST"));
    conf.put(Config.NIMBUS_THRIFT_PORT, CONFIG.getInt("LOCAL_NIMBUS_PORT"));
    conf.put(Config.STORM_ZOOKEEPER_PORT, CONFIG.getInt("LOCAL_ZOOKEEPER_PORT"));
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
  } else {
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology(TRD_PROCESSING_TOPOLOGY, conf, builder.createTopology());
    Utils.sleep(CONFIG.getLong("LOCAL_CLUSTER_RUNTIME"));
    cluster.killTopology(TRD_PROCESSING_TOPOLOGY);
    cluster.shutdown();
  }
}
 
Example 24
Project: jstrom   File: FixedBatchSpout.java   Source Code and License Vote up 4 votes
@Override
public Map getComponentConfiguration() {
    Config conf = new Config();
    conf.setMaxTaskParallelism(1);
    return conf;
}
 
Example 25
Project: jstrom   File: TridentSpoutCoordinator.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    return ret;
}
 
Example 26
Project: Tstream   File: TransactionalSpoutCoordinator.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
	Config ret = new Config();
	ret.setMaxTaskParallelism(1);
	return ret;
}
 
Example 27
Project: big-data-system   File: Q2FetchTweetSpout.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
	Config ret = new Config();
	ret.setMaxTaskParallelism(1);
	return ret;
}
 
Example 28
Project: senti-storm   File: TwitterStreamSpout.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
  Config ret = new Config();
  ret.setMaxTaskParallelism(1);
  return ret;
}
 
Example 29
Project: big-data-system   File: TwitterNoKeywordSpout.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
	Config ret = new Config();
	ret.setMaxTaskParallelism(1);
	return ret;
}
 
Example 30
Project: docker-kafka-storm   File: WordCountTopology.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws Exception {

        Config conf = new Config();
        String TOPOLOGY_NAME;

        if (args != null && args.length > 0) {
            TOPOLOGY_NAME = args[0];
            /**
             * Remote deployment as part of Docker Compose multi-application setup
             *
             * @TOPOLOGY_NAME:       Name of Storm topology
             * @ZK_HOST:             Host IP address of ZooKeeper
             * @ZK_PORT:             Port of ZooKeeper
             * @TOPIC:               Kafka Topic which this Storm topology is consuming from
             */
            LOG.info("Submitting topology " + TOPOLOGY_NAME + " to remote cluster.");
            String ZK_HOST = args[1];
            int ZK_PORT = Integer.parseInt(args[2]);
            String TOPIC = args[3];
            String NIMBUS_HOST = args[4];
            int NIMBUS_THRIFT_PORT = Integer.parseInt(args[5]);

            conf.setDebug(false);
            conf.setNumWorkers(2);
            conf.setMaxTaskParallelism(5);
            conf.put(Config.NIMBUS_HOST, NIMBUS_HOST);
            conf.put(Config.NIMBUS_THRIFT_PORT, NIMBUS_THRIFT_PORT);
            conf.put(Config.STORM_ZOOKEEPER_PORT, ZK_PORT);
            conf.put(Config.STORM_ZOOKEEPER_SERVERS, Arrays.asList(ZK_HOST));

            WordCountTopology wordCountTopology = new WordCountTopology(ZK_HOST, String.valueOf(ZK_PORT));
            StormSubmitter.submitTopology(TOPOLOGY_NAME, conf, wordCountTopology.buildTopology(TOPIC));

        }
        else {
            TOPOLOGY_NAME = "wordcount-topology";
            /**
             * Local mode (only for testing purposes)
             */
            LOG.info("Starting topology " + TOPOLOGY_NAME + " in LocalMode.");

            conf.setDebug(false);
            conf.setNumWorkers(2);
            conf.setMaxTaskParallelism(2);

            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology(TOPOLOGY_NAME, conf, new WordCountTopology().buildTopology());

            Thread.sleep(10000);
            cluster.shutdown();
        }
    }
 
Example 31
Project: learn_jstorm   File: TransactionalSpoutCoordinator.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
	Config ret = new Config();
	ret.setMaxTaskParallelism(1);
	return ret;
}
 
Example 32
Project: aeolus   File: SocketClientSpoutPure.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
	Config conf = new Config();
	conf.setMaxTaskParallelism(1);
	return conf;
}
 
Example 33
Project: incubator-samoa   File: StormDoTask.java   Source Code and License Vote up 4 votes
/**
 * The main method.
 * 
 * @param args
 *          the arguments
 */
public static void main(String[] args) {

  List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));

  boolean isLocal = isLocal(tmpArgs);
  int numWorker = StormSamoaUtils.numWorkers(tmpArgs);

  args = tmpArgs.toArray(new String[0]);

  // convert the arguments into Storm topology
  StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);
  String topologyName = stormTopo.getTopologyName();

  Config conf = new Config();
  conf.putAll(Utils.readStormConfig());
  conf.setDebug(false);

  if (isLocal) {
    // local mode
    conf.setMaxTaskParallelism(numWorker);

    backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster();
    cluster.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology());

    backtype.storm.utils.Utils.sleep(600 * 1000);

    cluster.killTopology(topologyName);
    cluster.shutdown();

  } else {
    // cluster mode
    conf.setNumWorkers(numWorker);
    try {
      backtype.storm.StormSubmitter.submitTopology(topologyName, conf,
          stormTopo.getStormBuilder().createTopology());
    } catch (backtype.storm.generated.AlreadyAliveException ale) {
      ale.printStackTrace();
    } catch (backtype.storm.generated.InvalidTopologyException ite) {
      ite.printStackTrace();
    }
  }
}
 
Example 34
Project: big-data-system   File: TwitterSampleSpout.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
	Config ret = new Config();
	ret.setMaxTaskParallelism(1);
	return ret;
}
 
Example 35
Project: big-data-system   File: Q2SeqTwitterSpout.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
	Config ret = new Config();
	ret.setMaxTaskParallelism(1);
	return ret;
}
 
Example 36
Project: Big-Data-tutorial   File: WordCountTopology.java   Source Code and License Vote up 3 votes
public static void main(String[] args) throws Exception {

    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new RandomSentenceSpout(), 5);

    builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));

    Config conf = new Config();
    conf.setDebug(true);


    if (args != null && args.length > 0) {
      conf.setNumWorkers(3);

      StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    }
    else {
      conf.setMaxTaskParallelism(3);

      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("word-count", conf, builder.createTopology());

      Thread.sleep(100000);

      cluster.shutdown();
    }
  }