Java Code Examples for backtype.storm.topology.TopologyBuilder.setSpout()

The following are Jave code examples for showing how to use setSpout() of the backtype.storm.topology.TopologyBuilder class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
Example 1
Project: es-hadoop-v2.2.0   File: AbstractStormIndexPatternBoltTests.java   Source Code and License Vote up 6 votes
@Test
public void test1WriteIndexPattern() throws Exception {
    List doc1 = ImmutableList.of("one", "1", "two", "2", "number", 1);
    List doc2 = ImmutableList.of("OTP", "Otopeni", "SFO", "San Fran", "number", 2);

    String target = index + "/write-{number}";
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("test-spout-3", new TestSpout(ImmutableList.of(doc2, doc1), new Fields("key1", "val1", "key2",
            "val2", "key3", "number")));
    builder.setBolt("es-bolt-3", new TestBolt(new EsBolt(target, conf))).shuffleGrouping("test-spout-3");

    MultiIndexSpoutStormSuite.run(index + "write-pattern", builder.createTopology(), COMPONENT_HAS_COMPLETED);

    COMPONENT_HAS_COMPLETED.waitFor(1, TimeValue.timeValueSeconds(20));

    Thread.sleep(1000);
    RestUtils.refresh(index);
    assertTrue(RestUtils.exists(index + "/write-1"));
    assertTrue(RestUtils.exists(index + "/write-2"));

    String results = RestUtils.get(index + "/write-1" + "/_search?");
    assertThat(results, containsString("two"));

    results = RestUtils.get(index + "/write-2" + "/_search?");
    assertThat(results, containsString("SFO"));
}
 
Example 2
Project: storm-demo   File: LogStatisticsTopology.java   Source Code and License Vote up 6 votes
public static void main(String[] args) {
    Config config = new Config();

    HdfsBolt hdfsBolt = makeHdfsBolt();
    KafkaSpout kafkaSpout = makeKafkaSpout(TOPIC, TOPOLOGY_NAME);

    LOG.info("Topology name is {}", TOPOLOGY_NAME);

    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout(KAFKA_SPOUT_ID, kafkaSpout, 10);
    topologyBuilder.setBolt(CROP_BOLT_ID, new CropBolt(), 10).shuffleGrouping(KAFKA_SPOUT_ID);
    topologyBuilder.setBolt(SPLIT_FIELDS_BOLT_ID, new SplitFieldsBolt(), 10).shuffleGrouping(CROP_BOLT_ID);
    topologyBuilder.setBolt(STORM_HDFS_BOLT_ID, hdfsBolt, 4).fieldsGrouping(SPLIT_FIELDS_BOLT_ID, new Fields("timestamp", "fieldvalues"));

    if (args != null && args.length > 0) {
        config.setDebug(false);
        config.setNumWorkers(3);

        try {
            StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology());
        } catch (InvalidTopologyException | AlreadyAliveException | AuthorizationException e) {
            e.printStackTrace();
        }
    }
}
 
Example 3
Project: es-hadoop-v2.2.0   File: AbstractStormJsonSimpleBoltTests.java   Source Code and License Vote up 6 votes
@Test
public void testSimpleWriteTopology() throws Exception {
    List doc1 = Collections.singletonList("{\"reason\" : \"business\",\"airport\" : \"SFO\"}");
    List doc2 = Collections.singletonList("{\"participants\" : 5,\"airport\" : \"OTP\"}");

    String target = index + "/json-simple-write";
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("test-spout-1", new TestSpout(ImmutableList.of(doc1, doc2), new Fields("json")));
    builder.setBolt("es-bolt-1", new TestBolt(new EsBolt(target, conf))).shuffleGrouping("test-spout-1");

    MultiIndexSpoutStormSuite.run(index + "json-simple", builder.createTopology(), COMPONENT_HAS_COMPLETED);

    COMPONENT_HAS_COMPLETED.waitFor(1, TimeValue.timeValueSeconds(10));

    RestUtils.refresh(index);
    assertTrue(RestUtils.exists(target));
    String results = RestUtils.get(target + "/_search?");
    assertThat(results, containsString("SFO"));
}
 
Example 4
Project: es-hadoop-v2.2.0   File: AbstractSpoutSimpleRead.java   Source Code and License Vote up 6 votes
@Test
public void testSimpleRead() throws Exception {
    String target = index + "/basic-read";

    RestUtils.touch(index);
    RestUtils.postData(target, "{\"message\" : \"Hello World\",\"message_date\" : \"2014-05-25\"}".getBytes());
    RestUtils.postData(target, "{\"message\" : \"Goodbye World\",\"message_date\" : \"2014-05-25\"}".getBytes());
    RestUtils.refresh(index);

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("es-spout", new TestSpout(new EsSpout(target)));
    builder.setBolt("test-bolt", new CapturingBolt()).shuffleGrouping("es-spout");

    MultiIndexSpoutStormSuite.run(index + "simple", builder.createTopology(), COMPONENT_HAS_COMPLETED);

    COMPONENT_HAS_COMPLETED.waitFor(1, TimeValue.timeValueSeconds(10));

    assertTrue(RestUtils.exists(target));
    String results = RestUtils.get(target + "/_search?");
    assertThat(results, containsString("Hello"));
    assertThat(results, containsString("Goodbye"));

    System.out.println(CapturingBolt.CAPTURED);
    assertThat(CapturingBolt.CAPTURED.size(), is(2));
}
 
Example 5
Project: StreamBench   File: AppTest.java   Source Code and License Vote up 6 votes
public static void main(String[] args) throws WorkloadException {
    TopologyBuilder builder = new TopologyBuilder();
    BrokerHosts hosts = new ZkHosts("localhost:2181");
    SpoutConfig spoutConfig = new SpoutConfig(hosts, "WordCount", "/" + "WordCount", UUID.randomUUID().toString());
    spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
    spoutConfig.ignoreZkOffsets = true;

    builder.setSpout("spout", new KafkaSpout(spoutConfig));
    builder.setBolt("split", new SplitSentence()).shuffleGrouping("spout");
    builder.setBolt("counter", new CounterBolt(), 3).fieldsGrouping("split", new Fields("wordCountPair"));

    Config conf = new Config();
    conf.setDebug(true);
    conf.setNumWorkers(3);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("kafka-spout", conf, builder.createTopology());
}
 
Example 6
Project: StreamBench   File: TickTest.java   Source Code and License Vote up 6 votes
public static void main(String[] args) throws WorkloadException {
    TopologyBuilder builder = new TopologyBuilder();
    BrokerHosts hosts = new ZkHosts("localhost:2181");
    SpoutConfig spoutConfig = new SpoutConfig(hosts, "WordCount", "/" + "WordCount", UUID.randomUUID().toString());
    spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
    spoutConfig.ignoreZkOffsets = true;

    builder.setSpout("spout", new KafkaSpout(spoutConfig));
    builder.setBolt("split", new SplitSentence()).shuffleGrouping("spout");
    builder.setBolt("counter", new CounterBolt(), 3).shuffleGrouping("split");
    builder.setBolt("aggregator", new AggregatorBolt(), 1)
            .fieldsGrouping("counter", Utils.DEFAULT_STREAM_ID, new Fields("word"))
            .allGrouping("counter", "tick");

    Config conf = new Config();
    conf.setDebug(true);
    conf.setNumWorkers(3);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("kafka-spout", conf, builder.createTopology());
}
 
Example 7
Project: LogRTA   File: KafkaTopology.java   Source Code and License Vote up 6 votes
public static void main(String[] args) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new KafkaSpoutTest(""), 1);
    builder.setBolt("bolt1", new Bolt1(), 2).shuffleGrouping("spout");
    builder.setBolt("bolt2", new Bolt2(), 2).fieldsGrouping("bolt1",new Fields("word"));
 
    Map conf = new HashMap();
    conf.put(Config.TOPOLOGY_WORKERS, 1);
    conf.put(Config.TOPOLOGY_DEBUG, true);
 
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("flume-kafka-storm-integration", conf, builder.createTopology());
     
    Utils.sleep(1000*60*5);
    cluster.shutdown();
}
 
Example 8
Project: storm-lib   File: FunctionTopology.java   Source Code and License Vote up 6 votes
void run(String[] args) {
TopologyBuilder builder = new TopologyBuilder();

builder.setSpout("word_spout", new TestWordSpout(), 2);
builder.setBolt("tuple_double", new Function(new Double(), new Fields("word1", "word2")), 2).shuffleGrouping("word_spout");
	
Map conf = new HashMap();
conf.put(Config.TOPOLOGY_WORKERS, 4);
conf.put(Config.TOPOLOGY_DEBUG, true);
	
if(args.length==0) {
           LocalCluster cluster = new LocalCluster();
           cluster.submitTopology("MockIngest", conf, builder.createTopology());
       } else {
    try {
	StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } catch(Exception e) {
	e.printStackTrace();
    }
}
   }
 
Example 9
Project: Practical-Real-time-Processing-and-Analytics   File: FlinkStormExample.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws Exception {
	TopologyBuilder topologyBuilder = new TopologyBuilder();
	
	topologyBuilder.setSpout("spout", new FileSpout("/tmp/device-data.txt"), 1);
	topologyBuilder.setBolt("parser", new ParserBolt(), 1).shuffleGrouping("spout");
	topologyBuilder.setBolt("tdrCassandra", new TDRCassandraBolt("localhost", "tdr"), 1).shuffleGrouping("parser", "tdrstream");
	
	FlinkTopology.createTopology(topologyBuilder).execute();
}
 
Example 10
Project: Practical-Real-time-Processing-and-Analytics   File: FlinkStormExample.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws Exception {
	TopologyBuilder topologyBuilder = new TopologyBuilder();
	
	topologyBuilder.setSpout("spout", new FileSpout("/tmp/device-data.txt"), 1);
	topologyBuilder.setBolt("parser", new ParserBolt(), 1).shuffleGrouping("spout");
	topologyBuilder.setBolt("tdrCassandra", new TDRCassandraBolt("localhost", "tdr"), 1).shuffleGrouping("parser", "tdrstream");
	
	FlinkTopology.createTopology(topologyBuilder).execute();
}
 
Example 11
Project: Mastering-Apache-Storm   File: Topology.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws AlreadyAliveException,
			InvalidTopologyException {
		TopologyBuilder builder = new TopologyBuilder();
		
		List<String> zks = new ArrayList<String>();
		zks.add("192.168.41.122");
		
		List<String> cFs = new ArrayList<String>();
		cFs.add("personal");
		cFs.add("company");
		
		// set the spout class
		builder.setSpout("spout", new SampleSpout(), 2);
		// set the bolt class
		builder.setBolt("bolt", new StormRedisBolt("192.168.41.122",2181), 2).shuffleGrouping("spout");

		Config conf = new Config();
		conf.setDebug(true);
		// create an instance of LocalCluster class for
		// executing topology in local mode.
		LocalCluster cluster = new LocalCluster();

		// LearningStormTopolgy is the name of submitted topology.
		cluster.submitTopology("StormRedisTopology", conf,
				builder.createTopology());
		try {
			Thread.sleep(10000);
		} catch (Exception exception) {
			System.out.println("Thread interrupted exception : " + exception);
		}
		// kill the LearningStormTopology
		cluster.killTopology("StormRedisTopology");
		// shutdown the storm test cluster
		cluster.shutdown();
}
 
Example 12
Project: RealEstate-Streaming   File: PhoenixTest.java   Source Code and License Vote up 5 votes
public int configureKafkaSpout(TopologyBuilder builder) {
    KafkaSpout kafkaSpout = constructKafkaSpout();

    //int spoutCount = Integer.valueOf(topologyConfig.getProperty("spout.thread.count"));
    //int boltCount = Integer.valueOf(topologyConfig.getProperty("bolt.thread.count"));
    
    int spoutCount = Integer.valueOf(1);
    int boltCount = Integer.valueOf(1);

    builder.setSpout("kafkaSpout", kafkaSpout, spoutCount);
    return boltCount;
}
 
Example 13
Project: splice-community-sample-code   File: SpliceDumperTopology.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws SQLException {

        ArrayList<String> columnNames = new ArrayList<String>();
        ArrayList<String> columnTypes = new ArrayList<String>();
        // this table must exist in splice
        // create table testTable (word varchar(100), number int);
        String tableName = "testTable";
        String server = "localhost";

        // add the column names and the respective types in the two arraylists
        columnNames.add("word");
        columnNames.add("number");

        // add the types
        columnTypes.add("varchar (100)");
        columnTypes.add("int");

        TopologyBuilder builder = new TopologyBuilder();

        // set the spout for the topology
        builder.setSpout("spout", new SpliceIntegerSpout(), 10);

        // dump the stream data into splice       
        SpliceDumperBolt dumperBolt = new SpliceDumperBolt(server, tableName);
        builder.setBolt("dumperBolt", dumperBolt, 1).shuffleGrouping("spout");
        Config conf = new Config();
        conf.setDebug(true);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("splice-topology", conf, builder.createTopology());
        Utils.sleep(10000);
        cluster.shutdown();
    }
 
Example 14
Project: preliminary.demo   File: RaceTopologyLocal.java   Source Code and License Vote up 5 votes
public static void main(String[] args) {
	LocalCluster cluster = new LocalCluster();

	
	/* begin young-define*/
	Config conf = new Config();
	TopologyBuilder builder = new TopologyBuilder();
	builder.setSpout("spout", new SpoutLocal(), 1);
       builder.setBolt("split", new SplitSentenceLocal(), 1).shuffleGrouping("spout");
       builder.setBolt("count", new WordCountLocal(), 1).fieldsGrouping("split", new Fields("word"));
       /* end young-define */
       
	
	//建议加上这行,使得每个bolt/spout的并发度都为1
	conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 1);

	//提交拓扑
	cluster.submitTopology("SequenceTest", conf, builder.createTopology());

	//等待1分钟, 1分钟后会停止拓扑和集群, 视调试情况可增大该数值
	try {
		Thread.sleep(60000);
	} catch (InterruptedException e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}        

	//结束拓扑
	cluster.killTopology("SequenceTest");

	cluster.shutdown();
}
 
Example 15
Project: java   File: DeliveryTopology.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
  TopologyBuilder builder = new TopologyBuilder();

  LOGGER.info("Starting..");
  builder.setSpout("trade", new DeliveryCheckSpout(), 1);
  builder.setBolt("eligibility", new DeliveryCheckBolt(), 10).shuffleGrouping("trade");
  builder.setBolt("odd", new DeliveryCheckOddBolt(), 10).shuffleGrouping("eligibility",
      "oddstream");
  builder.setBolt("even", new DeliveryCheckEvenBolt(), 10).shuffleGrouping("eligibility",
      "evenstream");

  Config conf = new Config();
  conf.setDebug(false);
  conf.setMaxSpoutPending(5);

  if (args != null && args.length > 0) {
    conf.setNumWorkers(1);
    LOGGER.info("Submitting DeliveryTopology");
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
  } else {

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("DeliveryTopology", conf, builder.createTopology());
    Utils.sleep(100000000);
    cluster.killTopology("DeliveryTopology");
    cluster.shutdown();
  }
}
 
Example 16
Project: miner   File: TopologyMain.java   Source Code and License Vote up 5 votes
public static void main(String[] args) {
	try{
		TopologyBuilder topologyBuilder = new TopologyBuilder();
		topologyBuilder.setSpout("Spout", new EmitMessageSpout(), 1);

		topologyBuilder.setBolt("generate", new ParseLoopBolt(), 1)
				.shuffleGrouping("Spout");



		topologyBuilder.setBolt("Store", new PrintBolt(), 1)
				.shuffleGrouping("generate");
		
		Config config = new Config();
		config.setDebug(false);
		
		if(args != null && args.length>0){
			config.setNumWorkers(4);
			StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology());
		}else{
			config.setMaxTaskParallelism(2);
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology("test", config, topologyBuilder.createTopology());
		}
		
	}catch(Exception e){
		e.printStackTrace();
	}
}
 
Example 17
Project: miner   File: TopologyMain.java   Source Code and License Vote up 5 votes
public static void main(String[] args) {
	try{
		TopologyBuilder topologyBuilder = new TopologyBuilder();
		topologyBuilder.setSpout("spout-number", new ProduceRecordSpout(Type.NUMBER, new String[]{"111 222 333", "80966 31"}), 1);
		topologyBuilder.setSpout("spout-string", new ProduceRecordSpout(Type.STRING, new String[]{"abc ddd fasko", "hello the world"}), 1);
		topologyBuilder.setSpout("spout-sign", new ProduceRecordSpout(Type.SIGN, new String[]{"++ -*% *** @@", "{+-} ^#######"}), 1);

		topologyBuilder.setBolt("bolt-splitter", new SplitRecordBolt(), 2)
				.shuffleGrouping("spout-number")
				.shuffleGrouping("spout-string")
				.shuffleGrouping("spout-sign");

		topologyBuilder.setBolt("bolt-distributor", new DistributeWordByTypeBolt(), 1)
				.fieldsGrouping("bolt-splitter", new Fields("type"));

		topologyBuilder.setBolt("bolt-number-saver", new SaveDataBolt(Type.NUMBER), 1)
				.shuffleGrouping("bolt-distributor", "stream-number-saver");
		topologyBuilder.setBolt("bolt-string-saver", new SaveDataBolt(Type.STRING), 1)
				.shuffleGrouping("bolt-distributor", "stream-string-saver");
		topologyBuilder.setBolt("bolt-sign-saver", new SaveDataBolt(Type.SIGN), 1)
				.shuffleGrouping("bolt-distributor", "stream-sign-saver");

		Config config = new Config();
		config.setDebug(false);
		
		if(args != null && args.length>0){
			config.setNumWorkers(4);
			StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology());
		}else{
			config.setMaxTaskParallelism(2);
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology("test", config, topologyBuilder.createTopology());
		}
		
	}catch(Exception e){
		e.printStackTrace();
	}
}
 
Example 18
Project: miner   File: ExclaimBasicTopo.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new RandomSpout());
    builder.setBolt("exclaim", new ProxyBolt()).shuffleGrouping("spout");
    builder.setBolt("print", new PrintBolt()).shuffleGrouping("exclaim");

    Config conf = new Config();
    conf.setDebug(false);

    /* Config里封装了Redis的配置 */
    conf.put("ip","127.0.0.1");
    conf.put("port","6379");
    conf.put("password","password");

    if (args != null && args.length > 0) {
        conf.setNumWorkers(1);

        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {

        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", conf, builder.createTopology());
        Utils.sleep(10*1000);
        cluster.killTopology("test");
        cluster.shutdown();
    }
}
 
Example 19
Project: sourcevirtues-samples   File: DummyJsonTerminalLogTopology.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws Exception {
    Config config = new Config();

    RandomJsonTestSpout spout = new RandomJsonTestSpout().withComplexJson(false);

    String2ByteArrayTupleMapper tuppleMapper = new String2ByteArrayTupleMapper();
    tuppleMapper.configure(CmnStormCons.TUPLE_FIELD_MSG);

    MorphlinesBolt morphBolt = new MorphlinesBolt()
            .withTupleMapper(tuppleMapper)
            .withMorphlineId("json_terminal_log")
            .withMorphlineConfFile("target/test-classes/morphline_confs/json_terminal_log.conf");

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("WORD_SPOUT", spout, 1);
    builder.setBolt("MORPH_BOLT", morphBolt, 1).shuffleGrouping("WORD_SPOUT");

    if (args.length == 0) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("MyDummyJsonTerminalLogTopology", config, builder.createTopology());
        Thread.sleep(10000);
        cluster.killTopology("MyDummyJsonTerminalLogTopology");
        cluster.shutdown();
        System.exit(0);
    } else if (args.length == 1) {
        StormSubmitter.submitTopology(args[0], config, builder.createTopology());
    } else {
        System.out.println("Usage: DummyJsonTerminalLogTopology <topology_name>");
    }
}
 
Example 20
Project: jstrom   File: SingleJoinTest.java   Source Code and License Vote up 5 votes
@Test
public void test_single_join() {
    try {
        FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
        FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("gender", genderSpout);
        builder.setSpout("age", ageSpout);
        builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))).fieldsGrouping("gender", new Fields("id"))
                .fieldsGrouping("age", new Fields("id"));

        Config conf = new Config();
        conf.setDebug(true);

        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("join-example", conf, builder.createTopology());

        for (int i = 0; i < 10; i++) {
            String gender;
            if (i % 2 == 0) {
                gender = "male";
            } else {
                gender = "female";
            }
            genderSpout.feed(new Values(i, gender));
        }

        for (int i = 9; i >= 0; i--) {
            ageSpout.feed(new Values(i, i + 20));
        }

        JStormUtils.sleepMs(60 * 1000);
        cluster.shutdown();
    } catch (Exception e) {
        Assert.fail("Failed to run SingleJoinExample");
    }
}
 
Example 21
Project: LearnStorm   File: TridentKafkaWordCount.java   Source Code and License Vote up 5 votes
/**
 * A topology that produces random sentences using {@link RandomSentenceSpout} and
 * publishes the sentences using a KafkaBolt to kafka "test" topic.
 *
 * @return the storm topology
 */
public StormTopology buildProducerTopology() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new RandomSentenceSpout(), 2);
    /**
     * The output field of the RandomSentenceSpout ("word") is provided as the boltMessageField
     * so that this gets written out as the message in the kafka topic.
     */
    KafkaBolt bolt = new KafkaBolt()
            .withTopicSelector(new DefaultTopicSelector(KAFKA_TOPIC))
            .withTupleToKafkaMapper(new FieldNameBasedTupleToKafkaMapper("key", "word"));
    builder.setBolt("forwardToKafka", bolt, 1).shuffleGrouping("spout");
    return builder.createTopology();
}
 
Example 22
Project: storm-kafka-hdfs-example   File: KafkaHdfsTopologyTest.java   Source Code and License Vote up 5 votes
@Test
public void testKafkaHbaseHdfsTopology() throws Exception {

    // Put messages in Kafka
    produceMessageToKafka();

    // Create the topology builder object
    TopologyBuilder topologyBuilder = new TopologyBuilder();

    // Setup the Kafka Spout and add it to the topology
    KafkaSpoutConfigBuilder kafkaSpoutConfigBuilder = getKafkaSpoutConfigBuilder();
    topologyBuilder.setSpout(kafkaSpoutConfigBuilder.getSpoutName(), kafkaSpoutConfigBuilder.getKafkaSpout(),
            kafkaSpoutConfigBuilder.getSpoutParallelismHint());


    // Setup the HDFS Bolt and add it to the topology
    HdfsBoltConfigBuilder hdfsBoltConfigBuilder = getHdfsBoltConfigBuilder();
    topologyBuilder.setBolt(hdfsBoltConfigBuilder.getBoltName(), hdfsBoltConfigBuilder.getHdfsBolt(),
            hdfsBoltConfigBuilder.getBoltParallelism()).shuffleGrouping(kafkaSpoutConfigBuilder.getSpoutName());


    // Setup the Storm configuration
    Config stormConfig = StormConfig.createStormConfig(
            Boolean.parseBoolean(propertyParser.getProperty(ConfigVars.STORM_ENABLE_DEBUG_KEY)),
            Integer.parseInt(propertyParser.getProperty(ConfigVars.STORM_NUM_WORKERS_KEY)));

    // Submit the topology
    stormLocalCluster.submitTopology(propertyParser.getProperty(ConfigVars.STORM_TOPOLOGY_NAME_KEY),
            stormConfig, topologyBuilder.createTopology());

    // Let the topology run
    Thread.sleep(10000);

    // Validate the results match the input
    validateHdfsResults();


}
 
Example 23
Project: iot-masterclass   File: TruckEventProcessorKafkaTopology.java   Source Code and License Vote up 5 votes
public int configureKafkaSpout(TopologyBuilder builder) {
  KafkaSpout kafkaSpout = constructKafkaSpout();

  int spoutCount = Integer.valueOf(topologyConfig.getProperty("spout.thread.count"));
  int boltCount = Integer.valueOf(topologyConfig.getProperty("bolt.thread.count"));

  builder.setSpout("kafkaSpout", kafkaSpout, spoutCount);
  return boltCount;
}
 
Example 24
Project: sourcevirtues-samples   File: DummyJsonTerminalLogTopology.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws Exception {
    Config config = new Config();

    RandomJsonTestSpout spout = new RandomJsonTestSpout().withComplexJson(false);

    String2ByteArrayTupleMapper tuppleMapper = new String2ByteArrayTupleMapper();
    tuppleMapper.configure(CmnStormCons.TUPLE_FIELD_MSG);

    MorphlinesBolt morphBolt = new MorphlinesBolt()
            .withTupleMapper(tuppleMapper)
            .withMorphlineId("json_terminal_log")
            .withMorphlineConfFile("target/test-classes/morphline_confs/json_terminal_log.conf");

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("WORD_SPOUT", spout, 1);
    builder.setBolt("MORPH_BOLT", morphBolt, 1).shuffleGrouping("WORD_SPOUT");

    if (args.length == 0) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("MyDummyJsonTerminalLogTopology", config, builder.createTopology());
        Thread.sleep(10000);
        cluster.killTopology("MyDummyJsonTerminalLogTopology");
        cluster.shutdown();
        System.exit(0);
    } else if (args.length == 1) {
        StormSubmitter.submitTopology(args[0], config, builder.createTopology());
    } else {
        System.out.println("Usage: DummyJsonTerminalLogTopology <topology_name>");
    }
}
 
Example 25
Project: docker-kafka-storm   File: WordCountTopology.java   Source Code and License Vote up 5 votes
/**
 * WordCountTopology with Kafka
 *
 * @return      StormTopology Object
 */
public StormTopology buildTopology(String TOPIC) {

    SpoutConfig kafkaConf = new SpoutConfig(brokerHosts, TOPIC, "", "storm");
    kafkaConf.scheme = new SchemeAsMultiScheme(new StringScheme());

    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout(KAFKA_SPOUT_ID, new KafkaSpout(kafkaConf));
    builder.setBolt(SPLITTER_BOLT_ID, new SplitterBolt(), 4).shuffleGrouping(KAFKA_SPOUT_ID);
    builder.setBolt(COUNTER_BOLT_ID, new CounterBolt(), 4).fieldsGrouping(SPLITTER_BOLT_ID, new Fields("word"));
    builder.setBolt(RANKER_BOLT_ID, new RankerBolt()).globalGrouping(COUNTER_BOLT_ID);

    return builder.createTopology();
}
 
Example 26
Project: storm-hbase-1.0.x   File: LookupWordCount.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws Exception {
    Config config = new Config();

    Map<String, Object> hbConf = new HashMap<String, Object>();
    if(args.length > 0){
        hbConf.put("hbase.rootdir", args[0]);
    }
    config.put("hbase.conf", hbConf);

    WordSpout spout = new WordSpout();
    TotalWordCounter totalBolt = new TotalWordCounter();

    SimpleHBaseMapper mapper = new SimpleHBaseMapper().withRowKeyField("word");
    HBaseProjectionCriteria projectionCriteria = new HBaseProjectionCriteria();
    projectionCriteria.addColumn(new HBaseProjectionCriteria.ColumnMetaData("cf", "count"));

    WordCountValueMapper rowToTupleMapper = new WordCountValueMapper();

    HBaseLookupBolt hBaseLookupBolt = new HBaseLookupBolt("WordCount", mapper, rowToTupleMapper)
            .withConfigKey("hbase.conf")
            .withProjectionCriteria(projectionCriteria);

    //wordspout -> lookupbolt -> totalCountBolt
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(WORD_SPOUT, spout, 1);
    builder.setBolt(LOOKUP_BOLT, hBaseLookupBolt, 1).shuffleGrouping(WORD_SPOUT);
    builder.setBolt(TOTAL_COUNT_BOLT, totalBolt, 1).fieldsGrouping(LOOKUP_BOLT, new Fields("columnName"));

    if (args.length == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", config, builder.createTopology());
        Thread.sleep(30000);
        cluster.killTopology("test");
        cluster.shutdown();
        System.exit(0);
    } else if (args.length == 2) {
        StormSubmitter.submitTopology(args[1], config, builder.createTopology());
    } else{
        System.out.println("Usage: LookupWordCount <hbase.rootdir>");
    }
}
 
Example 27
Project: ignite-book-code-samples   File: WordCountTopology.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws Exception {
    // Ignite Stream Ibolt
    final StormStreamer<String, String> stormStreamer = new StormStreamer<>();

    stormStreamer.setAutoFlushFrequency(10L);
    stormStreamer.setAllowOverwrite(true);
    stormStreamer.setCacheName("testCache");
    stormStreamer.setIgniteTupleField("ignite");
    stormStreamer.setIgniteConfigFile("/Users/shamim/Development/workshop/assembla/ignite-book/chapters/chapter-cep-storm/src/main/resources/example-ignite.xml");

    //Used to build the topology
    TopologyBuilder builder = new TopologyBuilder();
    //Add the spout, with a name of 'spout'
    //and parallelism hint of 5 executors
    builder.setSpout("spout", new RandomSentenceSpout(), 5);
    //Add the SplitSentence bolt, with a name of 'split'
    //and parallelism hint of 8 executors
    //shufflegrouping subscribes to the spout, and equally distributes
    //tuples (sentences) across instances of the SplitSentence bolt
    builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
    //Add the counter, with a name of 'count'
    //and parallelism hint of 12 executors
    //fieldsgrouping subscribes to the split bolt, and
    //ensures that the same word is sent to the same instance (group by field 'word')
    builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));
    // set ignite bolt
    builder.setBolt("ignite-bolt", stormStreamer,STORM_EXECUTORS).shuffleGrouping("count");

    //new configuration
    Config conf = new Config();
    //Set to false to disable debug information
    // when running in production mode.
    conf.setDebug(false);

    //If there are arguments, we are running on a cluster
    if (args != null && args.length > 0) {
        //parallelism hint to set the number of workers
        conf.setNumWorkers(3);
        //submit the topology
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    }
    //Otherwise, we are running locally
    else {
        //Cap the maximum number of executors that can be spawned
        //for a component to 3
        conf.setMaxTaskParallelism(3);
        //LocalCluster is used to run locally
        LocalCluster cluster = new LocalCluster();
        //submit the topology
        cluster.submitTopology("word-count", conf, builder.createTopology());
        //sleep
        Thread.sleep(10000);
        //shut down the cluster
        cluster.shutdown();
    }
}
 
Example 28
Project: java   File: TradeProcessingTopology.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
  TopologyBuilder builder = new TopologyBuilder();

  LOGGER.info("Building Trade Processing Topology..");

  builder.setSpout(TRD_COLLECTOR_SPOUT, new TradeCollectorSpout(),
      CONFIG.getNumber("TRD_COLLECTOR_SPOUT_PARALLELISM"));

  builder
      .setBolt(TRD_ELIGIBILITY_BOLT, new TradeEligibilityBolt(),
          CONFIG.getNumber("TRD_ELIGIBILITY_BOLT_PARALLELISM"))
      .shuffleGrouping(TRD_COLLECTOR_SPOUT);

  builder
      .setBolt(TRD_REPORTING_BOLT, new TradeReportPersistenceBolt(),
          CONFIG.getNumber("TRD_REPORTING_BOLT_PARALLELISM"))
      .shuffleGrouping(TRD_ELIGIBILITY_BOLT, REPORT_STREAM);

  builder
      .setBolt(TRD_EXCLUSION_BOLT, new TradeExclusionPersistenceBolt(),
          CONFIG.getNumber("TRD_EXCLUSION_BOLT_PARALLELISM"))
      .shuffleGrouping(TRD_ELIGIBILITY_BOLT, EXCLUDE_STREAM);

  Config conf = new Config();
  conf.setDebug(CONFIG.is("DEBUG_FLAG"));
  conf.setNumWorkers(CONFIG.getInt("NUMBER_OF_WORKERS"));
  conf.setMaxTaskParallelism(CONFIG.getInt("MAX_TASK_PARALLELISM"));
  conf.setMaxSpoutPending(CONFIG.getInt("MAX_SPOUT_PENDING"));
  conf.put(Config.TOPOLOGY_SLEEP_SPOUT_WAIT_STRATEGY_TIME_MS,
      CONFIG.getInt("MAX_SPOUT_PENDING_WAIT_MS"));
  conf.put(Config.TOPOLOGY_SPOUT_WAIT_STRATEGY, CONFIG.get("TOPOLOGY_WAIT_STRATEGY"));
  conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, CONFIG.getInt("TOPOLOGY_MESSAGE_TIMEOUT_SECS"));
  conf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS,
      CONFIG.is("TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS"));
  LOGGER.info("Submitting Trade Processing Topology..");
  if (args != null && args.length > 0) {
    conf.put(Config.NIMBUS_HOST, CONFIG.get("LOCAL_NIMBUS_HOST"));
    conf.put(Config.NIMBUS_THRIFT_PORT, CONFIG.getInt("LOCAL_NIMBUS_PORT"));
    conf.put(Config.STORM_ZOOKEEPER_PORT, CONFIG.getInt("LOCAL_ZOOKEEPER_PORT"));
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
  } else {
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology(TRD_PROCESSING_TOPOLOGY, conf, builder.createTopology());
    Utils.sleep(CONFIG.getLong("LOCAL_CLUSTER_RUNTIME"));
    cluster.killTopology(TRD_PROCESSING_TOPOLOGY);
    cluster.shutdown();
  }
}
 
Example 29
Project: storm-kafka-examples   File: HdfsTopology.java   Source Code and License Vote up 4 votes
public static void main(String[] args) {
    try{
        String zkhost = "wxb-1:2181,wxb-2:2181,wxb-3:2181";
        String topic = "order";
        String groupId = "id";
        int spoutNum = 3;
        int boltNum = 1;
        ZkHosts zkHosts = new ZkHosts(zkhost);//kafaka所在的zookeeper
        SpoutConfig spoutConfig = new SpoutConfig(zkHosts, topic, "/order", groupId);  // create /order /id
        spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
        KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);

        // HDFS bolt
        // use "|" instead of "," for field delimiter
        RecordFormat format = new DelimitedRecordFormat()
                .withFieldDelimiter("|");

        // sync the filesystem after every 1k tuples
        SyncPolicy syncPolicy = new CountSyncPolicy(1000);

        // rotate files when they reach 5MB
        FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
        // FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);

        FileNameFormat fileNameFormat = new DefaultFileNameFormat()
                .withPath("/tmp/").withPrefix("order_").withExtension(".log");

        HdfsBolt hdfsBolt = new HdfsBolt()
                .withFsUrl("hdfs://wxb-1:8020")
                .withFileNameFormat(fileNameFormat)
                .withRecordFormat(format)
                .withRotationPolicy(rotationPolicy)
                .withSyncPolicy(syncPolicy);

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("spout", kafkaSpout, spoutNum);
        builder.setBolt("check", new CheckOrderBolt(), boltNum).shuffleGrouping("spout");
        builder.setBolt("counter", new CounterBolt(),boltNum).shuffleGrouping("check");
        builder.setBolt("hdfs", hdfsBolt,boltNum).shuffleGrouping("counter");

        Config config = new Config();
        config.setDebug(true);

        if(args!=null && args.length > 0) {
            config.setNumWorkers(2);
            StormSubmitter.submitTopology(args[0], config, builder.createTopology());
        } else {
            config.setMaxTaskParallelism(2);

            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("Wordcount-Topology", config, builder.createTopology());

            Thread.sleep(500000);

            cluster.shutdown();
        }
    }catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 30
Project: hadooparchitecturebook   File: MovingAvgLocalTopologyRunner.java   Source Code and License Vote up 4 votes
/**
 * Return the object creating our moving average topology.
 */
private static StormTopology buildTopology() {

  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("stock-ticks-spout", new StockTicksSpout());

  builder.setBolt("hdfs-persister-bolt", createHdfsBolt())
    .shuffleGrouping("stock-ticks-spout");

  builder.setBolt("parse-ticks", new ParseTicksBolt())
    .shuffleGrouping("stock-ticks-spout");

  builder.setBolt("calc-moving-avg", new CalcMovingAvgBolt(), 2)
    .fieldsGrouping("parse-ticks", new Fields("ticker"));

  return builder.createTopology();
}
 
Example 31
Project: streaming_outliers   File: Topology.java   Source Code and License Vote up 4 votes
public static TopologyBuilder createTopology( DataPointExtractorConfig extractorConfig
                                            , com.caseystella.analytics.outlier.streaming.OutlierConfig streamingOutlierConfig
                                            , PersistenceConfig persistenceConfig
                                            , String kafkaTopic
                                            , String zkQuorum
                                            , String esNode
                                            , int numWorkers
                                            , int numSpouts
                                            , int numIndexers
                                            , String indexName
                                            , boolean startAtBeginning
                                     )
{
    TopologyBuilder builder = new TopologyBuilder();
    String spoutId = "outlier_filter";
    String boltId= "outlier";
    OutlierKafkaSpout spout = null;
    {
        //BrokerHosts hosts, String topic, String zkRoot, String id
        SpoutConfig spoutConfig = new SpoutConfig(new ZkHosts(zkQuorum)
                                                 , kafkaTopic
                                                 , "/kafka"
                                                 , "streaming_outlier"
                                                 );
        spoutConfig.startOffsetTime = startAtBeginning?kafka.api.OffsetRequest.EarliestTime()
                                                      :kafka.api.OffsetRequest.LatestTime()
                                                      ;
        if(startAtBeginning) {
            spoutConfig.ignoreZkOffsets = true;
        }
        spout = new OutlierKafkaSpout(spoutConfig
                                     , extractorConfig
                                     , streamingOutlierConfig.getGroupingKeys()
                                     , zkQuorum
                                     );
    }
    OutlierBolt bolt = null;
    {
        bolt = new OutlierBolt(kafkaTopic, streamingOutlierConfig, persistenceConfig);
    }
    builder.setSpout(spoutId, spout, numSpouts);
    builder.setBolt(boltId, bolt, numWorkers).fieldsGrouping(spoutId, new Fields(Constants.GROUP_ID));
    {
        Map conf = new HashMap();
        if(esNode != null) {
            conf.put(ConfigurationOptions.ES_NODES, esNode);
        }
        conf.put(ConfigurationOptions.ES_BATCH_SIZE_ENTRIES, "1");
        conf.put(ConfigurationOptions.ES_INDEX_AUTO_CREATE, true);
        builder.setBolt("es_bolt", new EsBolt(indexName, conf), numIndexers)
               .shuffleGrouping(boltId, OutlierBolt.STREAM_ID);
    }
    return builder;
}
 
Example 32
Project: Get-ENVS   File: WordCountTopology.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws Exception {
//Used to build the topology
  TopologyBuilder builder = new TopologyBuilder();
  //Add the spout, with a name of 'spout'
  //and parallelism hint of 5 executors
  builder.setSpout("spout", new RandomSentenceSpout(), 5);
  //Add the SplitSentence bolt, with a name of 'split'
  //and parallelism hint of 8 executors
  //shufflegrouping subscribes to the spout, and equally distributes
  //tuples (sentences) across instances of the SplitSentence bolt
  builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
  //Add the counter, with a name of 'count'
  //and parallelism hint of 12 executors
  //fieldsgrouping subscribes to the split bolt, and
  //ensures that the same word is sent to the same instance (group by field 'word')
  builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));

  //new configuration
  Config conf = new Config();
  conf.setDebug(true);

  //If there are arguments, we are running on a cluster
  if (args != null && args.length > 0) {
    //parallelism hint to set the number of workers
    conf.setNumWorkers(3);
    //submit the topology
    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
  }
  //Otherwise, we are running locally
  else {
    //Cap the maximum number of executors that can be spawned
    //for a component to 3
    conf.setMaxTaskParallelism(3);
    //LocalCluster is used to run locally
    LocalCluster cluster = new LocalCluster();
    //submit the topology
    cluster.submitTopology("word-count", conf, builder.createTopology());
    //sleep
    Thread.sleep(10000);
    //shut down the cluster
    cluster.shutdown();
  }
}
 
Example 33
Project: ignite-book-code-samples   File: SpeedViolationTopology.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws Exception {
    if (getProperties() == null || getProperties().isEmpty()) {
        System.out.println("Property file <ignite-storm.property> is not found or empty");
        return;
    }
    // Ignite Stream Ibolt
    final StormStreamer<String, String> stormStreamer = new StormStreamer<>();

    stormStreamer.setAutoFlushFrequency(10L);
    stormStreamer.setAllowOverwrite(true);
    stormStreamer.setCacheName(getProperties().getProperty("cache.name"));

    stormStreamer.setIgniteTupleField(getProperties().getProperty("tuple.name"));
    stormStreamer.setIgniteConfigFile(getProperties().getProperty("ignite.spring.xml"));


    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new FileSourceSpout(), 1);
    builder.setBolt("limit", new SpeedLimitBolt(), 1).fieldsGrouping("spout", new Fields("trafficLog"));
    // set ignite bolt
    builder.setBolt("ignite-bolt", stormStreamer, STORM_EXECUTORS).shuffleGrouping("limit");

    Config conf = new Config();
    conf.setDebug(false);

    conf.setMaxTaskParallelism(1);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("speed-violation", conf, builder.createTopology());
    Thread.sleep(10000);
    cluster.shutdown();

}
 
Example 34
Project: LearnStorm   File: PersistentWordCount.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws Exception {
    Config config = new Config();

    Map<String, Object> hbConf = new HashMap<String, Object>();
    if(args.length > 0){
        hbConf.put("hbase.rootdir", args[0]);
    }
    config.put("hbase.conf", hbConf);

    WordSpout spout = new WordSpout();
    WordCounter bolt = new WordCounter();

    SimpleHBaseMapper mapper = new SimpleHBaseMapper()
            .withRowKeyField("word")
            .withColumnFields(new Fields("word"))
            .withCounterFields(new Fields("count"))
            .withColumnFamily("cf");

    HBaseBolt hbase = new HBaseBolt("WordCount", mapper)
            .withConfigKey("hbase.conf");


    // wordSpout ==> countBolt ==> HBaseBolt
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout(WORD_SPOUT, spout, 1);
    builder.setBolt(COUNT_BOLT, bolt, 1).shuffleGrouping(WORD_SPOUT);
    builder.setBolt(HBASE_BOLT, hbase, 1).fieldsGrouping(COUNT_BOLT, new Fields("word"));


    if (args.length == 0) {
    	System.setProperty("hadoop.home.dir", "/tmp");
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", config, builder.createTopology());
        Thread.sleep(30000);
        cluster.killTopology("test");
        cluster.shutdown();
        System.exit(0);
    } else if (args.length == 2) {
        StormSubmitter.submitTopology(args[1], config, builder.createTopology());
    } else{
        System.out.println("Usage: HdfsFileTopology <hdfs url> [topology name]");
    }
}
 
Example 35
Project: storm-lib   File: FileParseTopology.java   Source Code and License Vote up 4 votes
void run(String[] args) {
TopologyBuilder builder = new TopologyBuilder();

// Build the record template for a given SiLK record
RecordTemplate SilkRecord = 
    (new RecordTemplate())
    .readBytes(8, "ts")
    .readBytes(4, "dur")
    .readBytes(2, "sport")
    .readBytes(2, "dport")
    .readBytes(1, "proto")
    .readBytes(1, "ct")
    .readBytes(2, "sensor")
    .readBytes(1, "flags")
    .readBytes(1, "init_flags")
    .readBytes(1, "sess_flags")
    .readBytes(1, "attr")
    .readBytes(2, "appl")
    .skipBytes(2)
    .readBytes(2, "snmp_in")
    .readBytes(2, "snmp_out")
    .readBytes(4, "packets")
    .readBytes(4, "bytes")
    .readBytes(4, "sip")
    .readBytes(4, "dip")
    .readBytes(4, "nhip");

builder.setSpout("poll_spout", new PollDirectory("/tmp/silk/dir", "/tmp/silk/dir/proc"), 1);
builder.setBolt("silk_atomize", new Parse(SilkRecord), 2).shuffleGrouping("poll_spout");
	
Map conf = new HashMap();
conf.put(Config.TOPOLOGY_WORKERS, 4);
conf.put(Config.TOPOLOGY_DEBUG, true);
	
if(args.length==0) {
           LocalCluster cluster = new LocalCluster();
           cluster.submitTopology("MockIngest", conf, builder.createTopology());
       } else {
    try {
	StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } catch(Exception e) {
	e.printStackTrace();
    }
}
   }
 
Example 36
Project: aeolus   File: BatchingFieldsGroupingMultipleITCase.java   Source Code and License Vote up 4 votes
@SuppressWarnings("rawtypes")
@Test(timeout = 30000)
public void testFieldsGroupingMultiple() {
	final String topologyName = "testTopology";
	final int maxValue = 1000;
	final int batchSize = 1 + this.r.nextInt(5);
	final int numberOfAttributes = 1 + this.r.nextInt(10);
	final int numberOfConsumers = 2 + this.r.nextInt(3);
	final Integer spoutDop = new Integer(1);
	final Integer[] boltDop = new Integer[numberOfConsumers];
	final Fields[] groupingFiels = new Fields[numberOfConsumers];
	final String[][] schema = new String[numberOfConsumers][];
	
	for(int i = 0; i < numberOfConsumers; ++i) {
		boltDop[i] = new Integer(1 + this.r.nextInt(5));
		
		LinkedList<String> attributes = new LinkedList<String>();
		for(int j = 0; j < numberOfAttributes; ++j) {
			attributes.add("" + (char)('a' + j));
		}
		
		schema[i] = new String[1 + this.r.nextInt(numberOfAttributes)];
		for(int j = 0; j < schema[i].length; ++j) {
			schema[i][j] = new String(attributes.remove(this.r.nextInt(attributes.size())));
		}
		groupingFiels[i] = new Fields(schema[i]);
	}
	
	
	
	LocalCluster cluster = new LocalCluster();
	TopologyBuilder builder = new TopologyBuilder();
	
	builder.setSpout(VerifyBolt.SPOUT_ID, new RandomSpout(numberOfAttributes, maxValue, new String[] {"stream1"},
		this.seed), spoutDop);
	builder.setSpout(VerifyBolt.BATCHING_SPOUT_ID, new SpoutOutputBatcher(new RandomSpout(numberOfAttributes,
		maxValue, new String[] {"stream2"}, this.seed), batchSize), spoutDop);
	
	for(int i = 0; i < numberOfConsumers; ++i) {
		builder
			.setBolt("Bolt-" + i, new InputDebatcher(new VerifyBolt(new Fields(schema[i]), groupingFiels[i])),
				boltDop[i]).fieldsGrouping(VerifyBolt.SPOUT_ID, "stream1", groupingFiels[i])
			.fieldsGrouping(VerifyBolt.BATCHING_SPOUT_ID, "stream2", groupingFiels[i])
			.directGrouping(VerifyBolt.BATCHING_SPOUT_ID, BatchingOutputFieldsDeclarer.STREAM_PREFIX + "stream2");
		
	}
	
	cluster.submitTopology(topologyName, new HashMap(), builder.createTopology());
	
	Utils.sleep(10 * 1000);
	cluster.killTopology(topologyName);
	Utils.sleep(5 * 1000); // give "kill" some time to clean up; otherwise, test might hang and time out
	cluster.shutdown();
	
	Assert.assertEquals(new LinkedList<String>(), VerifyBolt.errorMessages);
	Assert.assertTrue(VerifyBolt.matchedTuples > 0);
}
 
Example 37
Project: jstrom   File: SequenceTopologyTool.java   Source Code and License Vote up 4 votes
public StormTopology buildTopology()
{
	Config conf = getConf();
	TopologyBuilder builder = new TopologyBuilder();

	int spout_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
	int bolt_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

	builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
			new SequenceSpout(), spout_Parallelism_hint);

	boolean isEnableSplit = JStormUtils.parseBoolean(
			conf.get("enable.split"), false);

	if (isEnableSplit == false) {
		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).localFirstGrouping(
				SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
	} else {

		builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
				new SplitRecord(), bolt_Parallelism_hint)
				.localOrShuffleGrouping(
						SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

		builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.TRADE_STREAM_ID);
		builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.CUSTOMER_STREAM_ID);

		builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
				new MergeRecord(), bolt_Parallelism_hint)
				.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
						new Fields("ID"))
				.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
						new Fields("ID"));

		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).noneGrouping(
				SequenceTopologyDef.MERGE_BOLT_NAME);
	}

	boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
			false);
	if (kryoEnable == true) {
		System.out.println("Use Kryo ");
		boolean useJavaSer = JStormUtils.parseBoolean(
				conf.get("fall.back.on.java.serialization"), true);

		Config.setFallBackOnJavaSerialization(conf, useJavaSer);

		Config.registerSerialization(conf, TradeCustomer.class);
		Config.registerSerialization(conf, Pair.class);
	}
	int ackerNum = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
	Config.setNumAckers(conf, ackerNum);

	int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
			20);
	conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	return  builder.createTopology();	
}
 
Example 38
Project: big-data-system   File: Q2Stream.java   Source Code and License Vote up 4 votes
public static void main(String[] args) {
        String consumerKey = "42NRnxnkuqrghnolDWSqbiFyv"; 
        String consumerSecret = "zmwC0g6z1FOBBQigW8w2lrnLYncuH4p3QX25RUCNa8aU1QSCC5"; 
        String accessToken = "2809571326-fyBz1ITFXf4yjuqZvHKgGyy0QcQfNVr8y2OGYq6"; 
        String accessTokenSecret = "MAnEtUccHXheXf0z2pauV75oj2XOm6ag4hiLvbUOh6n6B";
        int interval = 2000;
        
        String[] keyWords = new String[]{"cat", "car", "Stockholm", "snow", "data", "system", "Trump", "Palantir", "blue", "badger",
        		"NFL", "apple", "google", "facebook", "perks", "spg", "cajun", "banana", "taco", "whatever", "weareone", "packers", "green",
        		"NBA", "mlb", "dog", "kitten", "blueberry", "romance", "princess", "phone", "nuts", "sheldon", "mad", "talk", "nasty",
        		"procrastination", "cook", "college", "patriots", "dumnass", "dough", "winter", "game", "thrones", "halloween", "warcraft",
        		"hiking", "intern", "park", "sweater", "epic", "dota", "year", "wrath", "waste", "Blake", "street", "toyota", "arrow", 
        		"warning", "travel", "flight", "reject", "karaoke", "bless", "empire", "survivor", "bank", "dating", "restaurant", "tinder",
        		"shopping", "win", "cold", "recap", "cop", "astronaut", "crime", "book", "http", "injured", "china", "awards", "join", 
        		"ugly", "birthday", "friend", "weather", "shirt", "student", "mail", "sleep", "pet", "sea", "dream", "chritmas", "thanksgiving",
        		"vacation", "california", "church", "love", "fuck", "vote", "election", "bernie", "parade", "disney", "today", "city",
        		"marathon", "trade", "cash", "miles", "fun", "work", "free", "photo", "hard", "water", "god", "speech", "gang", "bear", 
        		"stop", "luck", "vegas", "shame", "food", "fool", "weight", "football", "tennis", "concert", "cancer", "stock", "crazy",
        		"ticket", "play", "project", "russia", "cast", "star", "trailer", "yelp", "video", "hawaii", "law", "rage", "comic", "meme",
        		"swag", "snack", "drink", "alcohol", "peace", "swift", "maroon", "xbox", "surface", "flower", "sport", "music", "traffic", "family",
        		"autograph", "selfie", "fans"}; 
        
        TopologyBuilder builder = new TopologyBuilder();
        
        builder.setSpout("friendsCount", new Q2RandomFriendsCountSpout(interval));
        builder.setSpout("hashtags", new Q2RandomHashtagSpout(interval, 100));
        builder.setSpout("tweets", new Q2SeqTwitterSpout(consumerKey, consumerSecret, accessToken, accessTokenSecret, keyWords));
        builder.setBolt("join1", new SingleJoinBolt(new Fields("seq", "friendsCount", "hashtags")))
        	.fieldsGrouping("friendsCount", new Fields("seq"))
        	.fieldsGrouping("hashtags", new Fields("seq"));
        builder.setBolt("join", new SingleJoinBolt(new Fields("friendsCount", "hashtags", "tweetinfolist")))
        	.fieldsGrouping("join1", new Fields("seq"))
        	.fieldsGrouping("tweets", new Fields("seq"));
        builder.setBolt("filter", new Q2FilterTweetBolt())
        	.shuffleGrouping("join");
//        builder.setBolt("printFiltered", new Q2PrintTweetBolt())
//    		.shuffleGrouping("filter");
//        builder.setBolt("cleanWord", new CleanWordsBolt())
//        	.shuffleGrouping("filter", "filterStream");
        builder.setBolt("splitWord", new SplitWordBolt())
        	.shuffleGrouping("filter");
        builder.setBolt("wordCount", new WordCountBolt(), 12)
        	.fieldsGrouping("splitWord", new Fields("word"));
        builder.setBolt("intermediateRanking", new IntermediateRankingsBolt())
        	.shuffleGrouping("wordCount");
        builder.setBolt("totalRanking",  new TotalRankingsBolt())
        	.globalGrouping("intermediateRanking");
        builder.setBolt("printFinal", new Q2PrintTweetBolt())
        	.shuffleGrouping("totalRanking");

                
        Config conf = new Config();
        conf.setDebug(true);
        
        final LocalCluster cluster = new LocalCluster();
        StormTopology topo = builder.createTopology();
        cluster.submitTopology("Q2", conf, topo);
        
        //Utils.sleep(interval * 100);
		Runtime.getRuntime().addShutdownHook(new Thread() {
			@Override
			public void run() {
				cluster.killTopology("Q2");
				cluster.shutdown();
			}
		});
        //cluster.shutdown();
    }
 
Example 39
Project: jstrom   File: BatchTopologyBuilder.java   Source Code and License Vote up 4 votes
public BatchTopologyBuilder(String topologyName) {
    topologyBuilder = new TopologyBuilder();

    spoutDeclarer = topologyBuilder.setSpout(BatchDef.SPOUT_TRIGGER, new BatchSpoutTrigger(), 1);
}
 
Example 40
Project: streaming-benchmarks   File: AdvertisingTopology.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    Options opts = new Options();
    opts.addOption("conf", true, "Path to the config file.");

    CommandLineParser parser = new DefaultParser();
    CommandLine cmd = parser.parse(opts, args);
    String configPath = cmd.getOptionValue("conf");
    Map commonConfig = Utils.findAndReadConfigFile(configPath, true);

    String zkServerHosts = joinHosts((List<String>)commonConfig.get("zookeeper.servers"),
                                     Integer.toString((Integer)commonConfig.get("zookeeper.port")));
    String redisServerHost = (String)commonConfig.get("redis.host");
    String kafkaTopic = (String)commonConfig.get("kafka.topic");
    int kafkaPartitions = ((Number)commonConfig.get("kafka.partitions")).intValue();
    int workers = ((Number)commonConfig.get("storm.workers")).intValue();
    int ackers = ((Number)commonConfig.get("storm.ackers")).intValue();
    int cores = ((Number)commonConfig.get("process.cores")).intValue();
    int parallel = Math.max(1, cores/7);

    ZkHosts hosts = new ZkHosts(zkServerHosts);



    SpoutConfig spoutConfig = new SpoutConfig(hosts, kafkaTopic, "/" + kafkaTopic, UUID.randomUUID().toString());
    spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
    KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);

    builder.setSpout("ads", kafkaSpout, kafkaPartitions);
    builder.setBolt("event_deserializer", new DeserializeBolt(), parallel).shuffleGrouping("ads");
    builder.setBolt("event_filter", new EventFilterBolt(), parallel).shuffleGrouping("event_deserializer");
    builder.setBolt("event_projection", new EventProjectionBolt(), parallel).shuffleGrouping("event_filter");
    builder.setBolt("redis_join", new RedisJoinBolt(redisServerHost), parallel).shuffleGrouping("event_projection");
    builder.setBolt("campaign_processor", new CampaignProcessor(redisServerHost), parallel*2)
        .fieldsGrouping("redis_join", new Fields("campaign_id"));

    Config conf = new Config();

    if (args != null && args.length > 0) {
        conf.setNumWorkers(workers);
        conf.setNumAckers(ackers);
        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    }
    else {

        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("test", conf, builder.createTopology());
        backtype.storm.utils.Utils.sleep(10000);
        cluster.killTopology("test");
        cluster.shutdown();
    }
}
 
Example 41
Project: jstrom   File: TestTopology.java   Source Code and License Vote up 3 votes
private static TopologyBuilder setupBuilder() throws Exception {
	TopologyBuilder builder = new TopologyBuilder();

	int writerParallel = JStormUtils.parseInt(conf.get("topology.producer.parallel"), 1);

	int spoutParallel = JStormUtils.parseInt(conf.get("topology.consumer.parallel"), 1);

	builder.setSpout("OnsConsumer", new ConsumerSpout(), spoutParallel);

	builder.setBolt("OnsProducer", new ProducerBolt(), writerParallel).localFirstGrouping("OnsConsumer");

	return builder;
}
 
Example 42
Project: cdh-storm   File: WordCountTopology.java   Source Code and License Vote up 3 votes
public static void main(String[] args) throws Exception {

    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new RandomSentenceSpout(), 5);

    builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));

    Config conf = new Config();
    conf.setDebug(true);


    if (args != null && args.length > 0) {
      conf.setNumWorkers(3);

      StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    }
    else {
      conf.setMaxTaskParallelism(3);

      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("word-count", conf, builder.createTopology());

      Thread.sleep(10000);

      cluster.shutdown();
    }
  }
 
Example 43
Project: cloud-computing-specialization   File: TopWordFinderTopologyPartD.java   Source Code and License Vote up 2 votes
public static void main(String[] args) throws Exception {


        TopologyBuilder builder = new TopologyBuilder();

        Config config = new Config();
        config.setDebug(true);


    /*
    ----------------------TODO-----------------------
    Task: wire up the topology

    NOTE:make sure when connecting components together, using the functions setBolt(name,…) and setSpout(name,…),
    you use the following names for each component:
    
    FileReaderSpout -> "spout"
    SplitSentenceBolt -> "split"
    WordCountBolt -> "count"
	NormalizerBolt -> "normalize"
    TopNFinderBolt -> "top-n"


    ------------------------------------------------- */

        String spoutId = "spout";
        String splitId = "split";
        String countId = "count";
        String normalizeId = "normalize";
        String topn = "top-n";

        builder.setSpout(spoutId, new FileReaderSpout(args[0]), 5);
        builder.setBolt(splitId, new SplitSentenceBolt(), 8).shuffleGrouping(spoutId);
        builder.setBolt(normalizeId, new NormalizerBolt(), 12).fieldsGrouping(splitId, new Fields("word"));
        builder.setBolt(countId, new WordCountBolt(), 12).fieldsGrouping(normalizeId, new Fields("word"));
        builder.setBolt(topn, new TopNFinderBolt(N), 1).fieldsGrouping(countId, new Fields("word", "count"));


        config.setMaxTaskParallelism(3);

        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("word-count", config, builder.createTopology());

        //wait for 2 minutes and then kill the job
        Thread.sleep(2 * 60 * 1000);

        cluster.shutdown();
    }