backtype.storm.topology.TopologyBuilder Java Examples

The following examples show how to use backtype.storm.topology.TopologyBuilder. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TransactionTopologyBuilder.java    From jstorm with Apache License 2.0 6 votes vote down vote up
@Override
public BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelismHint) throws IllegalArgumentException{
    upToDownstreamComponentsMap.put(id, new HashSet<String>());
    validateUnusedId(id);
    IRichBolt boltExecutor;
    boolean isStatefulBolt = false;
    if (bolt instanceof ITransactionStatefulBoltExecutor) {
        isStatefulBolt = true;
        boltExecutor = new TransactionStatefulBolt((ITransactionStatefulBoltExecutor) bolt);
    } else {
        boltExecutor = new TransactionBolt((ITransactionBoltExecutor) bolt);
    }
    initCommon(id, boltExecutor, parallelismHint);
    _bolts.put(id, boltExecutor);
    BoltDeclarer ret = new TransactionBoltDeclarer(id);
    ret.addConfiguration(TransactionCommon.TRANSACTION_STATEFUL_BOLT, isStatefulBolt);

    // If using KvState bolt, the corresponding init operater would be registered here.
    if (bolt instanceof KvStatefulBoltExecutor) {
        ConfigExtension.registerTransactionTaskStateInitOp(TopologyBuilder.getStormConf(), id, KeyRangeStateTaskInit.class);
    }

    return ret;
}
 
Example #2
Source File: SlidingWindowTopology.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void test() {
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    
    try {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("integer", new RandomIntegerSpout(), 1);
        builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(new Count(30), new Count(10)), 1)
                .shuffleGrouping("integer");
        builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(new Count(3)), 1)
                .shuffleGrouping("slidingsum");
        builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg");
        
        conf.setDebug(true);
        
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60,
                new JStormHelper.CheckAckedFail(conf), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.fillInStackTrace();
        Assert.fail("Failed to submit topology");
    }
}
 
Example #3
Source File: PerformanceTestTopology.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void SetRemoteTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 
Example #4
Source File: TestTopology.java    From jstorm with Apache License 2.0 6 votes vote down vote up
private static void submitTopology(TopologyBuilder builder) {
	try {
		if (local_mode(conf)) {

			LocalCluster cluster = new LocalCluster();

			cluster.submitTopology(
					String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());

			Thread.sleep(200000);

			cluster.shutdown();
		} else {
			StormSubmitter.submitTopology(
					String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());
		}

	} catch (Exception e) {
		LOG.error(e.getMessage(), e.getCause());
	}
}
 
Example #5
Source File: SlidingTupleTsTopology.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void test() {
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    
    try {
        TopologyBuilder builder = new TopologyBuilder();
        BaseWindowedBolt bolt = new SlidingWindowSumBolt()
                .withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS))
                .withTimestampField("ts").withLag(new Duration(5, TimeUnit.SECONDS));
        builder.setSpout("integer", new RandomIntegerSpout(), 1);
        builder.setBolt("slidingsum", bolt, 1).shuffleGrouping("integer");
        builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("slidingsum");
        
        conf.setDebug(true);
        
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60,
                new JStormHelper.CheckAckedFail(conf), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}
 
Example #6
Source File: TridentMinMaxOfDevicesTopology.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void test() {
    TopologyBuilder builder = new TopologyBuilder();
    
    builder.setSpout("spout", new InOrderSpout(), 8);
    builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1"));
    
    conf.setMaxSpoutPending(20);
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    
    if (isLocal) {
        drpc = new LocalDRPC();
    }
    
    try {
        JStormHelper.runTopology(buildDevicesTopology(), topologyName, conf, 60,
                new JStormHelper.CheckAckedFail(conf), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}
 
Example #7
Source File: FastWordCountTest.java    From jstorm with Apache License 2.0 6 votes vote down vote up
@Test
public void testFastWordCount()
{
    int spout_Parallelism_hint = 1;
    int split_Parallelism_hint = 1;
    int count_Parallelism_hint = 2;

    TopologyBuilder builder = new TopologyBuilder();

    boolean isLocalShuffle = false;

    builder.setSpout("spout", new FastWordCountTopology.FastRandomSentenceSpout(), spout_Parallelism_hint);
    if (isLocalShuffle)
        builder.setBolt("split", new FastWordCountTopology.SplitSentence(), split_Parallelism_hint).localFirstGrouping("spout");
    else
        builder.setBolt("split", new FastWordCountTopology.SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout");
    builder.setBolt("count", new FastWordCountTopology.WordCount(), count_Parallelism_hint).fieldsGrouping("split", new Fields("word"));

    Map config = new HashMap();
    config.put(Config.TOPOLOGY_NAME, "FastWordCountTest");

    JStormUnitTestRunner.submitTopology(builder.createTopology(), config, 60, null);
}
 
Example #8
Source File: FastWordCountSessionEventTimeWindowTopology.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void test() throws Exception {
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1);

    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint);

    WordCount wordCountBolt = new WordCount();
    builder.setBolt("count", wordCountBolt.sessionEventTimeWindow(Time.milliseconds(3L))
            .withTimestampExtractor(wordCountBolt)
            .withWindowStateMerger(wordCountBolt), count_Parallelism_hint)
            .fieldsGrouping("spout", new Fields("word", "ts"));

    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];

    JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60,
            new JStormHelper.CheckAckedFail(conf), true);
}
 
Example #9
Source File: OozieAuditLogApplication.java    From eagle with Apache License 2.0 6 votes vote down vote up
@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfParserTask = config.getInt(PARSER_TASK_NUM);
    int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);

    OozieAuditLogParserBolt parserBolt = new OozieAuditLogParserBolt();
    BoltDeclarer parserBoltDeclarer = builder.setBolt("parserBolt", parserBolt, numOfParserTask);
    parserBoltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));

    OozieResourceSensitivityDataJoinBolt joinBolt = new OozieResourceSensitivityDataJoinBolt(config);
    BoltDeclarer boltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
    boltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));

    StormStreamSink sinkBolt = environment.getStreamSink("oozie_audit_log_stream", config);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
    kafkaBoltDeclarer.fieldsGrouping("joinBolt", new Fields("user"));
    return builder.createTopology();
}
 
Example #10
Source File: FastWordCountSessionProcessingTimeWindowTopology.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void test() {
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1);

    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint);

    WordCount wordCountBolt = new WordCount();
    builder.setBolt("count", wordCountBolt.sessionTimeWindow(Time.seconds(1L))
            .withWindowStateMerger(wordCountBolt), count_Parallelism_hint)
            .fieldsGrouping("spout", new Fields("word"));
    //.allGrouping("spout", Common.WATERMARK_STREAM_ID);

    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];

    try {
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60,
                new JStormHelper.CheckAckedFail(conf), true);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example #11
Source File: SparkHistoryJobApp.java    From eagle with Apache License 2.0 6 votes vote down vote up
@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    // 1. Init conf
    SparkHistoryJobAppConfig sparkHistoryJobAppConfig = SparkHistoryJobAppConfig.newInstance(config);

    final String jobFetchSpoutName = SparkHistoryJobAppConfig.SPARK_HISTORY_JOB_FETCH_SPOUT_NAME;
    final String jobParseBoltName = SparkHistoryJobAppConfig.SPARK_HISTORY_JOB_PARSE_BOLT_NAME;

    // 2. Config topology.
    TopologyBuilder topologyBuilder = new TopologyBuilder();

    topologyBuilder.setSpout(
            jobFetchSpoutName,
            new SparkHistoryJobSpout(sparkHistoryJobAppConfig), sparkHistoryJobAppConfig.stormConfig.numOfSpoutExecutors
    ).setNumTasks(sparkHistoryJobAppConfig.stormConfig.numOfSpoutTasks);

    topologyBuilder.setBolt(
            jobParseBoltName,
            new SparkHistoryJobParseBolt(sparkHistoryJobAppConfig),
            sparkHistoryJobAppConfig.stormConfig.numOfParserBoltExecutors
    ).setNumTasks(sparkHistoryJobAppConfig.stormConfig.numOfParserBoltTasks).shuffleGrouping(jobFetchSpoutName);

    return topologyBuilder.createTopology();
}
 
Example #12
Source File: TridentMapExample.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void test() {
    TopologyBuilder builder = new TopologyBuilder();
    
    builder.setSpout("spout", new InOrderSpout(), 8);
    builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1"));
    
    Config conf = new Config();
    conf.setMaxSpoutPending(20);
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    
    if (isLocal) {
        drpc = new LocalDRPC();
    }
    
    try {
        JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new DrpcValidator(), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}
 
Example #13
Source File: EventsimTopology.java    From storm-solr with Apache License 2.0 6 votes vote down vote up
public StormTopology build(StreamingApp app) throws Exception {
  SpringSpout eventsimSpout = new SpringSpout("eventsimSpout", spoutFields);
  SpringBolt collectionPerTimeFrameSolrBolt = new SpringBolt("collectionPerTimeFrameSolrBoltAction",
      app.tickRate("collectionPerTimeFrameSolrBoltAction"));

  // Send all docs for the same hash range to the same bolt instance,
  // which allows us to use a streaming approach to send docs to the leader
  int numShards = Integer.parseInt(String.valueOf(app.getStormConfig().get("spring.eventsimNumShards")));
  HashRangeGrouping hashRangeGrouping = new HashRangeGrouping(app.getStormConfig(), numShards);
  int tasksPerShard = hashRangeGrouping.getNumShards()*2;

  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout("eventsimSpout", eventsimSpout, app.parallelism("eventsimSpout"));
  builder.setBolt("collectionPerTimeFrameSolrBolt", collectionPerTimeFrameSolrBolt, tasksPerShard)
         .customGrouping("eventsimSpout", hashRangeGrouping);

  return builder.createTopology();
}
 
Example #14
Source File: RollingTopWordsTest.java    From jstorm with Apache License 2.0 6 votes vote down vote up
@Test
public void testRollingTopWords()
{
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("windowTestWordSpout", new WindowTestWordSpout(), 5);
    topologyBuilder.setBolt("windowTestRollingCountBolt", new WindowTestRollingCountBolt(9, 3), 4)
            .fieldsGrouping("windowTestWordSpout", new Fields("word")).addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
    topologyBuilder.setBolt("windowTestIntermediateRankingBolt", new WindowTestIntermediateRankingBolt(DEFAULT_COUNT), 4)
            .fieldsGrouping("windowTestRollingCountBolt", new Fields("obj"));
    topologyBuilder.setBolt("windowTestTotalRankingsBolt", new WindowTestTotalRankingsBolt(DEFAULT_COUNT))
            .globalGrouping("windowTestIntermediateRankingBolt");

    Map config = new HashMap();
    config.put(Config.TOPOLOGY_NAME, "RollingTopWordsTest");

    //I really don't know how to validate if the result is right since
    //the tick time is not precise. It makes the output after passing
    //a window is unpredictable.
    //Now I just let it pass all the time.
    //TODO:FIX ME: how to validate if the result is right?
    JStormUnitTestRunner.submitTopology(topologyBuilder.createTopology(), config, 90, null);
}
 
Example #15
Source File: Runner.java    From storm-camel-example with Apache License 2.0 6 votes vote down vote up
public static final void main(final String[] args) throws Exception {
    final ApplicationContext applicationContext = new ClassPathXmlApplicationContext("applicationContext.xml");

    final JmsProvider jmsQueueProvider = new SpringJmsProvider(applicationContext, "jmsConnectionFactory",
            "notificationQueue");

    final TopologyBuilder topologyBuilder = new TopologyBuilder();

    final JmsBolt jmsBolt = new JmsBolt();
    jmsBolt.setJmsProvider(jmsQueueProvider);
    jmsBolt.setJmsMessageProducer((session, input) -> {
        final String json = "{\"word\":\"" + input.getString(0) + "\", \"count\":" + String.valueOf(input.getInteger(1)) + "}";
        return session.createTextMessage(json);
    });

    topologyBuilder.setSpout("wordGenerator", new RandomWordFeeder());
    topologyBuilder.setBolt("counter", new WordCounterBolt()).shuffleGrouping("wordGenerator");
    topologyBuilder.setBolt("jmsBolt", jmsBolt).shuffleGrouping("counter");

    final Config config = new Config();
    config.setDebug(false);

    final LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("word-count", config, topologyBuilder.createTopology());
}
 
Example #16
Source File: TridentFastWordCount.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void test() {
    TopologyBuilder builder = new TopologyBuilder();
    
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    
    if (isLocal) {
        drpc = new LocalDRPC();
    }
    
    try {
        JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf),
                isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}
 
Example #17
Source File: Grep.java    From storm-benchmark with Apache License 2.0 6 votes vote down vote up
@Override
public StormTopology getTopology(Config config) {

  final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
  final int matBoltNum = BenchmarkUtils.getInt(config, FM_NUM, DEFAULT_MAT_BOLT_NUM);
  final int cntBoltNum = BenchmarkUtils.getInt(config, CM_NUM, DEFAULT_CNT_BOLT_NUM);
  final String ptnString = (String) Utils.get(config, PATTERN_STRING, DEFAULT_PATTERN_STR);

  spout = new KafkaSpout(KafkaUtils.getSpoutConfig(config, new SchemeAsMultiScheme(new StringScheme())));

  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout(SPOUT_ID, spout, spoutNum);
  builder.setBolt(FM_ID, new FindMatchingSentence(ptnString), matBoltNum)
          .localOrShuffleGrouping(SPOUT_ID);
  builder.setBolt(CM_ID, new CountMatchingSentence(), cntBoltNum)
          .fieldsGrouping(FM_ID, new Fields(FindMatchingSentence.FIELDS));

  return builder.createTopology();
}
 
Example #18
Source File: RollingSort.java    From storm-benchmark with Apache License 2.0 6 votes vote down vote up
@Override
public StormTopology getTopology(Config config) {
  final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
  final int boltNum =  BenchmarkUtils.getInt(config, SORT_BOLT_NUM, DEFAULT_SORT_BOLT_NUM);
  final int msgSize = BenchmarkUtils.getInt(config, RandomMessageSpout.MESSAGE_SIZE,
          RandomMessageSpout.DEFAULT_MESSAGE_SIZE);
  final int chunkSize = BenchmarkUtils.getInt(config, SortBolt.CHUNK_SIZE,
          SortBolt.DEFAULT_CHUNK_SIZE);
  final int emitFreq = BenchmarkUtils.getInt(config, SortBolt.EMIT_FREQ,
          SortBolt.DEFAULT_EMIT_FREQ);
  spout = new RandomMessageSpout(msgSize, BenchmarkUtils.ifAckEnabled(config));
  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout(SPOUT_ID, spout, spoutNum);
  builder.setBolt(SORT_BOLT_ID, new SortBolt(emitFreq, chunkSize), boltNum)
          .localOrShuffleGrouping(SPOUT_ID);
  return builder.createTopology();
}
 
Example #19
Source File: TestTopology.java    From jstorm with Apache License 2.0 6 votes vote down vote up
private static void submitTopology(TopologyBuilder builder) {
	try {
		if (local_mode(conf)) {

			LocalCluster cluster = new LocalCluster();

			cluster.submitTopology(String.valueOf(conf.get("topology.name")), conf, builder.createTopology());

			Thread.sleep(200000);

			cluster.shutdown();
		} else {
			StormSubmitter.submitTopology(String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());
		}

	} catch (Exception e) {
		e.printStackTrace();
	}
}
 
Example #20
Source File: TestStormStreamIdRouting.java    From eagle with Apache License 2.0 6 votes vote down vote up
@Ignore
@Test
public void testRoutingByStreamId() throws Exception {
    Config conf = new Config();
    conf.setNumWorkers(2); // use two worker processes
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("blue-spout", new BlueSpout()); // parallelism hint

    topologyBuilder.setBolt("green-bolt-1", new GreenBolt(1))
        .shuffleGrouping("blue-spout", "green-bolt-stream-1");
    topologyBuilder.setBolt("green-bolt-2", new GreenBolt(2))
        .shuffleGrouping("blue-spout", "green-bolt-stream-2");

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("mytopology", new HashMap(), topologyBuilder.createTopology());

    while (true) {
        try {
            Thread.sleep(1000);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 
Example #21
Source File: FastWordCountAccumulatedTimeWindowTopology.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void test(Config conf) throws Exception {
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int split_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1);
    int count_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1);

    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint);
    builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout");

    long windowSize = JStormUtils.parseLong(conf.get("window.size.sec"), 10);
    long stateWindowSize = JStormUtils.parseLong(conf.get("state.window.size.sec"), 60);
    builder.setBolt("count", new WordCount()
                    .timeWindow(Time.seconds(windowSize))
                    .withStateSize(Time.seconds(stateWindowSize)),
            count_Parallelism_hint).fieldsGrouping("split", new Fields("word"));

    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];

    boolean isLocal = true;
    if (conf.containsKey("storm.cluster.mode")) {
        isLocal = StormConfig.local_mode(conf);
    }
    // RUN_LONG_TIME = true
    JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60,
            new JStormHelper.CheckAckedFail(conf), isLocal);
}
 
Example #22
Source File: IndexBatchBoltTest.java    From storm-trident-elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public StormTopology buildTopology() {

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("batch", new StaticSpout()).setMaxTaskParallelism(1);
    builder.setBolt("index", newIndexBatchBolt()).shuffleGrouping("batch");

    return  builder.createTopology();
}
 
Example #23
Source File: WordCountTopology.java    From storm-example with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

        SentenceSpout spout = new SentenceSpout();
        SplitSentenceBolt splitBolt = new SplitSentenceBolt();
        WordCountBolt countBolt = new WordCountBolt();
        ReportBolt reportBolt = new ReportBolt();


        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout(SENTENCE_SPOUT_ID, spout, 2);
        // SentenceSpout --> SplitSentenceBolt
        builder.setBolt(SPLIT_BOLT_ID, splitBolt, 2)
                .setNumTasks(4)
                .shuffleGrouping(SENTENCE_SPOUT_ID);
        // SplitSentenceBolt --> WordCountBolt
        builder.setBolt(COUNT_BOLT_ID, countBolt, 4)
                .fieldsGrouping(SPLIT_BOLT_ID, new Fields("word"));
        // WordCountBolt --> ReportBolt
        builder.setBolt(REPORT_BOLT_ID, reportBolt)
                .globalGrouping(COUNT_BOLT_ID);

        Config config = new Config();
        config.setNumWorkers(2);

        LocalCluster cluster = new LocalCluster();

        cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
        waitForSeconds(10);
        cluster.killTopology(TOPOLOGY_NAME);
        cluster.shutdown();
    }
 
Example #24
Source File: WordCountTopology.java    From storm-example with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

        SentenceSpout spout = new SentenceSpout();
        SplitSentenceBolt splitBolt = new SplitSentenceBolt();
        WordCountBolt countBolt = new WordCountBolt();
        ReportBolt reportBolt = new ReportBolt();


        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout(SENTENCE_SPOUT_ID, spout, 2);
        // SentenceSpout --> SplitSentenceBolt
        builder.setBolt(SPLIT_BOLT_ID, splitBolt, 2)
                .setNumTasks(4)
                .shuffleGrouping(SENTENCE_SPOUT_ID);
        // SplitSentenceBolt --> WordCountBolt
        builder.setBolt(COUNT_BOLT_ID, countBolt, 4)
                .fieldsGrouping(SPLIT_BOLT_ID, new Fields("word"));
        // WordCountBolt --> ReportBolt
        builder.setBolt(REPORT_BOLT_ID, reportBolt)
                .globalGrouping(COUNT_BOLT_ID);

        Config config = new Config();
        config.setNumWorkers(2);

        LocalCluster cluster = new LocalCluster();

        cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
        waitForSeconds(10);
        cluster.killTopology(TOPOLOGY_NAME);
        cluster.shutdown();
    }
 
Example #25
Source File: UserDefinedHostsTopology.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void test() {
    TopologyBuilder builder = new TopologyBuilder();
    
    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).noneGrouping("word");
    builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1");
    
    String hostname = NetWorkUtils.hostname();
    List<String> hosts = new ArrayList<String>();
    hosts.add(hostname);
    
    /*********
     * 
     * This make sure all worker run on the user-defined hosts
     * 
     * 
     * 
     */
    conf.put(Config.ISOLATION_SCHEDULER_MACHINES, hosts);
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        JStormHelper.runTopology(builder.createTopology(), topologyName, conf, 60,
                new Validator(conf), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}
 
Example #26
Source File: TransactionTopologyBuilder.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public StormTopology createTopology() {
    TopologyBuilder.putStormConf(ConfigExtension.TASK_BATCH_TUPLE, "true");
    TopologyBuilder.putStormConf(Config.TOPOLOGY_ACKER_EXECUTORS, "0");
    TopologyBuilder.putStormConf(ConfigExtension.TRANSACTION_TOPOLOGY, true);
    return super.createTopology();
}
 
Example #27
Source File: SimpleTopologyWithConfigParam.java    From flux with Apache License 2.0 5 votes vote down vote up
public StormTopology getTopology(Config config) {
    TopologyBuilder builder = new TopologyBuilder();

    // spouts
    FluxShellSpout spout = new FluxShellSpout(
            new String[]{"node", "randomsentence.js"},
            new String[]{"word"});
    builder.setSpout("sentence-spout", spout, 1);

    // bolts
    builder.setBolt("log-bolt", new LogInfoBolt(), 1)
            .shuffleGrouping("sentence-spout");

    return builder.createTopology();
}
 
Example #28
Source File: TMUdfStreamTopology.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Map config = new Config();
    config.put(ConfigExtension.TOPOLOGY_MASTER_USER_DEFINED_STREAM_CLASS, "com.alipay.dw.jstorm.example.tm.TMUdfHandler");
    config.put(Config.TOPOLOGY_WORKERS, 2);

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("TMUdfSpout", new TMUdfSpout(), 2);
    builder.setBolt("TMUdfBolt", new TMUdfBolt(), 4);
    StormTopology topology = builder.createTopology();

    StormSubmitter.submitTopology("TMUdfTopology", config, topology);
}
 
Example #29
Source File: JStormApplication.java    From PoseidonX with Apache License 2.0 5 votes vote down vote up
/**
 * <默认构造函数>
 *
 */
public JStormApplication(StreamingConfig config, String appName, DriverContext driverContext) throws StreamingException {


	super(appName, config);

       this.driverContext = driverContext;
	builder = new TopologyBuilder();
	stormConf = new StormConf(config,this.driverContext.getCqlClient().getCustomizedConfigurationMap());
	streamingSecurity = SecurityFactory.createSecurity(config);
}
 
Example #30
Source File: StormAbstractCloudLiveTest.java    From brooklyn-library with Apache License 2.0 5 votes vote down vote up
private StormTopology createTopology()
        throws AlreadyAliveException, InvalidTopologyException {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
    
    return builder.createTopology();
}