org.apache.storm.topology.TopologyBuilder Java Examples

The following examples show how to use org.apache.storm.topology.TopologyBuilder. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AnchoredWordCount.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
protected int run(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new RandomSentenceSpout(), 4);

    builder.setBolt("split", new SplitSentence(), 4).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), 4).fieldsGrouping("split", new Fields("word"));

    Config conf = new Config();
    conf.setMaxTaskParallelism(3);

    String topologyName = "word-count";

    conf.setNumWorkers(3);

    if (args != null && args.length > 0) {
        topologyName = args[0];
    }
    return submit(topologyName, conf, builder);
}
 
Example #2
Source File: MultipleLoggerTopology.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1");

    Config conf = new Config();
    conf.setDebug(true);
    String topoName = MultipleLoggerTopology.class.getName();
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(2);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 
Example #3
Source File: AbstractSpoutMultiIndexRead.java    From elasticsearch-hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultiIndexRead() throws Exception {
    testRuns++;

    RestUtils.postData(index + "/foo",
            "{\"message\" : \"Hello World\",\"message_date\" : \"2014-05-25\"}".getBytes());
    RestUtils.postData(index + "/bar",
            "{\"message\" : \"Goodbye World\",\"message_date\" : \"2014-05-25\"}".getBytes());
    RestUtils.refresh(index);

    String target = "_all/foo";
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("es-spout", new TestSpout(new EsSpout(target)));
    builder.setBolt("test-bolt", new CapturingBolt()).shuffleGrouping("es-spout");

    MultiIndexSpoutStormSuite.run(index + "multi", builder.createTopology(), COMPONENT_HAS_COMPLETED);

    assumeTrue(COMPONENT_HAS_COMPLETED.is(2));
    COMPONENT_HAS_COMPLETED.waitFor(1, TimeValue.timeValueSeconds(10));

    String results = RestUtils.get(target + "/_search?");
    assertThat(results, containsString("Hello"));

    assertThat(CapturingBolt.CAPTURED.size(), greaterThanOrEqualTo(testRuns));
}
 
Example #4
Source File: ScottyDemoTopology.java    From scotty-window-processor with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    LocalCluster cluster = new LocalCluster();
    TopologyBuilder builder = new TopologyBuilder();

    Config conf = new Config();
    conf.setDebug(false);
    conf.setNumWorkers(1);
    conf.setMaxTaskParallelism(1);
    //Disable Acking
    conf.setNumAckers(0);

    KeyedScottyWindowOperator scottyBolt = new KeyedScottyWindowOperator<Integer, Integer>(new Sum(), 0);
    scottyBolt.addWindow(new TumblingWindow(WindowMeasure.Time, 1000));
    scottyBolt.addWindow(new SlidingWindow(WindowMeasure.Time, 1000, 250));
    scottyBolt.addWindow(new SessionWindow(WindowMeasure.Time, 1000));

    builder.setSpout("spout", new DataGeneratorSpout());
    builder.setBolt("scottyWindow", scottyBolt).fieldsGrouping("spout", new Fields("key"));
    builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("scottyWindow");

    cluster.submitTopology("testTopology", conf, builder.createTopology());
    //cluster.killTopology("testTopology");
    //cluster.shutdown();
}
 
Example #5
Source File: NginxStorm.java    From storm-nginx-log with MIT License 6 votes vote down vote up
public static void main(String[] argv) throws InterruptedException {

        Config config = new Config();
        config.setDebug(true);

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("LogSpout", new LogSpout(), 1);
        builder.setBolt("SpliteBolt", new SpliteBolt(), 1).shuffleGrouping("LogSpout");
        builder.setBolt("CounterBolt", new CounterBolt(), 1)
                .fieldsGrouping("SpliteBolt", new Fields("item"));

        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("NginxLog", config, builder.createTopology());
//        Thread.sleep(10000);
//
//        cluster.killTopology("NginxLog");
//        cluster.shutdown();
    }
 
Example #6
Source File: ClusterTopology.java    From nightwatch with GNU Lesser General Public License v3.0 6 votes vote down vote up
private static TopologyBuilder buildTopology() throws Exception {
        TopologyBuilder builder = new TopologyBuilder();
        String topicName = Configuration.getConfig().getString("rtc.mq.spout.topic");
        String groupName = Configuration.getConfig().getString("rtc.mq.spout.group");
        BrokerHosts hosts = new ZkHosts(Configuration.getConfig().getString("rtc.zk.hosts"));
        SpoutConfig spoutConfig = new SpoutConfig(hosts, topicName, "/consumers", groupName);

        spoutConfig.startOffsetTime = kafka.api.OffsetRequest.LatestTime();
        spoutConfig.zkServers = Arrays.asList(Configuration.getConfig().getString("rtc.storm.zkServers").split(","));
        spoutConfig.zkPort = Configuration.getConfig().getInt("rtc.storm.zkPort");
        spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
        KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
        builder.setSpout("MQSpout", kafkaSpout, Configuration.getConfig().getInt("rtc.storm.spout.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.spout.task"));
        builder.setBolt("ExtractBolt", new ExtractBolt(), Configuration.getConfig().getInt("rtc.storm.extract.bolt.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.extract.bolt.task")).shuffleGrouping("MQSpout");
        builder.setBolt("Statistic", new StatisticBolt(), Configuration.getConfig().getInt("rtc.storm.statistic.bolt.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.statistic.bolt.task")).fieldsGrouping("ExtractBolt", new Fields(new String[]{"hashKeys"}));
//        builder.setBolt("Alarm", new AlarmBolt(), Configuration.getConfig().getInt("rtc.storm.alarm.bolt.parallelismHint")).setNumTasks(Configuration.getConfig().getInt("rtc.storm.alarm.bolt.task")).fieldsGrouping("Statistic", new Fields(new String[]{"EventName"}));
        return builder;
    }
 
Example #7
Source File: WordCountTopologyNode.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {

        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout("spout", new RandomSentence(), 5);

        builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
        builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));

        Config conf = new Config();
        conf.setDebug(true);
        String topoName = "word-count";
        if (args != null && args.length > 0) {
            topoName = args[0];
        }
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
    }
 
Example #8
Source File: TopologyFactoryBean.java    From breeze with Apache License 2.0 6 votes vote down vote up
private StormTopology build() {
	run();
	verify();

	Map<String,BoltDeclarer> declaredBolts = new HashMap<>();

	TopologyBuilder builder = new TopologyBuilder();
	for (Map.Entry<ConfiguredSpout,List<ConfiguredBolt>> line : entrySet()) {
		ConfiguredSpout spout = line.getKey();
		String lastId = spout.getId();
		String streamId = spout.getOutputStreamId();
		builder.setSpout(lastId, spout, spout.getParallelism());
		for (ConfiguredBolt bolt : line.getValue()) {
			String id = bolt.getId();
			BoltDeclarer declarer = declaredBolts.get(id);
			if (declarer == null)
				declarer = builder.setBolt(id, bolt, bolt.getParallelism());
			declarer.noneGrouping(lastId, streamId);
			if (declaredBolts.put(id, declarer) != null) break;
			lastId = id;
			streamId = bolt.getOutputStreamId();
		}
	}

	return builder.createTopology();
}
 
Example #9
Source File: StreamFromEs.java    From elasticsearch-hadoop with Apache License 2.0 6 votes vote down vote up
public static void submitJob(String principal, String keytab, String esNodes) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("ES", new EsSpout("storm-test"));
    builder.setBolt("Output", new CapturingBolt()).shuffleGrouping("ES");

    // Nimbus needs to be started with the cred renewer and credentials plugins set in its config file

    Config conf = new Config();
    List<Object> plugins = new ArrayList<Object>();
    plugins.add(AutoElasticsearch.class.getName());
    conf.put(Config.TOPOLOGY_AUTO_CREDENTIALS, plugins);
    conf.put(ConfigurationOptions.ES_NODES, esNodes);
    conf.put(ConfigurationOptions.ES_SECURITY_AUTHENTICATION, "kerberos");
    conf.put(ConfigurationOptions.ES_NET_SPNEGO_AUTH_ELASTICSEARCH_PRINCIPAL, "HTTP/[email protected]");
    conf.put(ConfigurationOptions.ES_INPUT_JSON, "true");
    StormSubmitter.submitTopology("test-read", conf, builder.createTopology());
}
 
Example #10
Source File: AbstractStormJsonSimpleBoltTests.java    From elasticsearch-hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleWriteTopology() throws Exception {
    List doc1 = Collections.singletonList("{\"reason\" : \"business\",\"airport\" : \"SFO\"}");
    List doc2 = Collections.singletonList("{\"participants\" : 5,\"airport\" : \"OTP\"}");

    String target = index + "/json-simple-write";
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("test-spout-1", new TestSpout(ImmutableList.of(doc1, doc2), new Fields("json")));
    builder.setBolt("es-bolt-1", new TestBolt(new EsBolt(target, conf))).shuffleGrouping("test-spout-1");

    assumeTrue(COMPONENT_HAS_COMPLETED.is(2));

    MultiIndexSpoutStormSuite.run(index + "json-simple", builder.createTopology(), COMPONENT_HAS_COMPLETED);

    COMPONENT_HAS_COMPLETED.waitFor(1, TimeValue.timeValueSeconds(10));

    RestUtils.refresh(index);
    assertTrue(RestUtils.exists(target));
    String results = RestUtils.get(target + "/_search?");
    assertThat(results, containsString("SFO"));
}
 
Example #11
Source File: StormRangerAuthorizerTest.java    From ranger with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateTopologyBob() throws Exception {
    final Config conf = new Config();
    conf.setDebug(true);

    final TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("words", new WordSpout());
    builder.setBolt("counter", new WordCounterBolt()).shuffleGrouping("words");

    final Subject subject = new Subject();
    subject.getPrincipals().add(new SimplePrincipal("bob"));
    Subject.doAs(subject, new PrivilegedExceptionAction<Void>() {
        public Void run() throws Exception {
            try {
                cluster.submitTopology("word-count2", conf, builder.createTopology());
                Assert.fail("Authorization failure expected");
            } catch (Exception ex) {
                // expected
            }

            return null;
        }
    });
}
 
Example #12
Source File: ConfigurableTopology.java    From storm-crawler with Apache License 2.0 6 votes vote down vote up
/** Submits the topology under a specific name **/
protected int submit(String name, Config conf, TopologyBuilder builder) {

    // register Metadata for serialization with FieldsSerializer
    Config.registerSerialization(conf, Metadata.class);

    if (isLocal) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(name, conf, builder.createTopology());
        if (ttl != -1) {
            Utils.sleep(ttl * 1000);
            cluster.shutdown();
        }
    }

    else {
        try {
            StormSubmitter.submitTopology(name, conf,
                    builder.createTopology());
        } catch (Exception e) {
            e.printStackTrace();
            return -1;
        }
    }
    return 0;
}
 
Example #13
Source File: SentenceWordCountTopology.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  String name = "fast-word-count-topology";
  if (args != null && args.length > 0) {
    name = args[0];
  }

  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("spout", new FastRandomSentenceSpout(), 1);
  builder.setBolt("split", new SplitSentence(), 2).shuffleGrouping("spout");
  builder.setBolt("count", new WordCount(), 2).fieldsGrouping("split", new Fields("word"));

  Config conf = new Config();

  StormSubmitter.submitTopology(name, conf, builder.createTopology());
}
 
Example #14
Source File: StreamToEs.java    From elasticsearch-hadoop with Apache License 2.0 6 votes vote down vote up
public static void submitJob(String principal, String keytab, String esNodes) throws Exception {
    List doc1 = Collections.singletonList("{\"reason\" : \"business\",\"airport\" : \"SFO\"}");
    List doc2 = Collections.singletonList("{\"participants\" : 5,\"airport\" : \"OTP\"}");

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("Input", new TestSpout(ImmutableList.of(doc1, doc2), new Fields("json"), true));
    builder.setBolt("ES", new EsBolt("storm-test"))
            .shuffleGrouping("Input")
            .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 2);

    // Nimbus needs to be started with the cred renewer and credentials plugins set in its config file

    Config conf = new Config();
    List<Object> plugins = new ArrayList<Object>();
    plugins.add(AutoElasticsearch.class.getName());
    conf.put(Config.TOPOLOGY_AUTO_CREDENTIALS, plugins);
    conf.put(ConfigurationOptions.ES_NODES, esNodes);
    conf.put(ConfigurationOptions.ES_SECURITY_AUTHENTICATION, "kerberos");
    conf.put(ConfigurationOptions.ES_NET_SPNEGO_AUTH_ELASTICSEARCH_PRINCIPAL, "HTTP/[email protected]");
    conf.put(ConfigurationOptions.ES_INPUT_JSON, "true");
    StormSubmitter.submitTopology("test-run", conf, builder.createTopology());
}
 
Example #15
Source File: AbstractSpoutSimpleRead.java    From elasticsearch-hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testSimpleRead() throws Exception {
    String target = index + "/basic-read";

    RestUtils.touch(index);
    RestUtils.postData(target, "{\"message\" : \"Hello World\",\"message_date\" : \"2014-05-25\"}".getBytes());
    RestUtils.postData(target, "{\"message\" : \"Goodbye World\",\"message_date\" : \"2014-05-25\"}".getBytes());
    RestUtils.refresh(index);

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("es-spout", new TestSpout(new EsSpout(target)));
    builder.setBolt("test-bolt", new CapturingBolt()).shuffleGrouping("es-spout");

    MultiIndexSpoutStormSuite.run(index + "simple", builder.createTopology(), COMPONENT_HAS_COMPLETED);

    COMPONENT_HAS_COMPLETED.waitFor(1, TimeValue.timeValueSeconds(10));

    assertTrue(RestUtils.exists(target));
    String results = RestUtils.get(target + "/_search?");
    assertThat(results, containsString("Hello"));
    assertThat(results, containsString("Goodbye"));

    System.out.println(CapturingBolt.CAPTURED);
    assertThat(CapturingBolt.CAPTURED.size(), is(2));
}
 
Example #16
Source File: SlidingTupleTsTopology.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    BaseWindowedBolt bolt = new SlidingWindowSumBolt()
        .withWindow(new Duration(5, TimeUnit.SECONDS), new Duration(3, TimeUnit.SECONDS))
        .withTimestampField("ts")
        .withLag(new Duration(5, TimeUnit.SECONDS));
    builder.setSpout("integer", new RandomIntegerSpout(), 1);
    builder.setBolt("slidingsum", bolt, 1).shuffleGrouping("integer");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("slidingsum");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";

    if (args != null && args.length > 0) {
        topoName = args[0];
    }

    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 
Example #17
Source File: StatefulWindowingTopology.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new RandomIntegerSpout());
    builder.setBolt("sumbolt", new WindowSumBolt().withWindow(new Count(5), new Count(3))
                                                  .withMessageIdField("msgid"), 1).shuffleGrouping("spout");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("sumbolt");
    Config conf = new Config();
    conf.setDebug(false);
    //conf.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");

    String topoName = "test";
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 
Example #18
Source File: StormEcoBuilderTest.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
@Test
public void testBuildTopologyBuilder_BuildsAsExpected()
    throws IllegalAccessException, ClassNotFoundException,
    InstantiationException, NoSuchFieldException, InvocationTargetException {
  Config config = new Config();
  EcoExecutionContext context = new EcoExecutionContext(ecoTopologyDefinition, config);
  ObjectBuilder objectBuilder = new ObjectBuilder();
  subject.buildTopologyBuilder(context, objectBuilder);

  verify(mockSpoutBuilder).buildSpouts(same(context),
      any(TopologyBuilder.class), same(objectBuilder));
  verify(mockBoltBuilder).buildBolts(same(context), same(objectBuilder));
  verify(mockStreamBuilder).buildStreams(same(context), any(TopologyBuilder.class),
      same(objectBuilder));
  verify(mockComponentBuilder).buildComponents(same(context), same(objectBuilder));
}
 
Example #19
Source File: SlidingWindowTopology.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("integer", new RandomIntegerSpout(), 1);
    builder.setBolt("slidingsum", new SlidingWindowSumBolt().withWindow(Count.of(30), Count.of(10)), 1)
           .shuffleGrouping("integer");
    builder.setBolt("tumblingavg", new TumblingWindowAvgBolt().withTumblingWindow(Count.of(3)), 1)
           .shuffleGrouping("slidingsum");
    builder.setBolt("printer", new PrinterBolt(), 1).shuffleGrouping("tumblingavg");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = "test";
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 
Example #20
Source File: MultiSpoutExclamationTopology.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("word0", new TestWordSpout(), 2);
  builder.setSpout("word1", new TestWordSpout(), 2);
  builder.setSpout("word2", new TestWordSpout(), 2);
  builder.setBolt("exclaim1", new ExclamationBolt(), 2)
      .shuffleGrouping("word0")
      .shuffleGrouping("word1")
      .shuffleGrouping("word2");

  Config conf = new Config();
  conf.setDebug(true);
  conf.setMaxSpoutPending(10);
  conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
  if (args != null && args.length > 0) {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
  } else {
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    cluster.killTopology("test");
    cluster.shutdown();
  }
}
 
Example #21
Source File: AckingTopology.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  if (args.length != 1) {
    throw new RuntimeException("Specify topology name");
  }
  TopologyBuilder builder = new TopologyBuilder();

  int spouts = 2;
  int bolts = 2;
  builder.setSpout("word", new AckingTestWordSpout(), spouts);
  builder.setBolt("exclaim1", new ExclamationBolt(), bolts)
      .shuffleGrouping("word");

  Config conf = new Config();
  conf.setDebug(true);

  // Put an arbitrary large number here if you don't want to slow the topology down
  conf.setMaxSpoutPending(1000 * 1000 * 1000);

  // To enable acking, we need to setEnableAcking true
  conf.setNumAckers(1);
  conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");

  // Set the number of workers or stream managers
  conf.setNumWorkers(2);
  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
 
Example #22
Source File: SlidingWindowTopology.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout("integer", new RandomIntegerSpout(), 1);
  builder.setBolt("slidingsum", new SlidingWindowSumBolt()
      .withWindow(BaseWindowedBolt.Count.of(30), BaseWindowedBolt.Count.of(10)), 1)
      .shuffleGrouping("integer");
  builder.setBolt("tumblingavg", new TumblingWindowAvgBolt()
      .withTumblingWindow(BaseWindowedBolt.Count.of(3)), 1)
      .shuffleGrouping("slidingsum");
  builder.setBolt("printer", new PrinterBolt(), 1)
      .shuffleGrouping("tumblingavg");
  Config conf = new Config();
  conf.setDebug(true);
  String topoName = "test";

  if (args != null && args.length > 0) {
    topoName = args[0];
  }
  StormSubmitter.submitTopology(topoName, conf, builder.createTopology());
}
 
Example #23
Source File: StormRangerAuthorizerTest.java    From ranger with Apache License 2.0 6 votes vote down vote up
@org.junit.BeforeClass
public static void setup() throws Exception {
    cluster = new LocalCluster();

    final Config conf = new Config();
    conf.setDebug(true);

    final TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("words", new WordSpout());
    builder.setBolt("counter", new WordCounterBolt()).shuffleGrouping("words");

    // bob can create a new topology
    final Subject subject = new Subject();
    subject.getPrincipals().add(new SimplePrincipal("bob"));
    Subject.doAs(subject, new PrivilegedExceptionAction<Void>() {
        public Void run() throws Exception {
            cluster.submitTopology("word-count", conf, builder.createTopology());
            return null;
        }
    });

}
 
Example #24
Source File: TopologyModule.java    From monasca-thresh with Apache License 2.0 5 votes vote down vote up
@Provides
StormTopology topology() {
  TopologyBuilder builder = new TopologyBuilder();

  // Receives metrics
  builder.setSpout("metrics-spout", Injector.getInstance(IRichSpout.class, "metrics"),
      config.metricSpoutThreads).setNumTasks(config.metricSpoutTasks);

  // Receives events
  builder.setSpout("event-spout", Injector.getInstance(IRichSpout.class, "event"),
      config.eventSpoutThreads).setNumTasks(config.eventSpoutTasks);

  // Event -> Events
  builder
      .setBolt("event-bolt", new EventProcessingBolt(config.database), config.eventBoltThreads)
      .shuffleGrouping("event-spout").setNumTasks(config.eventBoltTasks);

  // Metrics / Event -> Filtering
  builder
      .setBolt("filtering-bolt", new MetricFilteringBolt(config.database),
          config.filteringBoltThreads)
      .fieldsGrouping("metrics-spout", new Fields(MetricSpout.FIELDS[0]))
      .allGrouping("event-bolt", EventProcessingBolt.METRIC_ALARM_EVENT_STREAM_ID)
      .allGrouping("event-bolt", EventProcessingBolt.ALARM_DEFINITION_EVENT_STREAM_ID)
      .setNumTasks(config.filteringBoltTasks);

  // Filtering /Event -> Alarm Creation
  builder
      .setBolt("alarm-creation-bolt", new AlarmCreationBolt(config.database),
          config.alarmCreationBoltThreads)
      .fieldsGrouping("filtering-bolt",
          MetricFilteringBolt.NEW_METRIC_FOR_ALARM_DEFINITION_STREAM,
          new Fields(AlarmCreationBolt.ALARM_CREATION_FIELDS[3]))
      .allGrouping("event-bolt", EventProcessingBolt.METRIC_SUB_ALARM_EVENT_STREAM_ID)
      .allGrouping("event-bolt", EventProcessingBolt.ALARM_EVENT_STREAM_ID)
      .allGrouping("event-bolt", EventProcessingBolt.ALARM_DEFINITION_EVENT_STREAM_ID)
      .setNumTasks(config.alarmCreationBoltTasks);

  // Filtering / Event / Alarm Creation -> Aggregation
  builder
      .setBolt("aggregation-bolt",
          new MetricAggregationBolt(config, config.database), config.aggregationBoltThreads)
      .fieldsGrouping("filtering-bolt", new Fields(MetricFilteringBolt.FIELDS[0]))
      .allGrouping("filtering-bolt", MetricAggregationBolt.METRIC_AGGREGATION_CONTROL_STREAM)
      .fieldsGrouping("filtering-bolt", AlarmCreationBolt.ALARM_CREATION_STREAM,
          new Fields(AlarmCreationBolt.ALARM_CREATION_FIELDS[1]))
      .allGrouping("event-bolt", EventProcessingBolt.METRIC_SUB_ALARM_EVENT_STREAM_ID)
      .fieldsGrouping("event-bolt", EventProcessingBolt.METRIC_ALARM_EVENT_STREAM_ID,
          new Fields(EventProcessingBolt.METRIC_ALARM_EVENT_STREAM_FIELDS[1]))
      .fieldsGrouping("alarm-creation-bolt", AlarmCreationBolt.ALARM_CREATION_STREAM,
          new Fields(AlarmCreationBolt.ALARM_CREATION_FIELDS[1]))
      .setNumTasks(config.aggregationBoltTasks);

  // Alarm Creation / Event
  // Aggregation / Event -> Thresholding
  builder
      .setBolt("thresholding-bolt",
          new AlarmThresholdingBolt(config.database, config.kafkaProducerConfig),
          config.thresholdingBoltThreads)
      .fieldsGrouping("aggregation-bolt", new Fields(MetricAggregationBolt.FIELDS[0]))
      .fieldsGrouping("event-bolt", EventProcessingBolt.ALARM_EVENT_STREAM_ID,
          new Fields(EventProcessingBolt.ALARM_EVENT_STREAM_FIELDS[1]))
      .allGrouping("event-bolt", EventProcessingBolt.ALARM_DEFINITION_EVENT_STREAM_ID)
      .allGrouping("event-bolt", EventProcessingBolt.METRIC_SUB_ALARM_EVENT_STREAM_ID)
      .setNumTasks(config.thresholdingBoltTasks);

  return builder.createTopology();
}
 
Example #25
Source File: StormKafkaProcess.java    From BigData with GNU General Public License v3.0 5 votes vote down vote up
public static void main(String[] args)
		throws InterruptedException, InvalidTopologyException, AuthorizationException, AlreadyAliveException {

	String topologyName = "TSAS";// 元组名
	// Zookeeper主机地址,会自动选取其中一个
	ZkHosts zkHosts = new ZkHosts("192.168.230.128:2181,192.168.230.129:2181,192.168.230.131:2181");
	String topic = "trademx";
	String zkRoot = "/storm";// storm在Zookeeper上的根路径
	String id = "tsaPro";

	// 创建SpoutConfig对象
	SpoutConfig spontConfig = new SpoutConfig(zkHosts, topic, zkRoot, id);

	TopologyBuilder builder = new TopologyBuilder();
	builder.setSpout("kafka", new KafkaSpout(spontConfig), 2);
	builder.setBolt("AccBolt", new AccBolt()).shuffleGrouping("kafka");
	builder.setBolt("ToDbBolt", new ToDbBolt()).shuffleGrouping("AccBolt");

	Config config = new Config();
	config.setDebug(false);

	if (args.length == 0) { // 本地运行,用于测试
		LocalCluster localCluster = new LocalCluster();
		localCluster.submitTopology(topologyName, config, builder.createTopology());
		Thread.sleep(1000 * 3600);
		localCluster.killTopology(topologyName);
		localCluster.shutdown();
	} else { // 提交至集群运行
		StormSubmitter.submitTopology(topologyName, config, builder.createTopology());
	}

}
 
Example #26
Source File: ConfigurableTopology.java    From storm-crawler with Apache License 2.0 5 votes vote down vote up
/** Submits the topology with the name taken from the configuration **/
protected int submit(Config conf, TopologyBuilder builder) {
    String name = ConfUtils.getString(conf, Config.TOPOLOGY_NAME);
    if (StringUtils.isBlank(name))
        throw new RuntimeException("No value found for "
                + Config.TOPOLOGY_NAME);
    return submit(name, conf, builder);
}
 
Example #27
Source File: WordCountApp.java    From java-study with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException {
 	//定义拓扑
     TopologyBuilder builder = new TopologyBuilder();
     builder.setSpout("word-reader" , new WordReader());
     builder.setBolt("word-normalizer" , new WordNormalizer()).shuffleGrouping("word-reader" );
     builder.setBolt("word-counter" , new WordCounter()).fieldsGrouping("word-normalizer" , new Fields("word"));
     StormTopology topology = builder.createTopology();
     //配置
     
     Config conf = new Config();
     String fileName ="words.txt" ;
     conf.put("fileName" , fileName );
     conf.setDebug(false);
 
      //运行拓扑
      System.out.println("开始...");
      if(args !=null&&args.length>0){ //有参数时,表示向集群提交作业,并把第一个参数当做topology名称
     	 System.out.println("远程模式");
          try {
	StormSubmitter.submitTopology(args[0], conf, topology);
} catch (AuthorizationException e) {
	e.printStackTrace();
}
    } else{//没有参数时,本地提交
      //启动本地模式
 	 System.out.println("本地模式");
      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("Getting-Started-Topologie" , conf , topology );
      Thread.sleep(5000);
      //关闭本地集群
      cluster.shutdown();
    }
      System.out.println("结束");
    
 }
 
Example #28
Source File: JoinBoltExample.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
    if (!NimbusClient.isLocalOverride()) {
        throw new IllegalStateException("This example only works in local mode.  "
                                        + "Run with storm local not storm jar");
    }
    FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
    FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("genderSpout", genderSpout);
    builder.setSpout("ageSpout", ageSpout);

    // inner join of 'age' and 'gender' records on 'id' field
    JoinBolt joiner = new JoinBolt("genderSpout", "id")
        .join("ageSpout", "id", "genderSpout")
        .select("genderSpout:id,ageSpout:id,gender,age")
        .withTumblingWindow(new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS));

    builder.setBolt("joiner", joiner)
           .fieldsGrouping("genderSpout", new Fields("id"))
           .fieldsGrouping("ageSpout", new Fields("id"));

    builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("joiner");

    Config conf = new Config();
    StormSubmitter.submitTopologyWithProgressBar("join-example", conf, builder.createTopology());

    generateGenderData(genderSpout);

    generateAgeData(ageSpout);
}
 
Example #29
Source File: LambdaTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Override
protected int run(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    // example. spout1: generate random strings
    // bolt1: get the first part of a string
    // bolt2: output the tuple

    // NOTE: Variable used in lambda expression should be final or effectively final
    // (or it will cause compilation error),
    // and variable type should implement the Serializable interface if it isn't primitive type
    // (or it will cause not serializable exception).
    Prefix prefix = new Prefix("Hello lambda:");
    String suffix = ":so cool!";
    int tag = 999;

    builder.setSpout("spout1", () -> UUID.randomUUID().toString());
    builder.setBolt("bolt1", (tuple, collector) -> {
        String[] parts = tuple.getStringByField("lambda").split("\\-");
        collector.emit(new Values(prefix + parts[0] + suffix, tag));
    }, "strValue", "intValue").shuffleGrouping("spout1");
    builder.setBolt("bolt2", tuple -> System.out.println(tuple)).shuffleGrouping("bolt1");

    Config conf = new Config();
    conf.setDebug(true);
    conf.setNumWorkers(2);

    return submit("lambda-demo", conf, builder);
}
 
Example #30
Source File: LocalWordCountStormTopology.java    From 163-bigdate-note with GNU General Public License v3.0 5 votes vote down vote up
public static void main(String[] args) {
    //根据Spout和Bolt构建TopologyBuilder
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("DataSourceSpout", new DataSourceSpout());
    builder.setBolt("SplitBolt", new SplitBolt()).shuffleGrouping("DataSourceSpout");
    builder.setBolt("CountBolt", new CountBolt()).shuffleGrouping("SplitBolt");

    //创建本地集群
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("LocalWordCountStormTopology", new Config(), builder.createTopology());

}