org.apache.storm.testing.TestWordSpout Java Examples

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1");

    Config conf = new Config();
    conf.setDebug(true);
    String topoName = MultipleLoggerTopology.class.getName();
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(2);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 

public static StormTopology createTestTopology() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("words", new TestWordSpout(), 10);
    builder.setBolt("count", new TestWordCounter(), 3).shuffleGrouping("words");
    builder.setBolt("globalCount", new TestGlobalCount(), 2).shuffleGrouping("count");

    return builder.createTopology();
}
 

@Test
@SuppressWarnings("rawtypes")
public void classForName_ReturnsCorrectClass() throws ClassNotFoundException {
  final String className = TestWordSpout.class.getName();

  Class clazz = subject.classForName(className);

  assertThat(clazz, notNullValue());
  assertThat(className, is(equalTo(clazz.getName())));
}
 

@Test
public void buildObject_NoArgs_BehavesAsExpected()
    throws ClassNotFoundException, InvocationTargetException,
    NoSuchFieldException, InstantiationException, IllegalAccessException {

  final Class fixedTupleClass = TestWordSpout.class;
  final String className = TestWordSpout.class.getName();
  List<ConfigurationMethodDefinition> methodDefinitions = new ArrayList<>();
  final String methodName = "close";

  methodDefinitions.add(mockMethodDefinition);

  when(mockObjectDefinition.getClassName()).thenReturn(className);
  when(mockObjectDefinition.hasConstructorArgs()).thenReturn(false);
  when(mockBuilderUtility.classForName(eq(className))).thenReturn(fixedTupleClass);
  when(mockObjectDefinition.getConfigMethods()).thenReturn(methodDefinitions);
  when(mockMethodDefinition.hasReferences()).thenReturn(false);
  when(mockMethodDefinition.getName()).thenReturn(methodName);

  subject.buildObject(mockObjectDefinition, mockContext);

  verify(mockObjectDefinition).getClassName();
  verify(mockObjectDefinition).hasConstructorArgs();
  verify(mockBuilderUtility).classForName(same(className));
  verify(mockBuilderUtility).applyProperties(same(mockObjectDefinition),
      anyObject(), same(mockContext));
  verify(mockObjectDefinition).getConfigMethods();
  verify(mockMethodDefinition).hasReferences();
  verify(mockMethodDefinition).getName();
  verify(mockMethodDefinition).getArgs();
}
 

public static StormTopology createTestTopology() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("words", new TestWordSpout(), 10);
    builder.setBolt("count", new TestWordCounter(), 3).shuffleGrouping("words");
    builder.setBolt("globalCount", new TestGlobalCount(), 2).shuffleGrouping("count");

    return builder.createTopology();
}
 

public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    // a stream of words
    builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2)
           .window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(2)))
           /*
            * create a stream of (word, 1) pairs
            */
           .mapToPair(w -> Pair.of(w, 1))
           /*
            * compute the word counts in the last two second window
            */
           .countByKey()
           /*
            * update the word counts in the state.
            * Here the first argument 0L is the initial value for the state
            * and the second argument is a function that adds the count to the current value in the state.
            */
           .updateStateByKey(0L, (state, count) -> state + count)
           /*
            * convert the state back to a stream and print the results
            */
           .toPairStream()
           .print();

    Config config = new Config();
    // use redis based state store for persistence
    config.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
    String topoName = "test";
    if (args.length > 0) {
        topoName = args[0];
    }
    config.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build());
}
 

public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();

    // Redis config parameters for the RedisStoreBolt
    JedisPoolConfig poolConfig = new JedisPoolConfig.Builder()
        .setHost("127.0.0.1").setPort(6379).build();
    // Storm tuple to redis key-value mapper
    RedisStoreMapper storeMapper = new WordCountStoreMapper();
    // The redis bolt (sink)
    IRichBolt redisStoreBolt = new RedisStoreBolt(poolConfig, storeMapper);

    // A stream of words
    builder.newStream(new TestWordSpout(), new ValueMapper<String>(0))
           /*
            * create a stream of (word, 1) pairs
            */
           .mapToPair(w -> Pair.of(w, 1))
           /*
            * aggregate the count
            */
           .countByKey()
           /*
            * The result of aggregation is forwarded to
            * the RedisStoreBolt. The forwarded tuple is a
            * key-value pair of (word, count) with ("key", "value")
            * being the field names.
            */
           .to(redisStoreBolt);

    Config config = new Config();
    String topoName = "test";
    if (args.length > 0) {
        topoName = args[0];
    }
    config.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build());
}
 

public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    StreamState<String, Long> ss = builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2)
                                          /*
                                           * Transform the stream of words to a stream of (word, 1) pairs
                                           */
                                          .mapToPair(w -> Pair.of(w, 1))
                                          /*
                                           *  Update the count in the state. Here the first argument 0L is the initial value for the
                                           *  count and
                                           *  the second argument is a function that increments the count for each value received.
                                           */
                                          .updateStateByKey(0L, (count, val) -> count + 1);

    /*
     * A stream of words emitted by the QuerySpout is used as
     * the keys to query the state.
     */
    builder.newStream(new QuerySpout(), new ValueMapper<String>(0))
           /*
            * Queries the state and emits the
            * matching (key, value) as results. The stream state returned
            * by the updateStateByKey is passed as the argument to stateQuery.
            */
           .stateQuery(ss).print();

    Config config = new Config();
    // use redis based state store for persistence
    config.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
    String topoName = "test";
    if (args.length > 0) {
        topoName = args[0];
    }
    config.setNumWorkers(1);
    StormSubmitter.submitTopologyWithProgressBar(topoName, config, builder.build());
}
 

/**
 * Submits (runs) the topology.
 *
 * Usage: "RollingTopWords [topology-name] [-local]"
 *
 * By default, the topology is run locally under the name
 * "slidingWindowCounts".
 *
 * Examples:
 *
 * ```
 *
 * # Runs in remote/cluster mode, with topology name "production-topology"
 * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.RollingTopWords production-topology ```
 *
 * @param args
 *          First positional argument (optional) is topology name, second
 *          positional argument (optional) defines whether to run the topology
 *          locally ("-local") or remotely, i.e. on a real cluster.
 * @throws Exception
 */
protected int run(String[] args) {
    String topologyName = "slidingWindowCounts";
    if (args.length >= 1) {
        topologyName = args[0];
    }
    TopologyBuilder builder = new TopologyBuilder();
    String spoutId = "wordGenerator";
    String counterId = "counter";
    String intermediateRankerId = "intermediateRanker";
    String totalRankerId = "finalRanker";
    builder.setSpout(spoutId, new TestWordSpout(), 5);
    builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).fieldsGrouping(spoutId, new Fields("word"));
    builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(counterId,
                                                                                                 new Fields("obj"));
    builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
    LOG.info("Topology name: " + topologyName);

    return submit(topologyName, conf, builder);
}
 

/**
 * Submits (runs) the topology.
 *
 * Usage: "SkewedRollingTopWords [topology-name] [-local]"
 *
 * By default, the topology is run locally under the name
 * "slidingWindowCounts".
 *
 * Examples:
 *
 * ```
 *
 * # Runs in remote/cluster mode, with topology name "production-topology"
 * $ storm jar storm-starter-jar-with-dependencies.jar org.apache.storm.starter.SkewedRollingTopWords production-topology ```
 *
 * @param args
 *          First positional argument (optional) is topology name, second
 *          positional argument (optional) defines whether to run the topology
 *          locally ("-local") or remotely, i.e. on a real cluster.
 * @throws Exception
 */
protected int run(String[] args) {
    String topologyName = "slidingWindowCounts";
    if (args.length >= 1) {
        topologyName = args[0];
    }
    TopologyBuilder builder = new TopologyBuilder();
    String spoutId = "wordGenerator";
    String counterId = "counter";
    String aggId = "aggregator";
    String intermediateRankerId = "intermediateRanker";
    String totalRankerId = "finalRanker";
    builder.setSpout(spoutId, new TestWordSpout(), 5);
    builder.setBolt(counterId, new RollingCountBolt(9, 3), 4).partialKeyGrouping(spoutId, new Fields("word"));
    builder.setBolt(aggId, new RollingCountAggBolt(), 4).fieldsGrouping(counterId, new Fields("obj"));
    builder.setBolt(intermediateRankerId, new IntermediateRankingsBolt(TOP_N), 4).fieldsGrouping(aggId, new Fields("obj"));
    builder.setBolt(totalRankerId, new TotalRankingsBolt(TOP_N)).globalGrouping(intermediateRankerId);
    LOG.info("Topology name: " + topologyName);

    return submit(topologyName, conf, builder);
}
 

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    //A topology can set resources in terms of CPU and Memory for each component
    // These can be chained (like with setting the CPU requirement)
    SpoutDeclarer spout = builder.setSpout("word", new TestWordSpout(), 10).setCPULoad(20);
    // Or done separately like with setting the
    // onheap and offheap memory requirement
    spout.setMemoryLoad(64, 16);
    //On heap memory is used to help calculate the heap of the java process for the worker
    // off heap memory is for things like JNI memory allocated off heap, or when using the
    // ShellBolt or ShellSpout.  In this case the 16 MB of off heap is just as an example
    // as we are not using it.

    // Some times a Bolt or Spout will have some memory that is shared between the instances
    // These are typically caches, but could be anything like a static database that is memory
    // mapped into the processes. These can be declared separately and added to the bolts and
    // spouts that use them.  Or if only one uses it they can be created inline with the add
    SharedOnHeap exclaimCache = new SharedOnHeap(100, "exclaim-cache");
    SharedOffHeapWithinNode notImplementedButJustAnExample =
        new SharedOffHeapWithinNode(500, "not-implemented-node-level-cache");

    //If CPU or memory is not set the values stored in topology.component.resources.onheap.memory.mb,
    // topology.component.resources.offheap.memory.mb and topology.component.cpu.pcore.percent
    // will be used instead
    builder
        .setBolt("exclaim1", new ExclamationBolt(), 3)
        .shuffleGrouping("word")
        .addSharedMemory(exclaimCache);

    builder
        .setBolt("exclaim2", new ExclamationBolt(), 2)
        .shuffleGrouping("exclaim1")
        .setMemoryLoad(100)
        .addSharedMemory(exclaimCache)
        .addSharedMemory(notImplementedButJustAnExample);

    Config conf = new Config();
    conf.setDebug(true);

    //Under RAS the number of workers is determined by the scheduler and the settings in the conf are ignored
    //conf.setNumWorkers(3);

    //Instead the scheduler lets you set the maximum heap size for any worker.
    conf.setTopologyWorkerMaxHeapSize(1024.0);
    //The scheduler generally will try to pack executors into workers until the max heap size is met, but
    // this can vary depending on the specific scheduling strategy selected.
    // The reason for this is to try and balance the maximum pause time GC might take (which is larger for larger heaps)
    // against better performance because of not needing to serialize/deserialize tuples.

    //The priority of a topology describes the importance of the topology in decreasing importance
    // starting from 0 (i.e. 0 is the highest priority and the priority importance decreases as the priority number increases).
    //Recommended range of 0-29 but no hard limit set.
    // If there are not enough resources in a cluster the priority in combination with how far over a guarantees
    // a user is will decide which topologies are run and which ones are not.
    conf.setTopologyPriority(29);

    //set to use the default resource aware strategy when using the MultitenantResourceAwareBridgeScheduler
    conf.setTopologyStrategy(
        "org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy");

    String topoName = "test";
    if (args != null && args.length > 0) {
        topoName = args[0];
    }

    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}
 

protected int run(String[] args) {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");

    conf.setDebug(true);

    String topologyName = "test";

    conf.setNumWorkers(3);

    if (args != null && args.length > 0) {
        topologyName = args[0];
    }

    return submit(topologyName, conf, builder);
}