Java Code Examples for backtype.storm.Config#setMaxSpoutPending()

The following examples show how to use backtype.storm.Config#setMaxSpoutPending() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ExampleRunner.java From flowmix with Apache License 2.0

6 votes

public void run() {

    StormTopology topology = new FlowmixBuilder()
        .setFlowLoader(new SimpleFlowLoaderSpout(provider.getFlows(), 60000))
        .setEventsLoader(new MockEventGeneratorSpout(getMockEvents(), 10))
        .setOutputBolt(new PrinterBolt())
        .setParallelismHint(6)
      .create()
    .createTopology();

    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.setMaxSpoutPending(5000);
    conf.setDebug(false);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("example-topology", conf, topology);
  }

Example 2

Source File: WordCountTrident.java From storm-hbase with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    if (args.length == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
        Thread.sleep(60 * 1000);
        cluster.killTopology("wordCounter");
        cluster.shutdown();
        System.exit(0);
    }
    else if(args.length == 2) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[1], conf, buildTopology(args[0]));
    } else{
        System.out.println("Usage: TridentFileTopology <hdfs url> [topology name]");
    }
}

Example 3

Source File: TridentWordCount.java From flink-perf with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
  Config conf = new Config();
  conf.setMaxSpoutPending(20);
  if (args.length == 0) {
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("wordCounter", conf, buildTopology(drpc));
    for (int i = 0; i < 100; i++) {
      System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped"));
      Thread.sleep(1000);
    }
  }
  else {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, buildTopology(null));
  }
}

Example 4

Source File: TridentMapExample.java From jstorm with Apache License 2.0

6 votes

public static void test() {
    TopologyBuilder builder = new TopologyBuilder();
    
    builder.setSpout("spout", new InOrderSpout(), 8);
    builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1"));
    
    Config conf = new Config();
    conf.setMaxSpoutPending(20);
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    
    if (isLocal) {
        drpc = new LocalDRPC();
    }
    
    try {
        JStormHelper.runTopology(buildTopology(drpc), topologyName, conf, 60, new DrpcValidator(), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}

Example 5

Source File: TestTridentTopology.java From jstorm with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception{
	if (args.length == 0) {
		System.err.println("Please input configuration file");
		System.exit(-1);
	}

	Map conf = LoadConfig.LoadConf(args[0]);	

	if (conf == null) {
		LOG.error("Failed to load config");
	} else {
		Config config = new Config();
		config.putAll(conf);
        config.setMaxSpoutPending(10);
        config.put(LoadConfig.TOPOLOGY_TYPE, "Trident");
        StormSubmitter.submitTopology("WordCount", config, buildTopology());
	}
}

Example 6

Source File: MultiStageAckingTopology.java From incubator-heron with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
  if (args.length != 1) {
    throw new RuntimeException("Please specify the name of the topology");
  }
  TopologyBuilder builder = new TopologyBuilder();

  int parallelism = 2;
  builder.setSpout("word", new AckingTestWordSpout(), parallelism);
  builder.setBolt("exclaim1", new ExclamationBolt(true), parallelism)
      .shuffleGrouping("word");
  builder.setBolt("exclaim2", new ExclamationBolt(false), parallelism)
      .shuffleGrouping("exclaim1");

  Config conf = new Config();
  conf.setDebug(true);

  // Put an arbitrary large number here if you don't want to slow the topology down
  conf.setMaxSpoutPending(1000 * 1000 * 1000);

  // To enable acking, we need to setEnableAcking true
  conf.setNumAckers(1);

  conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");

  conf.setNumWorkers(parallelism);
  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example 7

Source File: TaskHookTopology.java From incubator-heron with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
  if (args.length != 1) {
    throw new RuntimeException("Specify topology name");
  }
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("word", new AckingTestWordSpout(), 2);
  builder.setBolt("count", new CountBolt(), 2)
      .shuffleGrouping("word");

  Config conf = new Config();
  conf.setDebug(true);
  // Put an arbitrary large number here if you don't want to slow the topology down
  conf.setMaxSpoutPending(1000 * 1000 * 1000);
  // To enable acking, we need to setEnableAcking true
  conf.setNumAckers(1);
  conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");

  // Set the task hook
  List<String> taskHooks = new LinkedList<>();
  taskHooks.add("org.apache.heron.examples.TaskHookTopology$TestTaskHook");
  org.apache.heron.api.Config.setAutoTaskHooks(conf, taskHooks);

  // component resource configuration
  org.apache.heron.api.Config.setComponentRam(conf, "word", ByteAmount.fromMegabytes(512));
  org.apache.heron.api.Config.setComponentRam(conf, "count", ByteAmount.fromMegabytes(512));

  // container resource configuration
  org.apache.heron.api.Config.setContainerDiskRequested(conf, ByteAmount.fromGigabytes(2));
  org.apache.heron.api.Config.setContainerRamRequested(conf, ByteAmount.fromGigabytes(2));
  org.apache.heron.api.Config.setContainerCpuRequested(conf, 2);


  conf.setNumWorkers(2);
  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example 8

Source File: TridentFileTopology.java From storm-hdfs with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    if (args.length == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
        Thread.sleep(120 * 1000);
    }
    else if(args.length == 2) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[1], conf, buildTopology(args[0]));
    } else{
        System.out.println("Usage: TridentFileTopology <hdfs url> [topology name]");
    }
}

Example 9

Source File: TridentSequenceTopology.java From storm-hdfs with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    if (args.length == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
        Thread.sleep(120 * 1000);
    }
    else if(args.length == 2) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[1], conf, buildTopology(args[0]));
    } else{
        System.out.println("Usage: TridentFileTopology <hdfs url> [topology name]");
    }
}

Example 10

Source File: FirstStoryDetection.java From first-stories-twitter with MIT License

5 votes

private static Config createTopologyConfiguration(Properties prop,
		boolean localMode) {
	Config conf = new Config();
	List<String> dprcServers = new ArrayList<String>();
	dprcServers.add("localhost");

	conf.put(Config.DRPC_SERVERS, dprcServers);
	conf.put(Config.DRPC_PORT, 3772);
	if (!localMode)
		conf.put(Config.STORM_CLUSTER_MODE, new String("distributed"));

	conf.put("UNIQUE_WORDS_EXPECTED",
			prop.getProperty("UNIQUE_WORDS_EXPECTED"));
	conf.put("PATH_TO_OOV_FILE", prop.getProperty("PATH_TO_OOV_FILE"));
	conf.put("L", prop.getProperty("L"));
	conf.put("BucketsParallelism", prop.getProperty("BucketsParallelism"));
	conf.put("k", prop.getProperty("k"));
	conf.put("QUEUE_SIZE", prop.getProperty("QUEUE_SIZE"));
	List<String> countAggKeepFields = new ArrayList<String>();
	countAggKeepFields.add("tweet_obj");
	countAggKeepFields.add("coltweet_obj");
	conf.put("countAggKeepFields", countAggKeepFields);
	conf.put("THRESHOLD", prop.getProperty("THRESHOLD"));
	conf.put("RECENT_TWEETS_TO_COMPARE_WITH",
			prop.getProperty("RECENT_TWEETS_TO_COMPARE_WITH"));
	conf.setDebug(false);

	conf.setNumWorkers(Integer.valueOf((String) prop
			.get("NUMBER_OF_WORKERS")));
	conf.setMaxSpoutPending(50000000);
	return conf;
}

Example 11

Source File: SimpleBatchTopologyTest.java From jstorm with Apache License 2.0

5 votes

@Test
public void testSimpleBatchTopology() {
    BatchTopologyBuilder batchTopologyBuilder = new BatchTopologyBuilder("SimpleBatchTopology");
    batchTopologyBuilder.setSpout("batchSpout", new SimpleBatchTestSpout(), 1);
    batchTopologyBuilder.setBolt("batchBolt", new SimpleBatchTestBolt(), 2).shuffleGrouping("batchSpout");

    Config config = new Config();
    config.put(Config.TOPOLOGY_NAME, "SimpleBatchTopologyTest");
    config.setMaxSpoutPending(1);

    JStormUnitTestRunner.submitTopology(batchTopologyBuilder.getTopologyBuilder().createTopology(), config, 120, null);
}

Example 12

Source File: TransactionalWordsTest.java From jstorm with Apache License 2.0

5 votes

@Test
public void test_transaction_word() {
    try {
        MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
        TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2);
        builder.setBolt("count", new KeyedCountUpdater(), 5).fieldsGrouping("spout", new Fields("word"));
        builder.setBolt("bucketize", new Bucketize()).shuffleGrouping("count");
        builder.setBolt("buckets", new BucketCountUpdater(), 5).fieldsGrouping("bucketize", new Fields("bucket"));

        LocalCluster cluster = new LocalCluster();

        Config config = new Config();
        config.setDebug(true);
        config.setMaxSpoutPending(3);

        cluster.submitTopology("top-n-topology", config, builder.buildTopology());

        JStormUtils.sleepMs(60 * 1000);
        
        
        assertEquals(false, outOfOrder.get() );
        assertNotSame(0, receiveCounter1.get());
        assertNotSame(0, receiveCounter2.get());
        
        //cluster.killTopology("top-n-topology");
        //cluster.shutdown();
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Failed to run simple transaction");
    }

}

Example 13

Source File: SampleTopology.java From aws-big-data-blog with Apache License 2.0

4 votes

public static void main(String[] args) throws IllegalArgumentException, KeeperException, InterruptedException, AlreadyAliveException, InvalidTopologyException, IOException {
    String propertiesFile = null;
    String mode = null;
    
    if (args.length != 2) {
        printUsageAndExit();
    } else {
        propertiesFile = args[0];
        mode = args[1];
    }
    
    configure(propertiesFile);

    final KinesisSpoutConfig config =
            new KinesisSpoutConfig(streamName, zookeeperEndpoint).withZookeeperPrefix(zookeeperPrefix)
                    .withInitialPositionInStream(initialPositionInStream)
                    .withRegion(Regions.fromName(regionName));

    final KinesisSpout spout = new KinesisSpout(config, new CustomCredentialsProviderChain(), new ClientConfiguration());
    TopologyBuilder builder = new TopologyBuilder();
    LOG.info("Using Kinesis stream: " + config.getStreamName());


    // Using number of shards as the parallelism hint for the spout.
    builder.setSpout("Kinesis", spout, 2);
    builder.setBolt("Parse", new ParseReferrerBolt(), 6).shuffleGrouping("Kinesis");
    builder.setBolt("Count", new RollingCountBolt(5, 2,elasticCacheRedisEndpoint), 6).fieldsGrouping("Parse", new Fields("referrer"));

    //builder.setBolt("Count", new CountReferrerBolt(), 12).fieldsGrouping("Parse", new Fields("referrer"));

    Config topoConf = new Config();
    topoConf.setFallBackOnJavaSerialization(true);
    topoConf.setDebug(false);

    if (mode.equals("LocalMode")) {
        LOG.info("Starting sample storm topology in LocalMode ...");
        new LocalCluster().submitTopology("test_spout", topoConf, builder.createTopology());
    } else if (mode.equals("RemoteMode")) {
        topoConf.setNumWorkers(1);
        topoConf.setMaxSpoutPending(5000);
        LOG.info("Submitting sample topology " + topologyName + " to remote cluster.");
        StormSubmitter.submitTopology(topologyName, topoConf, builder.createTopology());            
    } else {
        printUsageAndExit();
    }

}

Example 14

Source File: Throughput.java From flink-perf with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	int par = pt.getInt("para");

	TopologyBuilder builder = new TopologyBuilder();

	builder.setSpout("source0", new Generator(pt), pt.getInt("sourceParallelism"));
	int i = 0;
	for(; i < pt.getInt("repartitions", 1) - 1;i++) {
		System.out.println("adding source"+i+" --> source"+(i+1));
		builder.setBolt("source"+(i+1), new RepartPassThroughBolt(pt), pt.getInt("sinkParallelism")).fieldsGrouping("source" + i, new Fields("id"));
	}
	System.out.println("adding final source"+i+" --> sink");

	builder.setBolt("sink", new Sink(pt), pt.getInt("sinkParallelism")).fieldsGrouping("source"+i, new Fields("id"));


	Config conf = new Config();
	conf.setDebug(false);
	//System.exit(1);
	if(pt.has("ft") || pt.has("maxPending")) {
		conf.setMaxSpoutPending(pt.getInt("maxPending", 1000));
	}

	if (!pt.has("local")) {
		conf.setNumWorkers(par);

		StormSubmitter.submitTopologyWithProgressBar("throughput-"+pt.get("name", "no_name"), conf, builder.createTopology());
	}
	else {
		conf.setMaxTaskParallelism(par);

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("throughput", conf, builder.createTopology());

		Thread.sleep(300000);

		cluster.shutdown();
	}

}

Example 15

Source File: ForwardThroughput.java From flink-perf with Apache License 2.0

3 votes

public static void main(String[] args) throws Exception {
	ParameterTool pt = ParameterTool.fromArgs(args);

	int par = pt.getInt("para");

	TopologyBuilder builder = new TopologyBuilder();

	builder.setSpout("source0", new Generator(pt), pt.getInt("sourceParallelism"));

	//builder.setBolt("sink", new Sink(pt), pt.getInt("sinkParallelism")).noneGrouping("source0");
	builder.setBolt("sink", new Sink(pt), pt.getInt("sinkParallelism")).localOrShuffleGrouping("source0");


	Config conf = new Config();
	conf.setDebug(false);

	conf.setMaxSpoutPending(pt.getInt("maxPending", 1000));
	//System.exit(1);

	if (!pt.has("local")) {
		conf.setNumWorkers(par);

		StormSubmitter.submitTopologyWithProgressBar("forward-throughput-"+pt.get("name", "no_name"), conf, builder.createTopology());
	}
	else {
		conf.setMaxTaskParallelism(par);

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("forward-throughput", conf, builder.createTopology());

		Thread.sleep(300000);

		cluster.shutdown();
	}

}