Java Code Examples for org.apache.storm.Config#setMaxTaskParallelism()

The following examples show how to use org.apache.storm.Config#setMaxTaskParallelism() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ScottyDemoTopology.java From scotty-window-processor with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    LocalCluster cluster = new LocalCluster();
    TopologyBuilder builder = new TopologyBuilder();

    Config conf = new Config();
    conf.setDebug(false);
    conf.setNumWorkers(1);
    conf.setMaxTaskParallelism(1);
    //Disable Acking
    conf.setNumAckers(0);

    KeyedScottyWindowOperator scottyBolt = new KeyedScottyWindowOperator<Integer, Integer>(new Sum(), 0);
    scottyBolt.addWindow(new TumblingWindow(WindowMeasure.Time, 1000));
    scottyBolt.addWindow(new SlidingWindow(WindowMeasure.Time, 1000, 250));
    scottyBolt.addWindow(new SessionWindow(WindowMeasure.Time, 1000));

    builder.setSpout("spout", new DataGeneratorSpout());
    builder.setBolt("scottyWindow", scottyBolt).fieldsGrouping("spout", new Fields("key"));
    builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("scottyWindow");

    cluster.submitTopology("testTopology", conf, builder.createTopology());
    //cluster.killTopology("testTopology");
    //cluster.shutdown();
}

Example 2

Source File: FullPullerTopology.java From DBus with Apache License 2.0

6 votes

private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(FullPullConstants.FULL_SPLITTER_TOPOLOGY_ID, fullSplitterTopologyId);
    conf.put(FullPullConstants.FULL_PULLER_TOPOLOGY_ID, fullPullerTopologyId);
    conf.put(FullPullConstants.DS_NAME, topologyId);
    conf.put(FullPullConstants.ZKCONNECT, zkConnect);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, commonConfSplit.getProperty(FullPullConstants.TOPOLOGY_WORKER_CHILDOPTS));
    //设置message超时时间为，保证每个分片都能在该内拉完数据
    conf.setMessageTimeoutSecs(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_MESSAGE_TIMEOUT)));
    conf.setMaxSpoutPending(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_MAX_SPOUT_PENDING)));
    conf.setNumWorkers(Integer.parseInt(commonConfSplit.getProperty(FullPullConstants.STORM_NUM_WORKERS)));
    conf.setDebug(true);

    if (runAsLocal) {
        conf.setMaxTaskParallelism(3);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}

Example 3

Source File: DBusLogProcessorTopology.java From DBus with Apache License 2.0

6 votes

private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zkConnect);
    conf.put(Constants.TOPOLOGY_ID, topologyId);
    conf.setMessageTimeoutSecs(Integer.parseInt(properties.getProperty(Constants.LOG_MESSAGE_TIMEOUT)));
    //conf.setMaxSpoutPending(30);
    conf.setDebug(true);
    conf.setNumWorkers(Integer.parseInt(properties.getProperty(Constants.LOG_NUMWORKERS)));

    if (runAsLocal) {
        conf.setMaxTaskParallelism(10);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}

Example 4

Source File: SinkTopology.java From DBus with Apache License 2.0

6 votes

private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(Constants.ZOOKEEPER_SERVERS, zkConnect);
    conf.put(Constants.TOPOLOGY_ID, topologyId);
    conf.put(Constants.SINK_TYPE, sinkType);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, sinkerConf.getProperty(SinkerConstants.TOPOLOGY_WORKER_CHILDOPTS));

    conf.setMessageTimeoutSecs(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_MESSAGE_TIMEOUT)));
    conf.setMaxSpoutPending(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_MAX_SPOUT_PENDING)));
    conf.setDebug(true);
    conf.setNumWorkers(Integer.parseInt(sinkerConf.getProperty(SinkerConstants.STORM_NUM_WORKERS)));
    if (runAsLocal) {
        conf.setMaxTaskParallelism(10);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}

Example 5

Source File: DrpcTestTopologyCsharp.java From storm-net-adapter with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
  	TopologyBuilder builder = new TopologyBuilder();
	  
  	DRPCSpout drpcSpout = new DRPCSpout("simpledrpc");
    builder.setSpout("drpc-input", drpcSpout,1);

    builder.setBolt("simple", new SimpleDRPC(), 2)
    		.noneGrouping("drpc-input");
    
    builder.setBolt("return", new ReturnResults(),1)
	.noneGrouping("simple");

    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxTaskParallelism(1);
    
    try
    {
    	StormSubmitter.submitTopology("drpc-q", conf,builder.createTopology());
    }
    catch (Exception e)
    {
    	e.printStackTrace();
    }
}

Example 6

Source File: AnchoredWordCount.java From storm-net-adapter with Apache License 2.0

6 votes

protected int run(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("spout", new RandomSentenceSpout(), 4);

    builder.setBolt("split", new SplitSentence(), 4).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), 4).fieldsGrouping("split", new Fields("word"));

    Config conf = new Config();
    conf.setMaxTaskParallelism(3);

    String topologyName = "word-count";

    conf.setNumWorkers(3);

    if (args != null && args.length > 0) {
        topologyName = args[0];
    }
    return submit(topologyName, conf, builder);
}

Example 7

Source File: StormTestUtil.java From atlas with Apache License 2.0

5 votes

public static Config submitTopology(ILocalCluster stormCluster, String topologyName,
                                    StormTopology stormTopology) throws Exception {
    Config stormConf = new Config();
    stormConf.putAll(Utils.readDefaultConfig());
    stormConf.put("storm.cluster.mode", "local");
    stormConf.setDebug(true);
    stormConf.setMaxTaskParallelism(3);
    stormConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN,
            org.apache.atlas.storm.hook.StormAtlasHook.class.getName());

    stormCluster.submitTopology(topologyName, stormConf, stormTopology);

    Thread.sleep(10000);
    return stormConf;
}

Example 8

Source File: DBusRouterTopology.java From DBus with Apache License 2.0

5 votes

private void start(StormTopology topology, boolean runAsLocal) throws Exception {
    Config conf = new Config();
    conf.put(com.creditease.dbus.commons.Constants.ZOOKEEPER_SERVERS, zkConnect);
    conf.put(Constants.TOPOLOGY_ID, topologyId);
    conf.put(Constants.TOPOLOGY_ALIAS, alias);
    conf.put(Constants.ROUTER_PROJECT_NAME, projectName);

    String workerChildOpts = routerConf.getProperty(DBusRouterConstants.STORM_TOPOLOGY_WORKER_CHILDOPTS, "-Xmx2g");
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, workerChildOpts);

    int msgTimeout = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_MESSAGE_TIMEOUT, "10"));
    conf.setMessageTimeoutSecs(msgTimeout);

    int maxSpoutPending = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_MAX_SPOUT_PENDING, "100"));
    conf.setMaxSpoutPending(maxSpoutPending);
    conf.setDebug(true);

    int numWorks = Integer.valueOf(routerConf.getProperty(DBusRouterConstants.STORM_NUM_WORKS, "1"));
    conf.setNumWorkers(numWorks);

    if (runAsLocal) {
        conf.setMaxTaskParallelism(10);
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyName, conf, topology);
    } else {
        StormSubmitter.submitTopology(topologyName, conf, topology);
    }
}

Example 9

Source File: KafkaStormWordCountTopology.java From Building-Data-Streaming-Applications-with-Apache-Kafka with MIT License

5 votes

public static void main(String[] args) throws Exception {

        String zkConnString = "localhost:2181";
        String topic = "words";
        BrokerHosts hosts = new ZkHosts(zkConnString);

        SpoutConfig kafkaSpoutConfig = new SpoutConfig(hosts, topic, "/" + topic,
                "wordcountID");
        kafkaSpoutConfig.startOffsetTime = kafka.api.OffsetRequest.EarliestTime();
        kafkaSpoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());

        TopologyBuilder topologyBuilder = new TopologyBuilder();
        topologyBuilder.setSpout("kafkaspout", new KafkaSpout(kafkaSpoutConfig));
        topologyBuilder.setBolt("stringsplit", new StringToWordsSpliterBolt()).shuffleGrouping("kafkaspout");
        topologyBuilder.setBolt("counter", new WordCountCalculatorBolt()).shuffleGrouping("stringsplit");

        Config config = new Config();
        config.setDebug(true);
        if (args != null && args.length > 1) {
            config.setNumWorkers(3);
            StormSubmitter.submitTopology(args[1], config, topologyBuilder.createTopology());
        } else {
            // Cap the maximum number of executors that can be spawned
            // for a component to 3
            config.setMaxTaskParallelism(3);
            // LocalCluster is used to run locally
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("KafkaLocal", config, topologyBuilder.createTopology());
            // sleep
            try {
                Thread.sleep(10000);
            } catch (InterruptedException e) {
                // TODO Auto-generated catch block
                cluster.killTopology("KafkaToplogy");
                cluster.shutdown();
            }

            cluster.shutdown();
        }
    }

Example 10

Source File: AppMain.java From storm_spring_boot_demo with MIT License

5 votes

/**
 * 用于debug
 * @param name
 * @param builder
 * @throws InterruptedException
 */
private static void localSubmit(String name,TopologyBuilder builder, Config conf)
        throws InterruptedException {
    conf.setDebug(true);
    conf.setMaxTaskParallelism(3);
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology(name, conf, builder.createTopology());
    Thread.sleep(100000);
    cluster.shutdown();
}

Example 11

Source File: StormTestUtil.java From incubator-atlas with Apache License 2.0

5 votes

public static Config submitTopology(ILocalCluster stormCluster, String topologyName,
                                    StormTopology stormTopology) throws Exception {
    Config stormConf = new Config();
    stormConf.putAll(Utils.readDefaultConfig());
    stormConf.put("storm.cluster.mode", "local");
    stormConf.setDebug(true);
    stormConf.setMaxTaskParallelism(3);
    stormConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN,
            org.apache.atlas.storm.hook.StormAtlasHook.class.getName());

    stormCluster.submitTopology(topologyName, stormConf, stormTopology);

    Thread.sleep(10000);
    return stormConf;
}

Example 12

Source File: ThresholdingEngineAlarmTest.java From monasca-thresh with Apache License 2.0

5 votes

@BeforeMethod
public void beforeMethod() throws Exception {
  // Fixtures
  alarmDefinitionDAO = mock(AlarmDefinitionDAO.class);

  // Mocks
  alarmDAO = new MockAlarmDAO();

  // Bindings
  Injector.reset();
  Injector.registerModules(new AbstractModule() {
    protected void configure() {
      bind(AlarmDAO.class).toInstance(alarmDAO);
      bind(AlarmDefinitionDAO.class).toInstance(alarmDefinitionDAO);
    }
  });

  // Config
  ThresholdingConfiguration threshConfig = new ThresholdingConfiguration();
  threshConfig.alarmDelay = 1;
  threshConfig.sporadicMetricNamespaces = new HashSet<String>();
  Serialization.registerTarget(KafkaProducerConfiguration.class);

  threshConfig.kafkaProducerConfig =
      Serialization
          .fromJson("{\"KafkaProducerConfiguration\":{\"topic\":\"alarm-state-transitions\",\"metadataBrokerList\":\"192.168.10.10:9092\",\"requestRequiredAcks\":1,\"requestTimeoutMs\":10000,\"producerType\":\"sync\",\"serializerClass\":\"kafka.serializer.StringEncoder\",\"keySerializerClass\":\"\",\"partitionerClass\":\"\",\"compressionCodec\":\"none\",\"compressedTopics\":\"\",\"messageSendMaxRetries\":3,\"retryBackoffMs\":100,\"topicMetadataRefreshIntervalMs\":600000,\"queueBufferingMaxMs\":5000,\"queueBufferingMaxMessages\":10000,\"queueEnqueueTimeoutMs\":-1,\"batchNumMessages\":200,\"sendBufferBytes\":102400,\"clientId\":\"Threshold_Engine\"}}");
  Config stormConfig = new Config();
  stormConfig.setMaxTaskParallelism(1);
  metricSpout = new FeederSpout(new Fields(MetricSpout.FIELDS));
  eventSpout = new FeederSpout(new Fields("event"));
  alarmEventForwarder = mock(AlarmEventForwarder.class);
  Injector
      .registerModules(new TopologyModule(threshConfig, stormConfig, metricSpout, eventSpout));
  Injector.registerModules(new ProducerModule(alarmEventForwarder));

  // Evaluate alarm stats every 5 seconds
  System.setProperty(MetricAggregationBolt.TICK_TUPLE_SECONDS_KEY, "5");

  startTopology();
}

Example 13

Source File: IPFraudDetectionTopology.java From Building-Data-Streaming-Applications-with-Apache-Kafka with MIT License

4 votes

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
    Intialize(args[0]);
    logger.info("Successfully loaded Configuration ");


    BrokerHosts hosts = new ZkHosts(zkhost);
    SpoutConfig spoutConfig = new SpoutConfig(hosts, inputTopic, "/" + KafkaBroker, consumerGroup);
    spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
    spoutConfig.startOffsetTime = kafka.api.OffsetRequest.EarliestTime();
    KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
    String[] partNames = {"status_code"};
    String[] colNames = {"date", "request_url", "protocol_type", "status_code"};

    DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper().withColumnFields(new Fields(colNames))
            .withPartitionFields(new Fields(partNames));


    HiveOptions hiveOptions;
    //make sure you change batch size and all paramtere according to requirement
    hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(250).withBatchSize(2)
            .withIdleTimeout(10).withCallTimeout(10000000);

    logger.info("Creating Storm Topology");
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("KafkaSpout", kafkaSpout, 1);

    builder.setBolt("frauddetect", new FraudDetectorBolt()).shuffleGrouping("KafkaSpout");
    builder.setBolt("KafkaOutputBolt",
            new IPFraudKafkaBolt(zkhost, "kafka.serializer.StringEncoder", KafkaBroker, outputTopic), 1)
            .shuffleGrouping("frauddetect");

    builder.setBolt("HiveOutputBolt", new IPFraudHiveBolt(), 1).shuffleGrouping("frauddetect");
    builder.setBolt("HiveBolt", new HiveBolt(hiveOptions)).shuffleGrouping("HiveOutputBolt");

    Config conf = new Config();
    if (args != null && args.length > 1) {
        conf.setNumWorkers(3);
        logger.info("Submiting  topology to storm cluster");

        StormSubmitter.submitTopology(args[1], conf, builder.createTopology());
    } else {
        // Cap the maximum number of executors that can be spawned
        // for a component to 3
        conf.setMaxTaskParallelism(3);
        // LocalCluster is used to run locally
        LocalCluster cluster = new LocalCluster();
        logger.info("Submitting  topology to local cluster");
        cluster.submitTopology("KafkaLocal", conf, builder.createTopology());
        // sleep
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            logger.error("Exception ocuured" + e);
            cluster.killTopology("KafkaToplogy");
            logger.info("Shutting down cluster");
            cluster.shutdown();
        }
        cluster.shutdown();

    }

}

Example 14

Source File: RandomNumberGeneratorSpout.java From storm-net-adapter with Apache License 2.0

4 votes

@Override
public Map<String, Object> getComponentConfiguration() {
    Config conf = new Config();
    conf.setMaxTaskParallelism(1);
    return conf;
}

Example 15

Source File: ThresholdingEngineTest.java From monasca-thresh with Apache License 2.0

4 votes

@BeforeMethod
public void befortMethod() throws Exception {
  // Fixtures
  final AlarmExpression expression =
      new AlarmExpression("max(cpu{id=5}) >= 3 or max(mem{id=5}) >= 5");
  final AlarmExpression expression2 = AlarmExpression.of(
    "count(log.error{id=5},deterministic) >= 1 OR count(log.warning{id=5},deterministic) >= 1"
  );
  final AlarmExpression expression3 = AlarmExpression.of(
    "max(cpu{id=5}) >= 3 AND count(log.warning{id=5},deterministic) >= 1"
  );

  cpuMetricDef = expression.getSubExpressions().get(0).getMetricDefinition();
  memMetricDef = expression.getSubExpressions().get(1).getMetricDefinition();
  logErrorMetricDef = expression2.getSubExpressions().get(0).getMetricDefinition();
  logWarningMetricDef = expression2.getSubExpressions().get(1).getMetricDefinition();

  extraMemMetricDefDimensions = new HashMap<>(memMetricDef.dimensions);
  extraMemMetricDefDimensions.put("Group", "group A");

  alarmDefinition =
      new AlarmDefinition(TEST_ALARM_TENANT_ID, TEST_ALARM_NAME,
          TEST_ALARM_DESCRIPTION, expression, "LOW", true, new ArrayList<String>());
  this.deterministicAlarmDefinition = new AlarmDefinition(
    DET_TEST_ALARM_TENANT_ID,
    DET_TEST_ALARM_NAME,
    DET_TEST_ALARM_DESCRIPTION,
    expression2,
    "LOW",
    true,
    new ArrayList<String>()
  );
  this.mixedAlarmDefinition = new AlarmDefinition(
    MIXED_TEST_ALARM_TENANT_ID,
    MIXED_TEST_ALARM_NAME,
    MIXED_TEST_ALARM_DESCRIPTION,
    expression3,
    "LOW",
    true,
    new ArrayList<String>()
  );

  // Mocks
  alarmDAO = mock(AlarmDAO.class);
  alarmDefinitionDAO = mock(AlarmDefinitionDAO.class);

  // Bindings
  Injector.reset();
  Injector.registerModules(new AbstractModule() {
    protected void configure() {
      bind(AlarmDAO.class).toInstance(alarmDAO);
      bind(AlarmDefinitionDAO.class).toInstance(alarmDefinitionDAO);
    }
  });

  // Config
  ThresholdingConfiguration threshConfig = new ThresholdingConfiguration();
  threshConfig.alarmDelay = 1;
  threshConfig.sporadicMetricNamespaces = new HashSet<String>();
  Serialization.registerTarget(KafkaProducerConfiguration.class);

  threshConfig.kafkaProducerConfig =
      Serialization
          .fromJson("{\"KafkaProducerConfiguration\":{\"topic\":\"alarm-state-transitions\",\"metadataBrokerList\":\"192.168.10.10:9092\",\"requestRequiredAcks\":1,\"requestTimeoutMs\":10000,\"producerType\":\"sync\",\"serializerClass\":\"kafka.serializer.StringEncoder\",\"keySerializerClass\":\"\",\"partitionerClass\":\"\",\"compressionCodec\":\"none\",\"compressedTopics\":\"\",\"messageSendMaxRetries\":3,\"retryBackoffMs\":100,\"topicMetadataRefreshIntervalMs\":600000,\"queueBufferingMaxMs\":5000,\"queueBufferingMaxMessages\":10000,\"queueEnqueueTimeoutMs\":-1,\"batchNumMessages\":200,\"sendBufferBytes\":102400,\"clientId\":\"Threshold_Engine\"}}");
  Config stormConfig = new Config();
  stormConfig.setMaxTaskParallelism(1);
  metricSpout = new FeederSpout(new Fields(MetricSpout.FIELDS));
  eventSpout = new FeederSpout(new Fields("event"));
  alarmEventForwarder = mock(AlarmEventForwarder.class);
  Injector
      .registerModules(new TopologyModule(threshConfig, stormConfig, metricSpout, eventSpout));
  Injector.registerModules(new ProducerModule(alarmEventForwarder));

}

Example 16

Source File: WordCountTopologyCsharp.java From storm-net-adapter with Apache License 2.0

3 votes

public static void main(String[] args) throws Exception {

		TopologyBuilder builder = new TopologyBuilder();

		builder.setSpout("generator", new Generator(), 1);

		builder.setBolt("splitter", new Splitter(), 5).fieldsGrouping("generator",
				new Fields("word"));
		
		builder.setBolt("counter", new Counter(), 8).fieldsGrouping("splitter",
				new Fields("word", "count"));

		Config conf = new Config();
		//conf.setDebug(true);

		if (args != null && args.length > 0) {
			conf.setNumWorkers(3);

			StormSubmitter.submitTopologyWithProgressBar(args[0], conf,
					builder.createTopology());
		} else {
			conf.setMaxTaskParallelism(3);

			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology("WordCount", conf, builder.createTopology());

			Thread.sleep(10000);

			cluster.shutdown();
		}
	}