backtype.storm.StormSubmitter Java Examples

The following examples show how to use backtype.storm.StormSubmitter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: TransactionFastWordCountWindowedState.java From jstorm with Apache License 2.0

6 votes

public static void test() throws Exception {
    TransactionTopologyBuilder builder = new TransactionTopologyBuilder();
    if (isLocal) {
        conf.put("tuple.num.per.batch", 100);
        conf.put("transaction.scheduler.spout", false);
        conf.put("transaction.exactly.cache.type", "default");
        conf.put("transaction.topology", true);
    }

    int spoutParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int splitParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1);
    int countParallelismHint = JStormUtils.parseInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 1);

    builder.setSpout("spout", new TxFastRandomSentenceSpout(), spoutParallelismHint);
    builder.setBolt("split", new TxSplitSentence(), splitParallelismHint).localOrShuffleGrouping("spout");
    WordCount wordCount = new WordCount();
    builder.setBolt("count", wordCount
                    .timeWindow(Time.seconds(60L))
                    .withTransactionStateOperator(wordCount),
            countParallelismHint).fieldsGrouping("split", new Fields("word"));
    builder.enableHdfs();

    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}

Example #2

Source File: TestTopology.java From jstorm with Apache License 2.0

6 votes

private static void submitTopology(TopologyBuilder builder) {
	try {
		if (local_mode(conf)) {

			LocalCluster cluster = new LocalCluster();

			cluster.submitTopology(
					String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());

			Thread.sleep(200000);

			cluster.shutdown();
		} else {
			StormSubmitter.submitTopology(
					String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());
		}

	} catch (Exception e) {
		LOG.error(e.getMessage(), e.getCause());
	}
}

Example #3

Source File: WordCountTopology.java From jstorm with Apache License 2.0

6 votes

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length != 1) {
        throw new RuntimeException("Specify topology name");
    }

    int parallelism = 10;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new WordSpout(), parallelism);
    builder.setBolt("consumer", new ConsumerBolt(), parallelism)
            .fieldsGrouping("word", new Fields("word"));
    Config conf = new Config();
    conf.setNumStmgrs(parallelism);
/*
Set config here
*/

    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example #4

Source File: PerformanceTestTopology.java From jstorm with Apache License 2.0

6 votes

public static void SetRemoteTopology()
        throws Exception {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}

Example #5

Source File: WordCountTrident.java From storm-hbase with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    if (args.length == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
        Thread.sleep(60 * 1000);
        cluster.killTopology("wordCounter");
        cluster.shutdown();
        System.exit(0);
    }
    else if(args.length == 2) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[1], conf, buildTopology(args[0]));
    } else{
        System.out.println("Usage: TridentFileTopology <hdfs url> [topology name]");
    }
}

Example #6

Source File: TopologyMgmtResourceImplTest.java From eagle with Apache License 2.0

6 votes

@Test
public void testStartTopology1() throws Exception {
    IMetadataDao dao = MetadataDaoFactory.getInstance().getMetadataDao();
    TopologyMgmtResourceImpl service = new TopologyMgmtResourceImpl();
    Field daoField = TopologyMgmtResourceImpl.class.getDeclaredField("dao");
    daoField.setAccessible(true);
    Field modifiersField = Field.class.getDeclaredField("modifiers");
    modifiersField.setAccessible(true);
    modifiersField.setInt(daoField, daoField.getModifiers() & ~Modifier.FINAL);
    daoField.set(null, dao);
    // set data
    Topology topology = new Topology("test", 1, 1);
    StreamingCluster cluster =new StreamingCluster();
    dao.clear();
    dao.addTopology(topology);
    dao.addCluster(cluster);
    PowerMockito.mockStatic(StormSubmitter.class);
    PowerMockito.doNothing().when(StormSubmitter.class, "submitTopology",Mockito.eq("test"), Mockito.anyMap(), Mockito.any(StormTopology.class));
    TopologyMgmtResourceImpl spy = PowerMockito.spy(service);
    PowerMockito.doReturn(null).when(spy,"createTopology", Mockito.any(Topology.class));
    spy.startTopology("test");
}

Example #7

Source File: WordCountTopology.java From incubator-heron with Apache License 2.0

6 votes

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
  if (args.length < 1) {
    throw new RuntimeException("Specify topology name");
  }

  int parallelism = 1;
  if (args.length > 1) {
    parallelism = Integer.parseInt(args[1]);
  }
  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout("word", new WordSpout(), parallelism);
  builder.setBolt("consumer", new ConsumerBolt(), parallelism)
      .fieldsGrouping("word", new Fields("word"));
  Config conf = new Config();
  conf.setNumWorkers(parallelism);

  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example #8

Source File: TestTopology.java From jstorm with Apache License 2.0

6 votes

private static void submitTopology(TopologyBuilder builder) {
	try {
		if (local_mode(conf)) {

			LocalCluster cluster = new LocalCluster();

			cluster.submitTopology(String.valueOf(conf.get("topology.name")), conf, builder.createTopology());

			Thread.sleep(200000);

			cluster.shutdown();
		} else {
			StormSubmitter.submitTopology(String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());
		}

	} catch (Exception e) {
		e.printStackTrace();
	}
}

Example #9

Source File: PerformanceTestTopology.java From jstorm with Apache License 2.0

6 votes

public static void SetRemoteTopology()
        throws Exception {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}

Example #10

Source File: TestTridentTopology.java From jstorm with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception{
	if (args.length == 0) {
		System.err.println("Please input configuration file");
		System.exit(-1);
	}

	Map conf = LoadConfig.LoadConf(args[0]);	

	if (conf == null) {
		LOG.error("Failed to load config");
	} else {
		Config config = new Config();
		config.putAll(conf);
        config.setMaxSpoutPending(10);
        config.put(LoadConfig.TOPOLOGY_TYPE, "Trident");
        StormSubmitter.submitTopology("WordCount", config, buildTopology());
	}
}

Example #11

Source File: TransactionStateTest.java From jstorm with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    Map conf = JStormHelper.getConfig(args);
    int spoutParallelism = JStormUtils.parseInt(conf.get(SPOUT_PARALLELISM_HINT), 1);
    int splitParallelism = JStormUtils.parseInt(conf.get(SPLIT_PARALLELISM_HINT), 2);
    int countParallelism = JStormUtils.parseInt(conf.get(COUNT_PARALLELISM_HINT), 2);

    TransactionTopologyBuilder builder = new TransactionTopologyBuilder();
    builder.setSpout("spout", new ScheduleTxSpout(), spoutParallelism);
    builder.setBolt("split", new TxSplitSentence(), splitParallelism).localOrShuffleGrouping("spout");
    builder.setBolt("count", new RocksDbCount(), countParallelism).fieldsGrouping("split", new Fields("word"));
    builder.enableHdfs();

    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}

Example #12

Source File: TransactionTestTopology.java From jstorm with Apache License 2.0

6 votes

public static void test() throws Exception {
    TransactionTopologyBuilder builder = new TransactionTopologyBuilder();
    if (isLocal) {
        conf.put("tuple.num.per.batch", 5);
        conf.put("transaction.scheduler.spout", false);
        conf.put("transaction.exactly.cache.type", "default");
    }

    int spoutParallelism = JStormUtils.parseInt(conf.get(SPOUT_PARALLELISM_HINT), 1);
    int splitParallelism = JStormUtils.parseInt(conf.get(SPLIT_PARALLELISM_HINT), 2);
    int countParallelism = JStormUtils.parseInt(conf.get(COUNT_PARALLELISM_HINT), 2);

    boolean isScheduleSpout = JStormUtils.parseBoolean(conf.get("transaction.scheduler.spout"), true);
    if (isScheduleSpout)
        // Generate batch by configured time. "transaction.schedule.batch.delay.ms: 1000 # 1sec"
        builder.setSpout("spout", new ScheduleTxSpout(), spoutParallelism);
    else
        // Generate batch by user when calling emitBarrier
        builder.setSpout("spout", new BasicTxSpout(), spoutParallelism, false);
    builder.setBolt("split", new TxSplitSentence(), splitParallelism).localOrShuffleGrouping("spout");
    builder.setBolt("count", new TxWordCount(), countParallelism).fieldsGrouping("split", new Fields("word"));

    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}

Example #13

Source File: SequenceTopologyTool.java From jstorm with Apache License 2.0

6 votes

public void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    Config conf = getConf();
    StormTopology topology = buildTopology();
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        streamName = "SequenceTest";
    }
    
    if (streamName.contains("zeromq")) {
        conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.zeroMq.MQContext");
        
    } else {
        conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.netty.NettyContext");
    }
    
    StormSubmitter.submitTopology(streamName, conf, topology);
    
}

Example #14

Source File: TridentWordCount.java From flink-perf with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
  Config conf = new Config();
  conf.setMaxSpoutPending(20);
  if (args.length == 0) {
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("wordCounter", conf, buildTopology(drpc));
    for (int i = 0; i < 100; i++) {
      System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped"));
      Thread.sleep(1000);
    }
  }
  else {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, buildTopology(null));
  }
}

Example #15

Source File: BatchAckerTest.java From jstorm with Apache License 2.0

6 votes

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Config conf = JStormHelper.getConfig(args);
    int spoutParallelism = JStormUtils.parseInt(conf.get(SPOUT_PARALLELISM_HINT), 1);
    int splitParallelism = JStormUtils.parseInt(conf.get(SPLIT_PARALLELISM_HINT), 2);
    int countParallelism = JStormUtils.parseInt(conf.get(COUNT_PARALLELISM_HINT), 2);
    boolean isValueSpout = JStormUtils.parseBoolean(conf.get("is.value.spout"), false);

    TransactionTopologyBuilder builder = new TransactionTopologyBuilder();
    if (isValueSpout)
        builder.setSpoutWithAck("spout", new BatchAckerValueSpout(), spoutParallelism);
    else
        builder.setSpoutWithAck("spout", new BatchAckerSpout(), spoutParallelism);
    builder.setBoltWithAck("split", new BatchAckerSplit(), splitParallelism).localOrShuffleGrouping("spout");;
    builder.setBoltWithAck("count", new BatchAckerCount(), countParallelism).fieldsGrouping("split", new Fields("word"));
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}

Example #16

Source File: JStormHelper.java From jstorm with Apache License 2.0

6 votes

public static void runTopologyRemotely(StormTopology topology, String topologyName, Config conf,
                                       int runtimeInSeconds, Callback callback) throws Exception {
    if (conf.get(Config.TOPOLOGY_WORKERS) == null) {
        conf.setNumWorkers(3);
    }

    StormSubmitter.submitTopology(topologyName, conf, topology);

    if (JStormUtils.parseBoolean(conf.get("RUN_LONG_TIME"), false)) {
        LOG.info(topologyName + " will run long time");
        return;
    }

    if (runtimeInSeconds < 120) {
        JStormUtils.sleepMs(120 * 1000);
    } else {
        JStormUtils.sleepMs(runtimeInSeconds * 1000);
    }

    if (callback != null) {
        callback.execute(topologyName);
    }

    killTopology(conf, topologyName);
}

Example #17

Source File: Skeleton.java From trident-tutorial with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();


    if (args.length == 2) {
        // Ready & submit the topology
        String name = args[0];
        BrokerHosts hosts = new ZkHosts(args[1]);
        TransactionalTridentKafkaSpout kafkaSpout = TestUtils.testTweetSpout(hosts);

        StormSubmitter.submitTopology(name, conf, buildTopology(kafkaSpout));

    }else{
        System.err.println("<topologyName> <zookeeperHost>");
    }

}

Example #18

Source File: ClusterTestTopology.java From trident-tutorial with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();

    // Submits the topology
    String topologyName = args[0];
    conf.setNumWorkers(8); // Our Vagrant environment has 8 workers

    FakeTweetsBatchSpout fakeTweets = new FakeTweetsBatchSpout(10);

    TridentTopology topology = new TridentTopology();
    TridentState countState =
            topology
                    .newStream("spout", fakeTweets)
                    .groupBy(new Fields("actor"))
                    .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count"));

    topology
            .newDRPCStream("count_per_actor")
            .stateQuery(countState, new Fields("args"), new MapGet(), new Fields("count"));

    StormSubmitter.submitTopology(topologyName, conf, topology.build());

}

Example #19

Source File: TopHashtagFollowerCountGrouping.java From trident-tutorial with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setNumWorkers(6);

    if (args.length == 2) {
        // Ready & submit the topology
        String name = args[0];
        BrokerHosts hosts = new ZkHosts(args[1]);
        TransactionalTridentKafkaSpout kafkaSpout = TestUtils.testTweetSpout(hosts);

        StormSubmitter.submitTopology(name, conf, buildTopology(kafkaSpout));

    }else{
        System.err.println("<topologyName> <zookeeperHost>");
    }

}

Example #20

Source File: TopHashtagByCountry.java From trident-tutorial with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();


    if (args.length == 2) {
        // Ready & submit the topology
        String name = args[0];
        BrokerHosts hosts = new ZkHosts(args[1]);
        TransactionalTridentKafkaSpout kafkaSpout = TestUtils.testTweetSpout(hosts);

        StormSubmitter.submitTopology(name, conf, buildTopology(kafkaSpout));

    }else{
        System.err.println("<topologyName> <zookeeperHost>");
    }

}

Example #21

Source File: RealTimeTextSearch.java From trident-tutorial with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();


    if (args.length == 2) {
        // Ready & submit the topology
        String name = args[0];
        BrokerHosts hosts = new ZkHosts(args[1]);
        TransactionalTridentKafkaSpout kafkaSpout = TestUtils.testTweetSpout(hosts);

        StormSubmitter.submitTopology(name, conf, buildTopology(kafkaSpout));

    }else{
        System.err.println("<topologyName> <zookeeperHost>");
    }

}

Example #22

Source File: JoinExample.java From trident-tutorial with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();

    if (args.length == 2) {
        // Ready & submit the topology
        String name = args[0];
        BrokerHosts hosts = new ZkHosts(args[1]);
        TransactionalTridentKafkaSpout kafkaSpout = TestUtils.testTweetSpout(hosts);

        StormSubmitter.submitTopology(name, conf, buildTopology(kafkaSpout));

    }else{
        System.err.println("<topologyName> <zookeeperHost>");
    }

}

Example #23

Source File: GlobalTop20Hashtags.java From trident-tutorial with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();

    if (args.length == 2) {
        // Ready & submit the topology
        String name = args[0];
        BrokerHosts hosts = new ZkHosts(args[1]);
        TransactionalTridentKafkaSpout kafkaSpout = TestUtils.testTweetSpout(hosts);

        StormSubmitter.submitTopology(name, conf, buildTopology(kafkaSpout));

    }else{
        System.err.println("<topologyName> <zookeeperHost>");
    }

}

Example #24

Source File: TopHashtagByFollowerClass.java From trident-tutorial with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();

    if (args.length == 2) {
        // Ready & submit the topology
        String name = args[0];
        BrokerHosts hosts = new ZkHosts(args[1]);
        TransactionalTridentKafkaSpout kafkaSpout = TestUtils.testTweetSpout(hosts);

        StormSubmitter.submitTopology(name, conf, buildTopology(kafkaSpout));

    }else{
        System.err.println("<topologyName> <zookeeperHost>");
    }

}

Example #25

Source File: SubmitTopologyHelper.java From galaxy-sdk-java with Apache License 2.0

6 votes

public static void submitTopology(StormTopology stormTopology, Map topologyConfig) throws Exception {
    // setup StormTopology

    Config submitConfig = new Config();

    // set the configuration for topology
    submitConfig.put(Config.TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS, 5000);
    submitConfig.put(Config.TOPOLOGY_ACKER_EXECUTORS, 100);
    submitConfig.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);

    // set the worker process number
    submitConfig.setNumWorkers(ConfigHelper.getInt(topologyConfig, ConfigKeys.STORM_WORKER_NUMBER));

    // get topologyName adn clusterMode;
    String topologyName = ConfigHelper.getString(topologyConfig, ConfigKeys.STORM_TOPOLOGY_NAME);
    String clusterMode = ConfigHelper.getString(topologyConfig, ConfigKeys.STORM_CLUSTER_MODE);

    if (clusterMode.equals("local")) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("storm-emq", submitConfig, stormTopology);
    } else {
        submitConfig.put(Config.NIMBUS_HOST, ConfigHelper.getString(topologyConfig, ConfigKeys.STORM_NIMBUS_HOSTNAME));
        StormSubmitter.submitTopology(topologyName, submitConfig, stormTopology);
    }

}

Example #26

Source File: PerformanceTestTopology.java From jstorm with Apache License 2.0

6 votes

public static void SetRemoteTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}

Example #27

Source File: TopologyMgmtResourceImpl.java From eagle with Apache License 2.0

5 votes

public void startTopology(String topologyName) throws Exception {
    Optional<Topology> tdop = TopologyMgmtResourceHelper.findById(dao.listTopologies(), topologyName);
    Topology topologyDef;
    if (tdop.isPresent()) {
        topologyDef = tdop.get();
    } else {
        topologyDef = new Topology();
        topologyDef.setName(topologyName);
    }
    StormSubmitter.submitTopology(topologyName, getStormConf(null, topologyDef.getClusterName()), createTopology(topologyDef));
}

Example #28

Source File: TaskHookTopology.java From incubator-heron with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
  if (args.length != 1) {
    throw new RuntimeException("Specify topology name");
  }
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("word", new AckingTestWordSpout(), 2);
  builder.setBolt("count", new CountBolt(), 2)
      .shuffleGrouping("word");

  Config conf = new Config();
  conf.setDebug(true);
  // Put an arbitrary large number here if you don't want to slow the topology down
  conf.setMaxSpoutPending(1000 * 1000 * 1000);
  // To enable acking, we need to setEnableAcking true
  conf.setNumAckers(1);
  conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");

  // Set the task hook
  List<String> taskHooks = new LinkedList<>();
  taskHooks.add("org.apache.heron.examples.TaskHookTopology$TestTaskHook");
  org.apache.heron.api.Config.setAutoTaskHooks(conf, taskHooks);

  // component resource configuration
  org.apache.heron.api.Config.setComponentRam(conf, "word", ByteAmount.fromMegabytes(512));
  org.apache.heron.api.Config.setComponentRam(conf, "count", ByteAmount.fromMegabytes(512));

  // container resource configuration
  org.apache.heron.api.Config.setContainerDiskRequested(conf, ByteAmount.fromGigabytes(2));
  org.apache.heron.api.Config.setContainerRamRequested(conf, ByteAmount.fromGigabytes(2));
  org.apache.heron.api.Config.setContainerCpuRequested(conf, 2);


  conf.setNumWorkers(2);
  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example #29

Source File: TridentFileTopology.java From storm-hdfs with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    if (args.length == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
        Thread.sleep(120 * 1000);
    }
    else if(args.length == 2) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[1], conf, buildTopology(args[0]));
    } else{
        System.out.println("Usage: TridentFileTopology <hdfs url> [topology name]");
    }
}

Example #30

Source File: UnitTopologyRunner.java From eagle with Apache License 2.0

5 votes

private void run(String topologyId,
                 int numOfTotalWorkers,
                 int numOfSpoutTasks,
                 int numOfRouterBolts,
                 int numOfAlertBolts,
                 int numOfPublishExecutors,
                 int numOfPublishTasks,
                 Config config,
                 boolean localMode) {

    backtype.storm.Config stormConfig = givenStormConfig == null ? new backtype.storm.Config() : givenStormConfig;
    // TODO: Configurable metric consumer instance number

    int messageTimeoutSecs = config.hasPath(MESSAGE_TIMEOUT_SECS) ? config.getInt(MESSAGE_TIMEOUT_SECS) : DEFAULT_MESSAGE_TIMEOUT_SECS;
    LOG.info("Set topology.message.timeout.secs as {}", messageTimeoutSecs);
    stormConfig.setMessageTimeoutSecs(messageTimeoutSecs);

    if (config.hasPath("metric")) {
        stormConfig.registerMetricsConsumer(StormMetricTaggedConsumer.class, config.root().render(ConfigRenderOptions.concise()), 1);
    }

    stormConfig.setNumWorkers(numOfTotalWorkers);
    StormTopology topology = buildTopology(topologyId, numOfSpoutTasks, numOfRouterBolts, numOfAlertBolts, numOfPublishExecutors, numOfPublishTasks, config).createTopology();

    if (localMode) {
        LOG.info("Submitting as local mode");
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyId, stormConfig, topology);
        Utils.sleep(Long.MAX_VALUE);
    } else {
        LOG.info("Submitting as cluster mode");
        try {
            StormSubmitter.submitTopologyWithProgressBar(topologyId, stormConfig, topology);
        } catch (Exception ex) {
            LOG.error("fail submitting topology {}", topology, ex);
            throw new IllegalStateException(ex);
        }
    }
}