Java Code Examples for backtype.storm.StormSubmitter

The following are top voted examples for showing how to use backtype.storm.StormSubmitter. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: RealEstate-Streaming   File: PhoenixTest.java   Source Code and License 7 votes vote down vote up
public void buildAndSubmit() throws Exception {
	 TopologyBuilder builder = new TopologyBuilder();
	 Config config = new Config();
     config.setDebug(true);
     // String nimbusHost = topologyConfig.getProperty("nimbus.host");
     config.put(Config.NIMBUS_HOST, "localhost");
     
     configureKafkaSpout(builder);
     //configureRouteBolt(builder);
     configurePhoenixTest(builder);
     
     /*
     builder.setBolt("submitter", new SubmitBolt())
        .shuffleGrouping(ROUTE_BOLT);
     */
     
     try {
         StormSubmitter.submitTopology("simple-topology", config, builder.createTopology());
     } catch (Exception e) {
         LOG.error("Error submiting Topology", e);
     }

}
 
Example 2
Project: LearnStorm   File: ReachTopology.java   Source Code and License 7 votes vote down vote up
public static void main(String[] args) throws Exception {
  LinearDRPCTopologyBuilder builder = construct();


  Config conf = new Config();

  if (args == null || args.length == 0) {
    conf.setMaxTaskParallelism(3);
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("reach-drpc", conf, builder.createLocalTopology(drpc));

    String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
    for (String url : urlsToTry) {
      System.out.println("Reach of " + url + ": " + drpc.execute("reach", url));
    }

    cluster.shutdown();
    drpc.shutdown();
  }
  else {
    conf.setNumWorkers(6);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createRemoteTopology());
  }
}
 
Example 3
Project: storm-hbase-1.0.x   File: WordCountTrident.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    if (args.length == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
        Thread.sleep(60 * 1000);
        cluster.killTopology("wordCounter");
        cluster.shutdown();
        System.exit(0);
    }
    else if(args.length == 2) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[1], conf, buildTopology(args[0]));
    } else{
        System.out.println("Usage: TridentFileTopology <hdfs url> [topology name]");
    }
}
 
Example 4
Project: preliminary.demo   File: RaceTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {

        Config conf = new Config();
        int spout_Parallelism_hint = 1;
        int split_Parallelism_hint = 2;
        int count_Parallelism_hint = 2;

        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout("spout", new RaceSentenceSpout(), spout_Parallelism_hint);
        builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout");
        builder.setBolt("count", new WordCount(), count_Parallelism_hint).fieldsGrouping("split", new Fields("word"));
        String topologyName = RaceConfig.JstormTopologyName;

        try {
            StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
            //begin by Young
            
            //end by Young
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }
 
Example 5
Project: RealEstate-Streaming   File: KafkaPhoenixTopology.java   Source Code and License 6 votes vote down vote up
public void buildAndSubmit() throws Exception {
	 TopologyBuilder builder = new TopologyBuilder();
	 Config config = new Config();
     config.setDebug(true);
     // String nimbusHost = topologyConfig.getProperty("nimbus.host");
     config.put(Config.NIMBUS_HOST, "localhost");
     
     configureKafkaSpout(builder);
     configureRouteBolt(builder);
     configureInsertBolt(builder);
     
     //builder.setBolt("submitter", new SubmitBolt())
     //   .shuffleGrouping(ROUTE_BOLT);
     
     try {
         StormSubmitter.submitTopology("realestate-topology", config, builder.createTopology());
     } catch (Exception e) {
         LOG.error("Error submiting Topology", e);
     }

}
 
Example 6
Project: storm-demo   File: LogStatisticsTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) {
    Config config = new Config();

    HdfsBolt hdfsBolt = makeHdfsBolt();
    KafkaSpout kafkaSpout = makeKafkaSpout(TOPIC, TOPOLOGY_NAME);

    LOG.info("Topology name is {}", TOPOLOGY_NAME);

    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout(KAFKA_SPOUT_ID, kafkaSpout, 10);
    topologyBuilder.setBolt(CROP_BOLT_ID, new CropBolt(), 10).shuffleGrouping(KAFKA_SPOUT_ID);
    topologyBuilder.setBolt(SPLIT_FIELDS_BOLT_ID, new SplitFieldsBolt(), 10).shuffleGrouping(CROP_BOLT_ID);
    topologyBuilder.setBolt(STORM_HDFS_BOLT_ID, hdfsBolt, 4).fieldsGrouping(SPLIT_FIELDS_BOLT_ID, new Fields("timestamp", "fieldvalues"));

    if (args != null && args.length > 0) {
        config.setDebug(false);
        config.setNumWorkers(3);

        try {
            StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology());
        } catch (InvalidTopologyException | AlreadyAliveException | AuthorizationException e) {
            e.printStackTrace();
        }
    }
}
 
Example 7
Project: jstorm-0.9.6.3-   File: SequenceTopologyTool.java   Source Code and License 6 votes vote down vote up
public  void SetRemoteTopology() throws AlreadyAliveException,
		InvalidTopologyException, TopologyAssignException {
	 Config conf = getConf();
	StormTopology topology = buildTopology();

	conf.put(Config.STORM_CLUSTER_MODE, "distributed");
	String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
	if (streamName == null) {
		streamName = "SequenceTest";
	}

	if (streamName.contains("zeromq")) {
		conf.put(Config.STORM_MESSAGING_TRANSPORT,
				"com.alibaba.jstorm.message.zeroMq.MQContext");

	} else {
		conf.put(Config.STORM_MESSAGING_TRANSPORT,
				"com.alibaba.jstorm.message.netty.NettyContext");
	}

	StormSubmitter.submitTopology(streamName, conf,topology);
	
}
 
Example 8
Project: jstorm-0.9.6.3-   File: TransactionalGlobalCount.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	    MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
	    TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 2);
	    builder.setBolt("partial-count", new BatchCount(), 3).noneGrouping("spout");
	    builder.setBolt("sum", new UpdateGlobalCount(), 1).globalGrouping("partial-count");

//	    LocalCluster cluster = new LocalCluster();

	    Config config = new Config();
	    config.setDebug(true);
	    config.setMaxSpoutPending(3);
	    config.put(Config.TOPOLOGY_WORKERS, 9);
	    Config.setNumAckers(config, 0);
	    
	    StormSubmitter.submitTopology("global-count-topology", config, builder.buildTopology());

//	    Thread.sleep(3000);
//	    cluster.shutdown();
	  }
 
Example 9
Project: jstorm-0.9.6.3-   File: TestTopology.java   Source Code and License 6 votes vote down vote up
private static void submitTopology(TopologyBuilder builder) {
	try {
		if (local_mode(conf)) {

			LocalCluster cluster = new LocalCluster();

			cluster.submitTopology(
					String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());

			Thread.sleep(200000);

			cluster.shutdown();
		} else {
			StormSubmitter.submitTopology(
					String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());
		}

	} catch (Exception e) {
		LOG.error(e.getMessage(), e.getCause());
	}
}
 
Example 10
Project: web-crawler   File: WebCrawlerTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {

        if(args.length < 1){
            System.err.println("[ERROR] Configuration File Required");
        }
        Config conf = new Config();

        // Store all the configuration in the Storm conf object
        conf.putAll(readConfigFile(args[0]));

        //Second arg should be local in order to run locally
        if(args.length  < 2 || (args.length  == 2 && !args[1].equals("local"))) {
            StormSubmitter.submitTopologyWithProgressBar("crawler_topology", conf, buildTopology(conf, null));
        }
        else {
            LocalDRPC drpc = new LocalDRPC();
            LocalCluster localcluster = new LocalCluster();
            localcluster.submitTopology("crawler_topology",conf,buildTopology(conf, drpc));

            String searchQuery = "elasticsearch";
            System.out.println("---* Result (search): " + drpc.execute("search",  searchQuery));
        }
    }
 
Example 11
Project: learn_jstorm   File: SequenceTopologyTool.java   Source Code and License 6 votes vote down vote up
public  void SetRemoteTopology() throws AlreadyAliveException,
		InvalidTopologyException, TopologyAssignException {
	 Config conf = getConf();
	StormTopology topology = buildTopology();

	conf.put(Config.STORM_CLUSTER_MODE, "distributed");
	String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
	if (streamName == null) {
		streamName = "SequenceTest";
	}

	if (streamName.contains("zeromq")) {
		conf.put(Config.STORM_MESSAGING_TRANSPORT,
				"com.alibaba.jstorm.message.zeroMq.MQContext");

	} else {
		conf.put(Config.STORM_MESSAGING_TRANSPORT,
				"com.alibaba.jstorm.message.netty.NettyContext");
	}

	StormSubmitter.submitTopology(streamName, conf,topology);
	
}
 
Example 12
Project: learn_jstorm   File: TransactionalGlobalCount.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	    MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
	    TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 2);
	    builder.setBolt("partial-count", new BatchCount(), 3).noneGrouping("spout");
	    builder.setBolt("sum", new UpdateGlobalCount(), 1).globalGrouping("partial-count");

//	    LocalCluster cluster = new LocalCluster();

	    Config config = new Config();
	    config.setDebug(true);
	    config.setMaxSpoutPending(3);
	    config.put(Config.TOPOLOGY_WORKERS, 9);
	    Config.setNumAckers(config, 0);
	    
	    StormSubmitter.submitTopology("global-count-topology", config, builder.buildTopology());

//	    Thread.sleep(3000);
//	    cluster.shutdown();
	  }
 
Example 13
Project: learn_jstorm   File: TestTopology.java   Source Code and License 6 votes vote down vote up
private static void submitTopology(TopologyBuilder builder) {
	try {
		if (local_mode(conf)) {

			LocalCluster cluster = new LocalCluster();

			cluster.submitTopology(
					String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());

			Thread.sleep(200000);

			cluster.shutdown();
		} else {
			StormSubmitter.submitTopology(
					String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());
		}

	} catch (Exception e) {
		LOG.error(e.getMessage(), e.getCause());
	}
}
 
Example 14
Project: alfresco-apache-storm-demo   File: ConfigurableTopology.java   Source Code and License 6 votes vote down vote up
protected int submit(String name, Config conf, TopologyBuilder builder) {

        // register Metadata for serialization with FieldsSerializer
        Config.registerSerialization(conf, Metadata.class);

        if (isLocal) {
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology(name, conf, builder.createTopology());
            if (ttl != -1) {
                Utils.sleep(ttl * 1000);
                cluster.shutdown();
            }
        }

        else {
            try {
                StormSubmitter.submitTopology(name, conf,
                        builder.createTopology());
            } catch (Exception e) {
                e.printStackTrace();
                return -1;
            }
        }
        return 0;
    }
 
Example 15
Project: big-data-system   File: TridentWordCount.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  Config conf = new Config();
  conf.setMaxSpoutPending(20);
  if (args.length == 0) {
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("wordCounter", conf, buildTopology(drpc));
    for (int i = 0; i < 100; i++) {
      System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped"));
      Thread.sleep(1000);
    }
  }
  else {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, buildTopology(null));
  }
}
 
Example 16
Project: big-data-system   File: ReachTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  LinearDRPCTopologyBuilder builder = construct();


  Config conf = new Config();

  if (args == null || args.length == 0) {
    conf.setMaxTaskParallelism(3);
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("reach-drpc", conf, builder.createLocalTopology(drpc));

    String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
    for (String url : urlsToTry) {
      System.out.println("Reach of " + url + ": " + drpc.execute("reach", url));
    }

    cluster.shutdown();
    drpc.shutdown();
  }
  else {
    conf.setNumWorkers(6);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createRemoteTopology());
  }
}
 
Example 17
Project: big-data-system   File: ExclamationTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("word", new TestWordSpout(), 10);
  builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
  builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");

  Config conf = new Config();
  conf.setDebug(true);

  if (args != null && args.length > 0) {
    conf.setNumWorkers(3);

    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
  }
  else {

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    cluster.killTopology("test");
    cluster.shutdown();
  }
}
 
Example 18
Project: big-data-system   File: BasicDRPCTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation");
  builder.addBolt(new ExclaimBolt(), 3);

  Config conf = new Config();

  if (args == null || args.length == 0) {
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();

    cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc));

    for (String word : new String[]{ "hello", "goodbye" }) {
      System.out.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word));
    }

    cluster.shutdown();
    drpc.shutdown();
  }
  else {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createRemoteTopology());
  }
}
 
Example 19
Project: jstrom   File: SequenceTopologyTool.java   Source Code and License 6 votes vote down vote up
public  void SetRemoteTopology() throws AlreadyAliveException,
		InvalidTopologyException, TopologyAssignException {
	 Config conf = getConf();
	StormTopology topology = buildTopology();

	conf.put(Config.STORM_CLUSTER_MODE, "distributed");
	String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
	if (streamName == null) {
		streamName = "SequenceTest";
	}

	if (streamName.contains("zeromq")) {
		conf.put(Config.STORM_MESSAGING_TRANSPORT,
				"com.alibaba.jstorm.message.zeroMq.MQContext");

	} else {
		conf.put(Config.STORM_MESSAGING_TRANSPORT,
				"com.alibaba.jstorm.message.netty.NettyContext");
	}

	StormSubmitter.submitTopology(streamName, conf,topology);
	
}
 
Example 20
Project: jstrom   File: TestTopology.java   Source Code and License 6 votes vote down vote up
private static void submitTopology(TopologyBuilder builder) {
	try {
		if (local_mode(conf)) {

			LocalCluster cluster = new LocalCluster();

			cluster.submitTopology(
					String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());

			Thread.sleep(200000);

			cluster.shutdown();
		} else {
			StormSubmitter.submitTopology(
					String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());
		}

	} catch (Exception e) {
		LOG.error(e.getMessage(), e.getCause());
	}
}
 
Example 21
Project: jstrom   File: TestTopology.java   Source Code and License 6 votes vote down vote up
private static void submitTopology(TopologyBuilder builder) {
	try {
		if (local_mode(conf)) {

			LocalCluster cluster = new LocalCluster();

			cluster.submitTopology(String.valueOf(conf.get("topology.name")), conf, builder.createTopology());

			Thread.sleep(200000);

			cluster.shutdown();
		} else {
			StormSubmitter.submitTopology(String.valueOf(conf.get("topology.name")), conf,
					builder.createTopology());
		}

	} catch (Exception e) {
		e.printStackTrace();
	}
}
 
Example 22
Project: Infrastructure   File: AbstractTopology.java   Source Code and License 6 votes vote down vote up
/**
 * Creates a standalone topology.
 * 
 * @param args the topology arguments
 * @param topo the topology instance
 * @throws Exception in case of creation problems
 */
public static void main(String[] args, AbstractTopology topo) throws Exception {
    Config config = new Config();
    config.setMessageTimeoutSecs(100);
    PipelineOptions options = new PipelineOptions(args);
    RecordingTopologyBuilder b = new RecordingTopologyBuilder(options);
    topo.createTopology(config, b);
    
    // main topology: int numWorkers = options.getNumberOfWorkers(2);
    options.toConf(config);
    
    if (args != null && args.length > 0) {
        config.setNumWorkers(2);
        StormSubmitter.submitTopology(args[0], config, b.createTopology());
    } else {
        config.setMaxTaskParallelism(2);
        final LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("testGenPip", config, b.createTopology());
    }
}
 
Example 23
Project: Infrastructure   File: Topology.java   Source Code and License 6 votes vote down vote up
/**
 * Creates a standalone topology.
 * 
 * @param args the topology arguments
 * @throws Exception in case of creation problems
 */
public static void main(String[] args) throws Exception {
    Config config = new Config();
    config.setMessageTimeoutSecs(100);
    PipelineOptions options = new PipelineOptions(args);
    RecordingTopologyBuilder b = new RecordingTopologyBuilder(options);
    createTopology(b);
    b.close(args[0], config);
    
    // main topology: int numWorkers = options.getNumberOfWorkers(2);
    options.toConf(config);
    
    if (args != null && args.length > 0) {
        config.setNumWorkers(2);
        StormSubmitter.submitTopology(args[0], config, b.createTopology());
    } else {
        config.setMaxTaskParallelism(2);
        final LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(Naming.PIPELINE_NAME, config, b.createTopology());
    }
}
 
Example 24
Project: Infrastructure   File: TestTopology.java   Source Code and License 6 votes vote down vote up
/**
 * Creates a standalone topology.
 * 
 * @param args the topology arguments
 * @throws Exception in case of creation problems
 */
public static void main(String[] args) throws Exception {
    Config config = new Config();
    Naming.setDefaultInitializeAlgorithms(config, defaultInitAlgorithms);
    config.setMessageTimeoutSecs(100);
    PipelineOptions options = new PipelineOptions(args);
    RecordingTopologyBuilder b = new RecordingTopologyBuilder(options);
    createTopology(b);
    b.close(args[0], config);
    
    // main topology: int numWorkers = options.getNumberOfWorkers(2);
    options.toConf(config);
    
    if (args != null && args.length > 0) {
        config.setNumWorkers(2);
        StormSubmitter.submitTopology(args[0], config, b.createTopology());
    } else {
        config.setMaxTaskParallelism(2);
        final LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(PIP_NAME, config, b.createTopology());
    }
}
 
Example 25
Project: Infrastructure   File: TestTopology.java   Source Code and License 6 votes vote down vote up
/**
 * Main method.
 * @param args the arguments
 */
public static void main(String[] args) {
  //create the main topology.
    options = new PipelineOptions(args);
    MainTopologyCreator topoCreator = new MainTopologyCreator();
    TopologyOutput topo = topoCreator.createMainTopology();
    //get the topology information
    config = topo.getConfig();
    TopologyBuilder builder = topo.getBuilder();
    int defNumWorkers = topo.getNumWorkers();
    options.toConf(config);
    
    
    if (args != null && args.length > 0) {
        config.setNumWorkers(defNumWorkers);
        try {
            StormSubmitter.submitTopology(args[0], config, builder.createTopology());
        } catch (AlreadyAliveException | InvalidTopologyException e) {
            e.printStackTrace();
        }
    } else {
        final LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
    }
}
 
Example 26
Project: storm-lib   File: FunctionTopology.java   Source Code and License 6 votes vote down vote up
void run(String[] args) {
TopologyBuilder builder = new TopologyBuilder();

builder.setSpout("word_spout", new TestWordSpout(), 2);
builder.setBolt("tuple_double", new Function(new Double(), new Fields("word1", "word2")), 2).shuffleGrouping("word_spout");
	
Map conf = new HashMap();
conf.put(Config.TOPOLOGY_WORKERS, 4);
conf.put(Config.TOPOLOGY_DEBUG, true);
	
if(args.length==0) {
           LocalCluster cluster = new LocalCluster();
           cluster.submitTopology("MockIngest", conf, builder.createTopology());
       } else {
    try {
	StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } catch(Exception e) {
	e.printStackTrace();
    }
}
   }
 
Example 27
Project: storm-lib   File: BatchMutationTopology.java   Source Code and License 6 votes vote down vote up
void run(String[] args) {
TopologyBuilder builder = new TopologyBuilder();

KVSchema names = new KVSchema("names", "word");
names.colFam.addStatic("NAME");
names.colQual.add("word").addStatic("\u0000").add("word");

builder.setSpout("word_spout", new TestWordSpout(), 2);
builder.setBolt("mutate", new BatchMutation(names, true), 2).shuffleGrouping("word_spout");
	
Map conf = new HashMap();
conf.put(Config.TOPOLOGY_WORKERS, 4);
conf.put(Config.TOPOLOGY_DEBUG, true);
	
if(args.length==0) {
           LocalCluster cluster = new LocalCluster();
           cluster.submitTopology("MockIngest", conf, builder.createTopology());
       } else {
    try {
	StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } catch(Exception e) {
	e.printStackTrace();
    }
}
   }
 
Example 28
Project: cdh-storm   File: TridentWordCount.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  Config conf = new Config();
  conf.setMaxSpoutPending(20);
  if (args.length == 0) {
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("wordCounter", conf, buildTopology(drpc));
    for (int i = 0; i < 100; i++) {
      System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped"));
      Thread.sleep(1000);
    }
  }
  else {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, buildTopology(null));
  }
}
 
Example 29
Project: cdh-storm   File: ReachTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  LinearDRPCTopologyBuilder builder = construct();


  Config conf = new Config();

  if (args == null || args.length == 0) {
    conf.setMaxTaskParallelism(3);
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("reach-drpc", conf, builder.createLocalTopology(drpc));

    String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
    for (String url : urlsToTry) {
      System.out.println("Reach of " + url + ": " + drpc.execute("reach", url));
    }

    cluster.shutdown();
    drpc.shutdown();
  }
  else {
    conf.setNumWorkers(6);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createRemoteTopology());
  }
}
 
Example 30
Project: cdh-storm   File: ExclamationTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("word", new TestWordSpout(), 10);
  builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
  builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");

  Config conf = new Config();
  conf.setDebug(true);

  if (args != null && args.length > 0) {
    conf.setNumWorkers(3);

    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
  }
  else {

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    cluster.killTopology("test");
    cluster.shutdown();
  }
}
 
Example 31
Project: cdh-storm   File: BasicDRPCTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation");
  builder.addBolt(new ExclaimBolt(), 3);

  Config conf = new Config();

  if (args == null || args.length == 0) {
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();

    cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc));

    for (String word : new String[]{ "hello", "goodbye" }) {
      System.out.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word));
    }

    cluster.shutdown();
    drpc.shutdown();
  }
  else {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createRemoteTopology());
  }
}
 
Example 32
Project: LearnStorm   File: LogAnalyzer.java   Source Code and License 6 votes vote down vote up
private void buildAndSubmit() throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
		final int numWorkers = Integer.valueOf(topologyConfig.getProperty("num.workers"));
		Config config = new Config();
		config.setDebug(DEBUG);
		config.setNumWorkers(numWorkers);
		config.setMaxSpoutPending(1000000);
		// https://github.com/apache/storm/tree/v0.10.0/external/storm-kafka
		config.setMessageTimeoutSecs(600);	// This value(30 secs by default) must
							// be larger than retryDelayMaxMs
							// (60 secs by default) in
							/// KafkaSpout.

		TopologyBuilder builder = new TopologyBuilder();
		configureKafkaSpout(builder, config);
		configureESBolts(builder, config);

//		LocalCluster cluster = new LocalCluster();
		StormSubmitter.submitTopology("LogAnalyzerV1", config, builder.createTopology());
	}
 
Example 33
Project: LearnStorm   File: ApLogAnalyzer.java   Source Code and License 6 votes vote down vote up
private void buildAndSubmit() throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
		final int numWorkers = Integer.valueOf(topologyConfig.getProperty("num.workers"));
		Config config = new Config();
		config.setDebug(DEBUG);
		config.setNumWorkers(numWorkers);
		config.setMaxSpoutPending(1000000);
		// https://github.com/apache/storm/tree/v0.10.0/external/storm-kafka
		config.setMessageTimeoutSecs(600);	// This value(30 secs by default) must
							// be larger than retryDelayMaxMs
							// (60 secs by default) in
							// KafkaSpout.
		TopologyBuilder builder = new TopologyBuilder();
		configureKafkaSpout(builder, config);
		configureESBolts(builder, config);
//		configureHBaseBolts(builder, config);

//		conf.put(Config.NIMBUS_HOST, "hdp01.localdomain");
//		System.setProperty("storm.jar", "/root/workspace//LearnStorm/target/LearnStorm-0.0.1-SNAPSHOT.jar");
//		System.setProperty("hadoop.home.dir", "/tmp");
//		LocalCluster cluster = new LocalCluster();
		StormSubmitter.submitTopology("ApLogAnalyzerV1", config, builder.createTopology());
	}
 
Example 34
Project: LearnStorm   File: DRPCTest.java   Source Code and License 6 votes vote down vote up
public static void main(String args[]) throws Exception {
	LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation");
	builder.addBolt(new ExclaimBolt(), 3);

	Config conf = new Config();

	if (args == null || args.length == 0) {
		LocalDRPC drpc = new LocalDRPC();
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc));
		for (String word : new String[] { "hello", "goodbye" }) {
			System.out.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word));
		}

		Thread.sleep(10000);
		drpc.shutdown();
		cluster.shutdown();
	} else {
		conf.setNumWorkers(3);
		StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createRemoteTopology());
	}
}
 
Example 35
Project: LearnStorm   File: WordCountTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {

		TopologyBuilder builder = new TopologyBuilder();
		builder.setSpout("spout", new RandomSentenceSpout(), 5);
		builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
		builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));

		Config conf = new Config();
		conf.setDebug(true);

		if (args != null && args.length > 0) {
			conf.setNumWorkers(3);
			StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
		} else {
			conf.setMaxTaskParallelism(3);
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology("word-count", conf, builder.createTopology());
			Thread.sleep(10000);
			cluster.shutdown();
		}
	}
 
Example 36
Project: LearnStorm   File: TridentWordCount.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	Config conf = new Config();
	conf.setMaxSpoutPending(20);
	if (args.length == 0) {
		LocalDRPC drpc = new LocalDRPC();
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("wordCounter", conf, buildTopology(drpc));
		for (int i = 0; i < 100; i++) {
			// The query takes as input a whitespace separated list of words
			// and return the sum of the counts for those words.
			System.out.println("DRPC RESULT: " + drpc.execute("words", "cat$$dog$$the$$man"));
			Thread.sleep(1000);
		}
	} else {
		conf.setNumWorkers(3);
		StormSubmitter.submitTopologyWithProgressBar(args[0], conf, buildTopology(null));
	}
}
 
Example 37
Project: LearnStorm   File: BasicDRPCTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation");
  builder.addBolt(new ExclaimBolt(), 3);

  Config conf = new Config();

  if (args == null || args.length == 0) {
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();

    cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc));

    for (String word : new String[]{ "hello", "goodbye", "apple", "orange", "banana" }) {
      System.out.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word));
    }

    Thread.sleep(10000);
    drpc.shutdown();
    cluster.shutdown();
  }
  else {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createRemoteTopology());
  }
}
 
Example 38
Project: heron   File: WordCountTopology.java   Source Code and License 6 votes vote down vote up
/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
  if (args.length < 1) {
    throw new RuntimeException("Specify topology name");
  }

  int parallelism = 1;
  if (args.length > 1) {
    parallelism = Integer.parseInt(args[1]);
  }
  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout("word", new WordSpout(), parallelism);
  builder.setBolt("consumer", new ConsumerBolt(), parallelism)
      .fieldsGrouping("word", new Fields("word"));
  Config conf = new Config();
  conf.setNumWorkers(parallelism);

  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
 
Example 39
Project: LogRTA   File: ExclamationTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");

    Config conf = new Config();
    conf.setDebug(true);

    if (args != null && args.length > 0) {
        conf.setNumWorkers(3);

        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    }
    else {

        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("www_nginx_accesslog_stat", conf, builder.createTopology());
        Utils.sleep(10000);
        cluster.killTopology("www_nginx_accesslog_stat");
        cluster.shutdown();
    }
}
 
Example 40
Project: StormSampleProject   File: SentimentAnalysisTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception
{
    BasicConfigurator.configure();

    if (args != null && args.length > 0)
    {
        StormSubmitter.submitTopology(
            args[0],
            createConfig(false),
            createTopology());
    }
    else
    {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(
            "sentiment-analysis",
            createConfig(true),
            createTopology());
        Thread.sleep(60000);
        cluster.shutdown();
    }
}
 
Example 41
Project: SmashBrosTwitterAnalytics   File: SmashBrosTwitterTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
	Config conf = new Config();
	conf.setDebug(true);

	if (args != null && args.length > 0) {
		conf.setNumWorkers(3);

		StormSubmitter.submitTopologyWithProgressBar(args[0], conf, buildTopology(null));
	} else {
		conf.setMaxTaskParallelism(3);
		LocalDRPC drpc = new LocalDRPC();
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology(SmashBrosTwitterTopology.class.getSimpleName(), conf, buildTopology(drpc));

		Utils.sleep(10 * 60 * 1000); // 10 minutes
		cluster.shutdown();
	}
}
 
Example 42
Project: resa   File: LoadSimulateTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
        Config conf = readConfig(new File(args[1]));

        if (conf == null) {
            throw new RuntimeException("cannot find conf file " + args[1]);
        }
        ResaConfig resaConfig = ResaConfig.create();
        resaConfig.putAll(conf);
        StormTopology topology = createTopology(resaConfig);
        if (args[0].equals("[local]")) {
            resaConfig.setDebug(false);
            LocalCluster localCluster = new LocalCluster();
            localCluster.submitTopology("local", resaConfig, topology);
        } else {
//            resaConfig.addDrsSupport();
            resaConfig.registerMetricsConsumer(RedisMetricsCollector.class);
            StormSubmitter.submitTopology(args[0], resaConfig, topology);
        }
    }
 
Example 43
Project: resa   File: SimulatedTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Config conf = readConfig(new File(args[1]));

    if (conf == null) {
        throw new RuntimeException("cannot find conf file " + args[1]);
    }
    ResaConfig resaConfig = ResaConfig.create();
    resaConfig.putAll(conf);
    StormTopology topology = createTopology(conf);
    if (args[0].equals("[local]")) {
        resaConfig.setDebug(false);
        LocalCluster localCluster = new LocalCluster();
        localCluster.submitTopology("local", resaConfig, topology);
    } else {
        resaConfig.addDrsSupport();
        resaConfig.registerMetricsConsumer(RedisMetricsCollector.class);
        StormSubmitter.submitTopology(args[0], resaConfig, topology);
    }
}
 
Example 44
Project: resa   File: LoadDataTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
        Config conf = readConfig(new File(args[1]));

        if (conf == null) {
            throw new RuntimeException("cannot find conf file " + args[1]);
        }
        ResaConfig resaConfig = ResaConfig.create();
        resaConfig.putAll(conf);
        StormTopology topology = createTopology(resaConfig);
        if (args[0].equals("[local]")) {
            resaConfig.setDebug(false);
            LocalCluster localCluster = new LocalCluster();
            localCluster.submitTopology("local", resaConfig, topology);
        } else {
//            resaConfig.addDrsSupport();
            resaConfig.registerMetricsConsumer(RedisMetricsCollector.class);
            List<Double> dataSizes = Files.readAllLines(Paths.get(args[2])).stream().map(String::trim)
                    .filter(s -> !s.isEmpty()).map(Double::valueOf).collect(Collectors.toList());
            resaConfig.put("dataSizes", dataSizes);
            StormSubmitter.submitTopology(args[0], resaConfig, topology);
        }
    }
 
Example 45
Project: resa   File: WordCountTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    Config conf = ConfigUtil.readConfig(new File(args[1]));
    if (conf == null) {
        throw new RuntimeException("cannot find conf file " + args[1]);
    }

    TopologyBuilder builder = new TopologyBuilder();

    if (!ConfigUtil.getBoolean(conf, "spout.redis", false)) {
        builder.setSpout("say", new RandomSentenceSpout(), ConfigUtil.getInt(conf, "spout.parallelism", 1));
    } else {
        String host = (String) conf.get("redis.host");
        int port = ((Number) conf.get("redis.port")).intValue();
        String queue = (String) conf.get("redis.queue");
        builder.setSpout("say", new RedisSentenceSpout(host, port, queue),
                ConfigUtil.getInt(conf, "spout.parallelism", 1));
    }
    builder.setBolt("split", new SplitSentence(), ConfigUtil.getInt(conf, "split.parallelism", 1))
            .shuffleGrouping("say");
    builder.setBolt("counter", new WordCount(), ConfigUtil.getInt(conf, "counter.parallelism", 1))
            .fieldsGrouping("split", new Fields("word"));
    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
 
Example 46
Project: siddhi-storm   File: FootballTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {

    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("source", new PlayStream(), 1);
    builder.setBolt("node1", configureSiddhiBolt(), 1).shuffleGrouping("source", "PlayStream1");
    builder.setBolt("LeafEcho", new EchoBolt(), 1).shuffleGrouping("node1", "LongAdvanceStream");

    Config conf = new Config();

    if (args != null && args.length > 0) {
      conf.setNumWorkers(3);

      StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    }
    else {
      conf.setMaxTaskParallelism(3);

      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("football-topology", conf, builder.createTopology());

//      Thread.sleep(10000);
//
//      cluster.shutdown();
    }
  }
 
Example 47
Project: incubator-storm   File: TridentWordCount.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  Config conf = new Config();
  conf.setMaxSpoutPending(20);
  if (args.length == 0) {
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("wordCounter", conf, buildTopology(drpc));
    for (int i = 0; i < 100; i++) {
      System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped"));
      Thread.sleep(1000);
    }
  }
  else {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopology(args[0], conf, buildTopology(null));
  }
}
 
Example 48
Project: incubator-storm   File: ReachTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  LinearDRPCTopologyBuilder builder = construct();


  Config conf = new Config();

  if (args == null || args.length == 0) {
    conf.setMaxTaskParallelism(3);
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("reach-drpc", conf, builder.createLocalTopology(drpc));

    String[] urlsToTry = new String[]{ "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
    for (String url : urlsToTry) {
      System.out.println("Reach of " + url + ": " + drpc.execute("reach", url));
    }

    cluster.shutdown();
    drpc.shutdown();
  }
  else {
    conf.setNumWorkers(6);
    StormSubmitter.submitTopology(args[0], conf, builder.createRemoteTopology());
  }
}
 
Example 49
Project: incubator-storm   File: ExclamationTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("word", new TestWordSpout(), 10);
  builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
  builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");

  Config conf = new Config();
  conf.setDebug(true);

  if (args != null && args.length > 0) {
    conf.setNumWorkers(3);

    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
  }
  else {

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, builder.createTopology());
    Utils.sleep(10000);
    cluster.killTopology("test");
    cluster.shutdown();
  }
}
 
Example 50
Project: incubator-storm   File: BasicDRPCTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder("exclamation");
  builder.addBolt(new ExclaimBolt(), 3);

  Config conf = new Config();

  if (args == null || args.length == 0) {
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();

    cluster.submitTopology("drpc-demo", conf, builder.createLocalTopology(drpc));

    for (String word : new String[]{ "hello", "goodbye" }) {
      System.out.println("Result for \"" + word + "\": " + drpc.execute("exclamation", word));
    }

    cluster.shutdown();
    drpc.shutdown();
  }
  else {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopology(args[0], conf, builder.createRemoteTopology());
  }
}
 
Example 51
Project: storm-hbase   File: WordCountTrident.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    Config conf = new Config();
    conf.setMaxSpoutPending(5);
    if (args.length == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("wordCounter", conf, buildTopology(args[0]));
        Thread.sleep(60 * 1000);
        cluster.killTopology("wordCounter");
        cluster.shutdown();
        System.exit(0);
    }
    else if(args.length == 2) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[1], conf, buildTopology(args[0]));
    } else{
        System.out.println("Usage: TridentFileTopology <hdfs url> [topology name]");
    }
}
 
Example 52
Project: probabilistic-counting   File: ProbCountTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException,
			InvalidTopologyException {
		Config conf = new Config();
//		conf.setDebug(true);
		if (args == null || args.length == 0) {
			// local mode
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology(TOPOLOGY_NAME, conf, buildTopology());
		} else {
			// distributed mode

			conf.put(Config.STORM_CLUSTER_MODE, new String("distributed"));
			conf.setNumWorkers(1);

			StormSubmitter.submitTopology(args[0], conf, buildTopology());
		}
	}
 
Example 53
Project: flink-perf   File: TridentWordCount.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  Config conf = new Config();
  conf.setMaxSpoutPending(20);
  if (args.length == 0) {
    LocalDRPC drpc = new LocalDRPC();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("wordCounter", conf, buildTopology(drpc));
    for (int i = 0; i < 100; i++) {
      System.out.println("DRPC RESULT: " + drpc.execute("words", "cat the dog jumped"));
      Thread.sleep(1000);
    }
  }
  else {
    conf.setNumWorkers(3);
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, buildTopology(null));
  }
}
 
Example 54
Project: galaxy-sdk-java   File: SubmitTopologyHelper.java   Source Code and License 6 votes vote down vote up
public static void submitTopology(StormTopology stormTopology, Map topologyConfig) throws Exception {
    // setup StormTopology

    Config submitConfig = new Config();

    // set the configuration for topology
    submitConfig.put(Config.TOPOLOGY_TRIDENT_BATCH_EMIT_INTERVAL_MILLIS, 5000);
    submitConfig.put(Config.TOPOLOGY_ACKER_EXECUTORS, 100);
    submitConfig.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);

    // set the worker process number
    submitConfig.setNumWorkers(ConfigHelper.getInt(topologyConfig, ConfigKeys.STORM_WORKER_NUMBER));

    // get topologyName adn clusterMode;
    String topologyName = ConfigHelper.getString(topologyConfig, ConfigKeys.STORM_TOPOLOGY_NAME);
    String clusterMode = ConfigHelper.getString(topologyConfig, ConfigKeys.STORM_CLUSTER_MODE);

    if (clusterMode.equals("local")) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("storm-emq", submitConfig, stormTopology);
    } else {
        submitConfig.put(Config.NIMBUS_HOST, ConfigHelper.getString(topologyConfig, ConfigKeys.STORM_NIMBUS_HOSTNAME));
        StormSubmitter.submitTopology(topologyName, submitConfig, stormTopology);
    }

}
 
Example 55
Project: storm-miclog   File: AppLogWriterMain.java   Source Code and License 6 votes vote down vote up
/**
 * @param args
 * @throws InvalidTopologyException 
 * @throws AlreadyAliveException 
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
	TopologyBuilder builder=new TopologyBuilder();
	builder.setSpout("log-writer", new AppLogWriterSpout(),20);
	builder.setBolt("log-writer-file", new AppVisitLogWriterBolt(),20)
					.shuffleGrouping("log-writer");
	Config conf=new Config();
	conf.setNumWorkers(3);
	conf.setDebug(false);
	// ��Ⱥģʽ
	if (args != null && args.length > 0) {
		StormSubmitter.submitTopology(args[0], conf,
				builder.createTopology());
	} else {
		// ����ģʽ
		conf.setMaxTaskParallelism(1);
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("logWriter", conf,builder.createTopology());
	}
}
 
Example 56
Project: storm-miclog   File: AppVisitAlarmMain.java   Source Code and License 6 votes vote down vote up
/**
 * Get data from db and send message and email for someone or group.
 * @param args
 * @throws InvalidTopologyException 
 * @throws AlreadyAliveException 
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
	TopologyBuilder builder=new TopologyBuilder();
	builder.setSpout("Log-VisitAlarm-Spout", new AppVisitAlarmSpout(),10);
	//�����ʼ�
	builder.setBolt("Log-Alarm-VisitErrorStatusCode-Bolt",new VisitErrorStatusBolt(), 10)
					.shuffleGrouping("Log-VisitAlarm-Spout");
	Config conf=new Config();
	conf.setDebug(false);
	conf.setNumWorkers(4);
	if (args != null && args.length > 0) {
		StormSubmitter.submitTopology(args[0], conf,builder.createTopology());
	} else {
		conf.setMaxTaskParallelism(1);
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("AlarmErrorStatusCode", conf,builder.createTopology());
	}
}
 
Example 57
Project: storm-miclog   File: AppLogRecordMain.java   Source Code and License 6 votes vote down vote up
/**��ȡENӦ����־������̨
 * @param args
 * @throws InvalidTopologyException 
 * @throws AlreadyAliveException 
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
	TopologyBuilder builder=new TopologyBuilder();
	//�����ȡkafka��Ϣ��spout
	builder.setSpout("Log-EN-AppMainSpout", new AppEnReaderSpout(),10);
	//������ϱ�����������������ݺ�λ����Ϣ���浽���ݿ�
	builder.setBolt("Log-SendEx-ToDb-Bolt",new ErrorOrFatalToDbBolt(), 10)
					.shuffleGrouping("Log-EN-AppMainSpout");
	
	Config conf=new Config();
	conf.setDebug(false);
	conf.setNumWorkers(2);
	// ��Ⱥģʽ
	if (args != null && args.length > 0) {
		StormSubmitter.submitTopology(args[0], conf,
				builder.createTopology());
	} else {
		// ����ģʽ
		conf.setMaxTaskParallelism(1);
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("logAppEnRecord", conf,
				builder.createTopology());
	}
}
 
Example 58
Project: storm-miclog   File: AppLogAlarmMain.java   Source Code and License 6 votes vote down vote up
/**
 * Get data from db and send message and email for someone or group.
 * @param args
 * @throws InvalidTopologyException 
 * @throws AlreadyAliveException 
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
	TopologyBuilder builder=new TopologyBuilder();
	/*����һ���Ӧkafka����������*/
	builder.setSpout("Log-Alarm-Spout", new AppAlarmSpout(),10);
	//��������
	builder.setBolt("Log-Alarm-Fatal-Bolt",new FatalBolt(), 10)
					.fieldsGrouping("Log-Alarm-Spout","FATAL",new Fields("appname"));
	//���ش���
	builder.setBolt("Log-Alarm-ERROR-Bolt",new ErrorBolt(), 10)
					.fieldsGrouping("Log-Alarm-Spout","ERROR",new Fields("appname"));
	builder.setBolt("Log-Alarm-WARN-Bolt",new WarnBolt(), 10)
	//�������
					.fieldsGrouping("Log-Alarm-Spout","WARN",new Fields("appname"));
	Config conf=new Config();
	conf.setDebug(false);
	conf.setNumWorkers(3);
	if (args != null && args.length > 0) {
		StormSubmitter.submitTopology(args[0], conf,builder.createTopology());
	} else {
		conf.setMaxTaskParallelism(1);
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("AlarmEM", conf,
		builder.createTopology());
	}
}
 
Example 59
Project: trident-tutorial   File: TopHashtagByFollowerClass.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    Config conf = new Config();

    if (args.length == 2) {
        // Ready & submit the topology
        String name = args[0];
        BrokerHosts hosts = new ZkHosts(args[1]);
        TransactionalTridentKafkaSpout kafkaSpout = TestUtils.testTweetSpout(hosts);

        StormSubmitter.submitTopology(name, conf, buildTopology(kafkaSpout));

    }else{
        System.err.println("<topologyName> <zookeeperHost>");
    }

}
 
Example 60
Project: trident-tutorial   File: GlobalTop20Hashtags.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    Config conf = new Config();

    if (args.length == 2) {
        // Ready & submit the topology
        String name = args[0];
        BrokerHosts hosts = new ZkHosts(args[1]);
        TransactionalTridentKafkaSpout kafkaSpout = TestUtils.testTweetSpout(hosts);

        StormSubmitter.submitTopology(name, conf, buildTopology(kafkaSpout));

    }else{
        System.err.println("<topologyName> <zookeeperHost>");
    }

}