Java Code Examples for backtype.storm.generated.AlreadyAliveException

The following are top voted examples for showing how to use backtype.storm.generated.AlreadyAliveException. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: storm-demo   File: LogStatisticsTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) {
    Config config = new Config();

    HdfsBolt hdfsBolt = makeHdfsBolt();
    KafkaSpout kafkaSpout = makeKafkaSpout(TOPIC, TOPOLOGY_NAME);

    LOG.info("Topology name is {}", TOPOLOGY_NAME);

    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout(KAFKA_SPOUT_ID, kafkaSpout, 10);
    topologyBuilder.setBolt(CROP_BOLT_ID, new CropBolt(), 10).shuffleGrouping(KAFKA_SPOUT_ID);
    topologyBuilder.setBolt(SPLIT_FIELDS_BOLT_ID, new SplitFieldsBolt(), 10).shuffleGrouping(CROP_BOLT_ID);
    topologyBuilder.setBolt(STORM_HDFS_BOLT_ID, hdfsBolt, 4).fieldsGrouping(SPLIT_FIELDS_BOLT_ID, new Fields("timestamp", "fieldvalues"));

    if (args != null && args.length > 0) {
        config.setDebug(false);
        config.setNumWorkers(3);

        try {
            StormSubmitter.submitTopology(args[0], config, topologyBuilder.createTopology());
        } catch (InvalidTopologyException | AlreadyAliveException | AuthorizationException e) {
            e.printStackTrace();
        }
    }
}
 
Example 2
Project: jstorm-0.9.6.3-   File: SequenceTopologyTool.java   Source Code and License 6 votes vote down vote up
public  void SetRemoteTopology() throws AlreadyAliveException,
		InvalidTopologyException, TopologyAssignException {
	 Config conf = getConf();
	StormTopology topology = buildTopology();

	conf.put(Config.STORM_CLUSTER_MODE, "distributed");
	String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
	if (streamName == null) {
		streamName = "SequenceTest";
	}

	if (streamName.contains("zeromq")) {
		conf.put(Config.STORM_MESSAGING_TRANSPORT,
				"com.alibaba.jstorm.message.zeroMq.MQContext");

	} else {
		conf.put(Config.STORM_MESSAGING_TRANSPORT,
				"com.alibaba.jstorm.message.netty.NettyContext");
	}

	StormSubmitter.submitTopology(streamName, conf,topology);
	
}
 
Example 3
Project: jstorm-0.9.6.3-   File: StormSubmitter.java   Source Code and License 6 votes vote down vote up
public static void submitTopology(String name, Map stormConf,
		StormTopology topology, SubmitOptions opts, List<File> jarFiles)
		throws AlreadyAliveException, InvalidTopologyException {
	if (jarFiles == null) {
		jarFiles = new ArrayList<File>();
	}
	Map<String, String> jars = new HashMap<String, String>(jarFiles.size());
	List<String> names = new ArrayList<String>(jarFiles.size());
	
	for (File f : jarFiles) {
		if (!f.exists()) {
			LOG.info(f.getName() + " is not existed: "
					+ f.getAbsolutePath());
			continue;
		}
		jars.put(f.getName(), f.getAbsolutePath());
		names.add(f.getName());
	}
	LOG.info("Files: " + names + " will be loaded");
	stormConf.put(GenericOptionsParser.TOPOLOGY_LIB_PATH, jars);
	stormConf.put(GenericOptionsParser.TOPOLOGY_LIB_NAME, names);
	submitTopology(name, stormConf, topology, opts);
}
 
Example 4
Project: learn_jstorm   File: SequenceTopologyTool.java   Source Code and License 6 votes vote down vote up
public  void SetRemoteTopology() throws AlreadyAliveException,
		InvalidTopologyException, TopologyAssignException {
	 Config conf = getConf();
	StormTopology topology = buildTopology();

	conf.put(Config.STORM_CLUSTER_MODE, "distributed");
	String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
	if (streamName == null) {
		streamName = "SequenceTest";
	}

	if (streamName.contains("zeromq")) {
		conf.put(Config.STORM_MESSAGING_TRANSPORT,
				"com.alibaba.jstorm.message.zeroMq.MQContext");

	} else {
		conf.put(Config.STORM_MESSAGING_TRANSPORT,
				"com.alibaba.jstorm.message.netty.NettyContext");
	}

	StormSubmitter.submitTopology(streamName, conf,topology);
	
}
 
Example 5
Project: learn_jstorm   File: StormSubmitter.java   Source Code and License 6 votes vote down vote up
public static void submitTopology(String name, Map stormConf,
		StormTopology topology, SubmitOptions opts, List<File> jarFiles)
		throws AlreadyAliveException, InvalidTopologyException {
	if (jarFiles == null) {
		jarFiles = new ArrayList<File>();
	}
	Map<String, String> jars = new HashMap<String, String>(jarFiles.size());
	List<String> names = new ArrayList<String>(jarFiles.size());
	
	for (File f : jarFiles) {
		if (!f.exists()) {
			LOG.info(f.getName() + " is not existed: "
					+ f.getAbsolutePath());
			continue;
		}
		jars.put(f.getName(), f.getAbsolutePath());
		names.add(f.getName());
	}
	LOG.info("Files: " + names + " will be loaded");
	stormConf.put(GenericOptionsParser.TOPOLOGY_LIB_PATH, jars);
	stormConf.put(GenericOptionsParser.TOPOLOGY_LIB_NAME, names);
	submitTopology(name, stormConf, topology, opts);
}
 
Example 6
Project: jstrom   File: SequenceTopologyTool.java   Source Code and License 6 votes vote down vote up
public  void SetRemoteTopology() throws AlreadyAliveException,
		InvalidTopologyException, TopologyAssignException {
	 Config conf = getConf();
	StormTopology topology = buildTopology();

	conf.put(Config.STORM_CLUSTER_MODE, "distributed");
	String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
	if (streamName == null) {
		streamName = "SequenceTest";
	}

	if (streamName.contains("zeromq")) {
		conf.put(Config.STORM_MESSAGING_TRANSPORT,
				"com.alibaba.jstorm.message.zeroMq.MQContext");

	} else {
		conf.put(Config.STORM_MESSAGING_TRANSPORT,
				"com.alibaba.jstorm.message.netty.NettyContext");
	}

	StormSubmitter.submitTopology(streamName, conf,topology);
	
}
 
Example 7
Project: Infrastructure   File: TestTopology.java   Source Code and License 6 votes vote down vote up
/**
 * Main method.
 * @param args the arguments
 */
public static void main(String[] args) {
  //create the main topology.
    options = new PipelineOptions(args);
    MainTopologyCreator topoCreator = new MainTopologyCreator();
    TopologyOutput topo = topoCreator.createMainTopology();
    //get the topology information
    config = topo.getConfig();
    TopologyBuilder builder = topo.getBuilder();
    int defNumWorkers = topo.getNumWorkers();
    options.toConf(config);
    
    
    if (args != null && args.length > 0) {
        config.setNumWorkers(defNumWorkers);
        try {
            StormSubmitter.submitTopology(args[0], config, builder.createTopology());
        } catch (AlreadyAliveException | InvalidTopologyException e) {
            e.printStackTrace();
        }
    } else {
        final LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
    }
}
 
Example 8
Project: Tstream   File: StormSubmitter.java   Source Code and License 6 votes vote down vote up
public static void submitTopology(String name, Map stormConf,
		StormTopology topology, SubmitOptions opts, List<File> jarFiles)
		throws AlreadyAliveException, InvalidTopologyException {
	Map<String, String> jars = new HashMap<String, String>(jarFiles.size());
	List<String> names = new ArrayList<String>(jarFiles.size());
	if (jarFiles == null)
		jarFiles = new ArrayList<File>();
	for (File f : jarFiles) {
		if (!f.exists()) {
			LOG.info(f.getName() + " is not existed: "
					+ f.getAbsolutePath());
			continue;
		}
		jars.put(f.getName(), f.getAbsolutePath());
		names.add(f.getName());
	}
	LOG.info("Files: " + names + " will be loaded");
	stormConf.put(GenericOptionsParser.TOPOLOGY_LIB_PATH, jars);
	stormConf.put(GenericOptionsParser.TOPOLOGY_LIB_NAME, names);
	submitTopology(name, stormConf, topology, opts);
}
 
Example 9
Project: LearnStorm   File: LogAnalyzer.java   Source Code and License 6 votes vote down vote up
private void buildAndSubmit() throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
		final int numWorkers = Integer.valueOf(topologyConfig.getProperty("num.workers"));
		Config config = new Config();
		config.setDebug(DEBUG);
		config.setNumWorkers(numWorkers);
		config.setMaxSpoutPending(1000000);
		// https://github.com/apache/storm/tree/v0.10.0/external/storm-kafka
		config.setMessageTimeoutSecs(600);	// This value(30 secs by default) must
							// be larger than retryDelayMaxMs
							// (60 secs by default) in
							/// KafkaSpout.

		TopologyBuilder builder = new TopologyBuilder();
		configureKafkaSpout(builder, config);
		configureESBolts(builder, config);

//		LocalCluster cluster = new LocalCluster();
		StormSubmitter.submitTopology("LogAnalyzerV1", config, builder.createTopology());
	}
 
Example 10
Project: LearnStorm   File: ApLogAnalyzer.java   Source Code and License 6 votes vote down vote up
private void buildAndSubmit() throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
		final int numWorkers = Integer.valueOf(topologyConfig.getProperty("num.workers"));
		Config config = new Config();
		config.setDebug(DEBUG);
		config.setNumWorkers(numWorkers);
		config.setMaxSpoutPending(1000000);
		// https://github.com/apache/storm/tree/v0.10.0/external/storm-kafka
		config.setMessageTimeoutSecs(600);	// This value(30 secs by default) must
							// be larger than retryDelayMaxMs
							// (60 secs by default) in
							// KafkaSpout.
		TopologyBuilder builder = new TopologyBuilder();
		configureKafkaSpout(builder, config);
		configureESBolts(builder, config);
//		configureHBaseBolts(builder, config);

//		conf.put(Config.NIMBUS_HOST, "hdp01.localdomain");
//		System.setProperty("storm.jar", "/root/workspace//LearnStorm/target/LearnStorm-0.0.1-SNAPSHOT.jar");
//		System.setProperty("hadoop.home.dir", "/tmp");
//		LocalCluster cluster = new LocalCluster();
		StormSubmitter.submitTopology("ApLogAnalyzerV1", config, builder.createTopology());
	}
 
Example 11
Project: heron   File: WordCountTopology.java   Source Code and License 6 votes vote down vote up
/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
  if (args.length < 1) {
    throw new RuntimeException("Specify topology name");
  }

  int parallelism = 1;
  if (args.length > 1) {
    parallelism = Integer.parseInt(args[1]);
  }
  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout("word", new WordSpout(), parallelism);
  builder.setBolt("consumer", new ConsumerBolt(), parallelism)
      .fieldsGrouping("word", new Fields("word"));
  Config conf = new Config();
  conf.setNumWorkers(parallelism);

  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
 
Example 12
Project: heron   File: LocalCluster.java   Source Code and License 6 votes vote down vote up
@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public void submitTopology(String topoName,
                           Map config,
                           StormTopology stormTopology)
    throws AlreadyAliveException, InvalidTopologyException {
  assertNotAlive();

  this.topologyName = topoName;
  this.conf = config;
  this.topology = stormTopology;

  simulator.submitTopology(topoName,
      ConfigUtils.translateConfig(config),
      stormTopology.getStormTopology());
}
 
Example 13
Project: SmashBrosTwitterAnalytics   File: SmashBrosTwitterTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
	Config conf = new Config();
	conf.setDebug(true);

	if (args != null && args.length > 0) {
		conf.setNumWorkers(3);

		StormSubmitter.submitTopologyWithProgressBar(args[0], conf, buildTopology(null));
	} else {
		conf.setMaxTaskParallelism(3);
		LocalDRPC drpc = new LocalDRPC();
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology(SmashBrosTwitterTopology.class.getSimpleName(), conf, buildTopology(drpc));

		Utils.sleep(10 * 60 * 1000); // 10 minutes
		cluster.shutdown();
	}
}
 
Example 14
Project: resa   File: SimulatedTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Config conf = readConfig(new File(args[1]));

    if (conf == null) {
        throw new RuntimeException("cannot find conf file " + args[1]);
    }
    ResaConfig resaConfig = ResaConfig.create();
    resaConfig.putAll(conf);
    StormTopology topology = createTopology(conf);
    if (args[0].equals("[local]")) {
        resaConfig.setDebug(false);
        LocalCluster localCluster = new LocalCluster();
        localCluster.submitTopology("local", resaConfig, topology);
    } else {
        resaConfig.addDrsSupport();
        resaConfig.registerMetricsConsumer(RedisMetricsCollector.class);
        StormSubmitter.submitTopology(args[0], resaConfig, topology);
    }
}
 
Example 15
Project: storm-benchmark   File: LocalRunner.java   Source Code and License 6 votes vote down vote up
private static void run(String name)
        throws ClassNotFoundException, IllegalAccessException,
        InstantiationException, AlreadyAliveException, InvalidTopologyException {
  LOG.info("running benchmark " + name);
  IBenchmark benchmark =  (IBenchmark) Runner.getApplicationFromName(PACKAGE + "." + name);
  Config config = new Config();
  config.putAll(Utils.readStormConfig());
  config.setDebug(true);
  StormTopology topology = benchmark.getTopology(config);
  LocalCluster localCluster = new LocalCluster();
  localCluster.submitTopology(name, config, topology);
  final int runtime = BenchmarkUtils.getInt(config, MetricsCollectorConfig.METRICS_TOTAL_TIME,
          MetricsCollectorConfig.DEFAULT_TOTAL_TIME);
  IMetricsCollector collector = benchmark.getMetricsCollector(config, topology);
  collector.run();
  try {
    Thread.sleep(runtime);
  } catch (InterruptedException e) {
    LOG.error("benchmark interrupted", e);
  }
  localCluster.shutdown();
}
 
Example 16
Project: probabilistic-counting   File: ProbCountTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException,
			InvalidTopologyException {
		Config conf = new Config();
//		conf.setDebug(true);
		if (args == null || args.length == 0) {
			// local mode
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology(TOPOLOGY_NAME, conf, buildTopology());
		} else {
			// distributed mode

			conf.put(Config.STORM_CLUSTER_MODE, new String("distributed"));
			conf.setNumWorkers(1);

			StormSubmitter.submitTopology(args[0], conf, buildTopology());
		}
	}
 
Example 17
Project: storm-miclog   File: AppLogWriterMain.java   Source Code and License 6 votes vote down vote up
/**
 * @param args
 * @throws InvalidTopologyException 
 * @throws AlreadyAliveException 
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
	TopologyBuilder builder=new TopologyBuilder();
	builder.setSpout("log-writer", new AppLogWriterSpout(),20);
	builder.setBolt("log-writer-file", new AppVisitLogWriterBolt(),20)
					.shuffleGrouping("log-writer");
	Config conf=new Config();
	conf.setNumWorkers(3);
	conf.setDebug(false);
	// ��Ⱥģʽ
	if (args != null && args.length > 0) {
		StormSubmitter.submitTopology(args[0], conf,
				builder.createTopology());
	} else {
		// ����ģʽ
		conf.setMaxTaskParallelism(1);
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("logWriter", conf,builder.createTopology());
	}
}
 
Example 18
Project: storm-miclog   File: AppVisitAlarmMain.java   Source Code and License 6 votes vote down vote up
/**
 * Get data from db and send message and email for someone or group.
 * @param args
 * @throws InvalidTopologyException 
 * @throws AlreadyAliveException 
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
	TopologyBuilder builder=new TopologyBuilder();
	builder.setSpout("Log-VisitAlarm-Spout", new AppVisitAlarmSpout(),10);
	//�����ʼ�
	builder.setBolt("Log-Alarm-VisitErrorStatusCode-Bolt",new VisitErrorStatusBolt(), 10)
					.shuffleGrouping("Log-VisitAlarm-Spout");
	Config conf=new Config();
	conf.setDebug(false);
	conf.setNumWorkers(4);
	if (args != null && args.length > 0) {
		StormSubmitter.submitTopology(args[0], conf,builder.createTopology());
	} else {
		conf.setMaxTaskParallelism(1);
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("AlarmErrorStatusCode", conf,builder.createTopology());
	}
}
 
Example 19
Project: storm-miclog   File: AppLogRecordMain.java   Source Code and License 6 votes vote down vote up
/**��ȡENӦ����־������̨
 * @param args
 * @throws InvalidTopologyException 
 * @throws AlreadyAliveException 
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
	TopologyBuilder builder=new TopologyBuilder();
	//�����ȡkafka��Ϣ��spout
	builder.setSpout("Log-EN-AppMainSpout", new AppEnReaderSpout(),10);
	//������ϱ�����������������ݺ�λ����Ϣ���浽���ݿ�
	builder.setBolt("Log-SendEx-ToDb-Bolt",new ErrorOrFatalToDbBolt(), 10)
					.shuffleGrouping("Log-EN-AppMainSpout");
	
	Config conf=new Config();
	conf.setDebug(false);
	conf.setNumWorkers(2);
	// ��Ⱥģʽ
	if (args != null && args.length > 0) {
		StormSubmitter.submitTopology(args[0], conf,
				builder.createTopology());
	} else {
		// ����ģʽ
		conf.setMaxTaskParallelism(1);
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("logAppEnRecord", conf,
				builder.createTopology());
	}
}
 
Example 20
Project: storm-miclog   File: AppLogAlarmMain.java   Source Code and License 6 votes vote down vote up
/**
 * Get data from db and send message and email for someone or group.
 * @param args
 * @throws InvalidTopologyException 
 * @throws AlreadyAliveException 
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
	TopologyBuilder builder=new TopologyBuilder();
	/*����һ���Ӧkafka����������*/
	builder.setSpout("Log-Alarm-Spout", new AppAlarmSpout(),10);
	//��������
	builder.setBolt("Log-Alarm-Fatal-Bolt",new FatalBolt(), 10)
					.fieldsGrouping("Log-Alarm-Spout","FATAL",new Fields("appname"));
	//���ش���
	builder.setBolt("Log-Alarm-ERROR-Bolt",new ErrorBolt(), 10)
					.fieldsGrouping("Log-Alarm-Spout","ERROR",new Fields("appname"));
	builder.setBolt("Log-Alarm-WARN-Bolt",new WarnBolt(), 10)
	//�������
					.fieldsGrouping("Log-Alarm-Spout","WARN",new Fields("appname"));
	Config conf=new Config();
	conf.setDebug(false);
	conf.setNumWorkers(3);
	if (args != null && args.length > 0) {
		StormSubmitter.submitTopology(args[0], conf,builder.createTopology());
	} else {
		conf.setMaxTaskParallelism(1);
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("AlarmEM", conf,
		builder.createTopology());
	}
}
 
Example 21
Project: mlstorm   File: CobwebClusteringTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length < 3) {
        Logger.getAnonymousLogger().log(Level.SEVERE, "where are the commandline args? -- use args -- folder numWorkers windowSize");
        System.exit(-1);
    }

    final FieldTemplate template = new MlStormFieldTemplate();
    final int numWorkers = Integer.valueOf(args[1]);
    final int windowSize = Integer.valueOf(args[2]);
    final StateUpdater stateUpdater = new CobwebClusterUpdater(template);
    final StateFactory stateFactory = new MlStormClustererFactory.CobwebClustererFactory(numWorkers, windowSize);
    final QueryFunction<CobwebClustererState, String> queryFunction = new MlStormClustererQuery.CobwebClustererQuery();
    final MlStormSpout features = new MddbFeatureExtractorSpout(args[0], template);
    final StormTopology stormTopology = WekaBaseLearningTopology.buildTopology(features, template, numWorkers, stateUpdater, stateFactory, queryFunction, null, TOPOLOGY_DRPC_NAME, null);

    if (numWorkers == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(TOPOLOGY_DRPC_NAME, MlStormConfig.getDefaultMlStormConfig(numWorkers), stormTopology);
    } else {
        StormSubmitter.submitTopology(TOPOLOGY_DRPC_NAME, MlStormConfig.getDefaultMlStormConfig(numWorkers), stormTopology);
    }
}
 
Example 22
Project: mlstorm   File: KmeansClusteringTopology.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length < 5) {
        System.err.println(" Where are all the arguments? -- use args -- folder numWorkers windowSize k parallelism");
        return;
    }

    final FieldTemplate template = new MlStormFieldTemplate();
    final int numWorkers = Integer.valueOf(args[1]);
    final int windowSize = Integer.valueOf(args[2]);
    final int k = Integer.valueOf(args[3]);
    final int parallelism = Integer.valueOf(args[4]);
    final StateUpdater stateUpdater = new KmeansClusterUpdater(template);
    final StateFactory stateFactory = new MlStormClustererFactory.KmeansClustererFactory(k, windowSize, template);
    final QueryFunction<KmeansClustererState, String> queryFunction = new MlStormClustererQuery.KmeansClustererQuery();
    final QueryFunction<KmeansClustererState, String> parameterUpdateFunction = new MlStormClustererQuery.KmeansNumClustersUpdateQuery();
    final MlStormSpout features = new MddbFeatureExtractorSpout(args[0], template);
    final StormTopology stormTopology = buildTopology(features, template, parallelism, stateUpdater, stateFactory, queryFunction, parameterUpdateFunction, "kmeans", "kUpdate");

    if (numWorkers == 1) {
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("kmeans", MlStormConfig.getDefaultMlStormConfig(numWorkers), stormTopology);
    } else {
        StormSubmitter.submitTopology("kmeans", MlStormConfig.getDefaultMlStormConfig(numWorkers), stormTopology);
    }
}
 
Example 23
Project: CadalWorkspace   File: RecTagTopology.java   Source Code and License 6 votes vote down vote up
/**
 * @param args
 * @throws Exception 
 * @throws AlreadyAliveException 
 */
public static void main(String[] args) throws AlreadyAliveException, Exception {
	TopologyBuilder builder = new TopologyBuilder();

	builder.setSpout("recTagPersonalHome", new RecTagRecPageSpout(), 1);
	builder.setBolt("recTag_ins", new RecTagBolt(), 3)
		.shuffleGrouping("recTagPersonalHome");
	
	Config conf = new Config();
	conf.setDebug(true);

	if (args != null && args.length > 0) {
		conf.setNumAckers(2);

		StormSubmitter.submitTopology(args[0], conf,
				builder.createTopology());
	} else {
		conf.setMaxTaskParallelism(3);
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("storm-test", conf, builder.createTopology());

		Thread.sleep(100000);

		cluster.shutdown();
	}
}
 
Example 24
Project: CadalWorkspace   File: SearchTermTopology.java   Source Code and License 6 votes vote down vote up
/**
 * @param args
 * @throws InvalidTopologyException 
 * @throws AlreadyAliveException 
 * @throws InterruptedException 
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, InterruptedException {
	TopologyBuilder builder = new TopologyBuilder();

	builder.setSpout("SearchTerm", new SearchTermSpout(), 1);
	builder.setBolt("SearchTerm_ins", new SearchTermBolt(), 6)
		.shuffleGrouping("SearchTerm");

	Config conf = new Config();
	conf.setDebug(true);

	if (args != null && args.length > 0) {
		conf.setNumAckers(2);

		StormSubmitter.submitTopology(args[0], conf,
				builder.createTopology());
	} else {
		conf.setMaxTaskParallelism(3);
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("storm-test", conf, builder.createTopology());

		Thread.sleep(100000);

		cluster.shutdown();
	}
}
 
Example 25
Project: CadalWorkspace   File: TopologyBuilderNode.java   Source Code and License 6 votes vote down vote up
/**
 * To build topology.
 * 
 * @param args
 * @throws Exception 
 * @throws AlreadyAliveException 
 */
public static void main(String [] args) throws AlreadyAliveException, Exception {
	TopologyBuilder builder = new TopologyBuilder();
	
	builder.setSpout("spout-sentence", new SpoutNode(), 1);
	builder.setBolt("bolt-insertDB", new BoltNode(), 12).shuffleGrouping("spout-sentence");
	
	Config conf = new Config();
	conf.setDebug(true);
	
	if(args!=null && args.length > 0) {
           conf.setNumWorkers(6);
           
           StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
       } else {
           conf.setMaxTaskParallelism(3);

           LocalCluster cluster = new LocalCluster();
           cluster.submitTopology("storm-test", conf, builder.createTopology());
       
           Thread.sleep(100000);

           cluster.shutdown();
       }
}
 
Example 26
Project: jstorm   File: PerformanceTestTopology.java   Source Code and License 6 votes vote down vote up
public static void SetRemoteTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 
Example 27
Project: jstorm   File: SequenceTopologyTool.java   Source Code and License 6 votes vote down vote up
public void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    Config conf = getConf();
    StormTopology topology = buildTopology();
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        streamName = "SequenceTest";
    }
    
    if (streamName.contains("zeromq")) {
        conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.zeroMq.MQContext");
        
    } else {
        conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.netty.NettyContext");
    }
    
    StormSubmitter.submitTopology(streamName, conf, topology);
    
}
 
Example 28
Project: jstorm   File: BatchAckerTest.java   Source Code and License 6 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Config conf = JStormHelper.getConfig(args);
    int spoutParallelism = JStormUtils.parseInt(conf.get(SPOUT_PARALLELISM_HINT), 1);
    int splitParallelism = JStormUtils.parseInt(conf.get(SPLIT_PARALLELISM_HINT), 2);
    int countParallelism = JStormUtils.parseInt(conf.get(COUNT_PARALLELISM_HINT), 2);
    boolean isValueSpout = JStormUtils.parseBoolean(conf.get("is.value.spout"), false);

    TransactionTopologyBuilder builder = new TransactionTopologyBuilder();
    if (isValueSpout)
        builder.setSpoutWithAck("spout", new BatchAckerValueSpout(), spoutParallelism);
    else
        builder.setSpoutWithAck("spout", new BatchAckerSpout(), spoutParallelism);
    builder.setBoltWithAck("split", new BatchAckerSplit(), splitParallelism).localOrShuffleGrouping("spout");;
    builder.setBoltWithAck("count", new BatchAckerCount(), countParallelism).fieldsGrouping("split", new Fields("word"));
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}
 
Example 29
Project: jstorm   File: WordCountTopology.java   Source Code and License 6 votes vote down vote up
/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length != 1) {
        throw new RuntimeException("Specify topology name");
    }

    int parallelism = 10;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new WordSpout(), parallelism);
    builder.setBolt("consumer", new ConsumerBolt(), parallelism)
            .fieldsGrouping("word", new Fields("word"));
    Config conf = new Config();
    conf.setNumStmgrs(parallelism);
/*
Set config here
*/

    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}
 
Example 30
Project: Mastering-Apache-Storm   File: Topology.java   Source Code and License 5 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException,
			InvalidTopologyException {
		TopologyBuilder builder = new TopologyBuilder();
		
		List<String> zks = new ArrayList<String>();
		zks.add("192.168.41.122");
		
		List<String> cFs = new ArrayList<String>();
		cFs.add("personal");
		cFs.add("company");
		
		// set the spout class
		builder.setSpout("spout", new SampleSpout(), 2);
		// set the bolt class
		builder.setBolt("bolt", new StormRedisBolt("192.168.41.122",2181), 2).shuffleGrouping("spout");

		Config conf = new Config();
		conf.setDebug(true);
		// create an instance of LocalCluster class for
		// executing topology in local mode.
		LocalCluster cluster = new LocalCluster();

		// LearningStormTopolgy is the name of submitted topology.
		cluster.submitTopology("StormRedisTopology", conf,
				builder.createTopology());
		try {
			Thread.sleep(10000);
		} catch (Exception exception) {
			System.out.println("Thread interrupted exception : " + exception);
		}
		// kill the LearningStormTopology
		cluster.killTopology("StormRedisTopology");
		// shutdown the storm test cluster
		cluster.shutdown();
}
 
Example 31
Project: java   File: DeliveryTopology.java   Source Code and License 5 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
  TopologyBuilder builder = new TopologyBuilder();

  LOGGER.info("Starting..");
  builder.setSpout("trade", new DeliveryCheckSpout(), 1);
  builder.setBolt("eligibility", new DeliveryCheckBolt(), 10).shuffleGrouping("trade");
  builder.setBolt("odd", new DeliveryCheckOddBolt(), 10).shuffleGrouping("eligibility",
      "oddstream");
  builder.setBolt("even", new DeliveryCheckEvenBolt(), 10).shuffleGrouping("eligibility",
      "evenstream");

  Config conf = new Config();
  conf.setDebug(false);
  conf.setMaxSpoutPending(5);

  if (args != null && args.length > 0) {
    conf.setNumWorkers(1);
    LOGGER.info("Submitting DeliveryTopology");
    StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
  } else {

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("DeliveryTopology", conf, builder.createTopology());
    Utils.sleep(100000000);
    cluster.killTopology("DeliveryTopology");
    cluster.shutdown();
  }
}
 
Example 32
Project: jstorm-0.9.6.3-   File: ServiceHandler.java   Source Code and License 5 votes vote down vote up
@Override
public void submitTopology(String name, String uploadedJarLocation,
		String jsonConf, StormTopology topology)
		throws AlreadyAliveException, InvalidTopologyException,
		TopologyAssignException, TException {
	SubmitOptions options = new SubmitOptions(TopologyInitialStatus.ACTIVE);

	submitTopologyWithOpts(name, uploadedJarLocation, jsonConf, topology,
			options);
}
 
Example 33
Project: jstorm-0.9.6.3-   File: ServiceHandler.java   Source Code and License 5 votes vote down vote up
/**
 * check whether the topology is bActive?
 * 
 * @param nimbus
 * @param topologyName
 * @param bActive
 * @throws Exception
 */
public void checkTopologyActive(NimbusData nimbus, String topologyName,
		boolean bActive) throws Exception {
	if (isTopologyActive(nimbus.getStormClusterState(), topologyName) != bActive) {
		if (bActive) {
			throw new NotAliveException(topologyName + " is not alive");
		} else {
			throw new AlreadyAliveException(topologyName
					+ " is already active");
		}
	}
}
 
Example 34
Project: jstorm-0.9.6.3-   File: SequenceTopologyTool.java   Source Code and License 5 votes vote down vote up
public  void SetDPRCTopology() throws AlreadyAliveException,
			InvalidTopologyException, TopologyAssignException {
//		LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(
//				"exclamation");
//
//		builder.addBolt(new TotalCount(), 3);
//
//		Config conf = new Config();
//
//		conf.setNumWorkers(3);
//		StormSubmitter.submitTopology("rpc", conf,
//				builder.createRemoteTopology());
		System.out.println("Please refer to com.alipay.dw.jstorm.example.drpc.ReachTopology");
	}
 
Example 35
Project: jstorm-0.9.6.3-   File: SequenceTopology.java   Source Code and License 5 votes vote down vote up
public static void SetDPRCTopology() throws AlreadyAliveException,
		InvalidTopologyException, TopologyAssignException {
	// LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(
	// "exclamation");
	//
	// builder.addBolt(new TotalCount(), 3);
	//
	// Config conf = new Config();
	//
	// conf.setNumWorkers(3);
	// StormSubmitter.submitTopology("rpc", conf,
	// builder.createRemoteTopology());
	System.out
			.println("Please refer to com.alipay.dw.jstorm.example.drpc.ReachTopology");
}
 
Example 36
Project: learn_jstorm   File: ServiceHandler.java   Source Code and License 5 votes vote down vote up
@Override
public void submitTopology(String name, String uploadedJarLocation,
		String jsonConf, StormTopology topology)
		throws AlreadyAliveException, InvalidTopologyException,
		TopologyAssignException, TException {
	SubmitOptions options = new SubmitOptions(TopologyInitialStatus.ACTIVE);

	submitTopologyWithOpts(name, uploadedJarLocation, jsonConf, topology,
			options);
}
 
Example 37
Project: learn_jstorm   File: ServiceHandler.java   Source Code and License 5 votes vote down vote up
/**
 * check whether the topology is bActive?
 * 
 * @param nimbus
 * @param topologyName
 * @param bActive
 * @throws Exception
 */
public void checkTopologyActive(NimbusData nimbus, String topologyName,
		boolean bActive) throws Exception {
	if (isTopologyActive(nimbus.getStormClusterState(), topologyName) != bActive) {
		if (bActive) {
			throw new NotAliveException(topologyName + " is not alive");
		} else {
			throw new AlreadyAliveException(topologyName
					+ " is already active");
		}
	}
}
 
Example 38
Project: learn_jstorm   File: SequenceTopologyTool.java   Source Code and License 5 votes vote down vote up
public  void SetDPRCTopology() throws AlreadyAliveException,
			InvalidTopologyException, TopologyAssignException {
//		LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(
//				"exclamation");
//
//		builder.addBolt(new TotalCount(), 3);
//
//		Config conf = new Config();
//
//		conf.setNumWorkers(3);
//		StormSubmitter.submitTopology("rpc", conf,
//				builder.createRemoteTopology());
		System.out.println("Please refer to com.alipay.dw.jstorm.example.drpc.ReachTopology");
	}
 
Example 39
Project: learn_jstorm   File: SequenceTopology.java   Source Code and License 5 votes vote down vote up
public static void SetDPRCTopology() throws AlreadyAliveException,
		InvalidTopologyException, TopologyAssignException {
	// LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(
	// "exclamation");
	//
	// builder.addBolt(new TotalCount(), 3);
	//
	// Config conf = new Config();
	//
	// conf.setNumWorkers(3);
	// StormSubmitter.submitTopology("rpc", conf,
	// builder.createRemoteTopology());
	System.out
			.println("Please refer to com.alipay.dw.jstorm.example.drpc.ReachTopology");
}
 
Example 40
Project: Big-Data-tutorial   File: FlightTopology.java   Source Code and License 5 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {

		String zkIp = "localhost";

		String nimbusHost = "sandbox.hortonworks.com";

		String zookeeperHost = zkIp +":2181";

		ZkHosts zkHosts = new ZkHosts(zookeeperHost);
		List<String> zkServers = new ArrayList<String>();
		zkServers.add(zkIp);
		SpoutConfig kafkaConfig = new SpoutConfig(zkHosts, "spertus-flight-events", "/spertus-flights-events","flight_id");
		kafkaConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
		kafkaConfig.startOffsetTime = kafka.api.OffsetRequest.EarliestTime();
		kafkaConfig.zkServers = zkServers;
		kafkaConfig.zkRoot = "/spertus-flight-events";
		kafkaConfig.zkPort = 2181;
		kafkaConfig.forceFromStart = true;
		KafkaSpout kafkaSpout = new KafkaSpout(kafkaConfig);

		TopologyBuilder builder = new TopologyBuilder();

		builder.setSpout("flight-events", kafkaSpout, 1);
		builder.setBolt("flight-stats", new GetFlightStatsBolt(), 1).shuffleGrouping("flight-events");

		Map conf = new HashMap();
		conf.put(backtype.storm.Config.TOPOLOGY_WORKERS, 4);
		conf.put(backtype.storm.Config.TOPOLOGY_DEBUG, true);
		if (args != null && args.length > 0) {
			StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
		}   else {
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology("flight-topology", conf, builder.createTopology());
		}
	}
 
Example 41
Project: aeolus   File: LRBTopologyMain.java   Source Code and License 5 votes vote down vote up
/**
 * Encapsulation of creation of the cluster after parameter parsing.
 * 
 * @param offset
 * @param executors
 * @param xways
 * @param host
 * @param port
 * @param histFile
 * @param tasks
 * @param submit
 * @param stormConfigDebug
 * @param workers
 * @param nameext
 * @param runtimeMillis
 * @throws AlreadyAliveException
 * @throws InvalidTopologyException
 * @throws java.io.FileNotFoundException
 */
public static void main0(int offset, int executors, int xways, String host, int port, String histFile, int tasks, boolean submit, boolean stormConfigDebug, int workers, String nameext, int runtimeMillis)
	throws AlreadyAliveException, InvalidTopologyException, FileNotFoundException {
	StopWatch stormTimer = new StopWatch(offset);
	String topologyNamePrefix = nameext + "_lrbNormal_" + "_L" + xways + "_" + workers + "W_T" + tasks + "_"
		+ executors + "E_O" + offset;
	Config conf = new Config();
	LRBTopology lRBTopology = new LRBTopology(nameext, xways, workers, tasks, executors, offset,
		new FileReaderSpout(), // add AbstractOrderedFileInputSpout.INPUT_FILE_NAME
								// and
								// AbstractOrderedFileInputSpout.INPUT_FILE_SUFFIXES
								// in Config below
		stormTimer, submit, histFile, topologyNamePrefix, conf);
	StormTopology topology = lRBTopology.getStormTopology();
	conf.setDebug(stormConfigDebug);
	conf.put(AbstractOrderedFileInputSpout.INPUT_FILE_NAME,
		LRBTopologyMain.class.getResource("/datafile20seconds.dat").getFile());
	
	Locale newLocale = new Locale("en", "US");
	LOGGER.debug(String.format("setting locale to %s", newLocale));
	Locale.setDefault(newLocale); // why??
	
	LOGGER.debug("starting cluster: " + "stormlrb" + topologyNamePrefix);
	if(submit) {
		
		conf.setNumWorkers(workers);
		conf.setNumAckers(workers);
		
		StormSubmitter.submitTopology(topologyNamePrefix, conf, topology);
		
	} else {
		
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology(TopologyControl.TOPOLOGY_NAME, conf, topology);
		
		Utils.sleep(runtimeMillis);
		cluster.killTopology(TopologyControl.TOPOLOGY_NAME);
		cluster.shutdown();
	}
}
 
Example 42
Project: jstrom   File: SequenceTopologyTool.java   Source Code and License 5 votes vote down vote up
public  void SetDPRCTopology() throws AlreadyAliveException,
			InvalidTopologyException, TopologyAssignException {
//		LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(
//				"exclamation");
//
//		builder.addBolt(new TotalCount(), 3);
//
//		Config conf = new Config();
//
//		conf.setNumWorkers(3);
//		StormSubmitter.submitTopology("rpc", conf,
//				builder.createRemoteTopology());
		System.out.println("Please refer to com.alipay.dw.jstorm.example.drpc.ReachTopology");
	}
 
Example 43
Project: jstrom   File: SequenceTopology.java   Source Code and License 5 votes vote down vote up
public static void SetDPRCTopology() throws AlreadyAliveException,
		InvalidTopologyException, TopologyAssignException {
	// LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(
	// "exclamation");
	//
	// builder.addBolt(new TotalCount(), 3);
	//
	// Config conf = new Config();
	//
	// conf.setNumWorkers(3);
	// StormSubmitter.submitTopology("rpc", conf,
	// builder.createRemoteTopology());
	System.out
			.println("Please refer to com.alipay.dw.jstorm.example.drpc.ReachTopology");
}
 
Example 44
Project: Tstream   File: ServiceHandler.java   Source Code and License 5 votes vote down vote up
@Override
public void submitTopology(String name, String uploadedJarLocation,
		String jsonConf, StormTopology topology)
		throws AlreadyAliveException, InvalidTopologyException,
		TopologyAssignException, TException {
	SubmitOptions options = new SubmitOptions(TopologyInitialStatus.ACTIVE);

	submitTopologyWithOpts(name, uploadedJarLocation, jsonConf, topology,
			options);
}
 
Example 45
Project: Tstream   File: ServiceHandler.java   Source Code and License 5 votes vote down vote up
/**
 * check whether the topology is bActive?
 * 
 * @param nimbus
 * @param topologyName
 * @param bActive
 * @throws Exception
 */
public void checkTopologyActive(NimbusData nimbus, String topologyName,
		boolean bActive) throws Exception {
	if (isTopologyActive(nimbus.getStormClusterState(), topologyName) != bActive) {
		if (bActive) {
			throw new NotAliveException(topologyName + " is not alive");
		} else {
			throw new AlreadyAliveException(topologyName
					+ " is already active");
		}
	}
}
 
Example 46
Project: LearnStorm   File: ApLogGenerator.java   Source Code and License 5 votes vote down vote up
private void buildAndSubmit() throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
		Config config = new Config();
		config.setDebug(DEBUG);
		config.setNumWorkers(1);

		TopologyBuilder builder = new TopologyBuilder();
		configureRandomLogSpout(builder, config);
		configureKafkaBolt(builder, config);

//		LocalCluster cluster = new LocalCluster();
		StormSubmitter.submitTopology("ApLogGeneratorV1", config, builder.createTopology());
	}
 
Example 47
Project: simple-kafka-storm-java   File: KafkaStormExample.java   Source Code and License 5 votes vote down vote up
/**
 * 
 * This example is very dangerous to the consistency of your bank accounts. Guess why, or read the
 * tutorial.
 * 
 * @throws AlreadyAliveException
 * @throws InvalidTopologyException
 * @throws AuthorizationException
 */
public static void main(String... args) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {

	// starting to build topology
	TopologyBuilder builder = new TopologyBuilder();

	// Kafka as a spout
	builder.setSpout(IDs.kafkaSpout, new KafkaSpoutBuilder(Conf.zookeeper, Conf.inputTopic).build());

	// bolt to println data
	builder.setBolt(IDs.printlnBolt, new PrintlnBolt()).shuffleGrouping(IDs.kafkaSpout);

	// bolt to perform transactions and simulate bank accounts
	builder.setBolt(IDs.userAccountBolt, new BankAccountBolt()).shuffleGrouping(IDs.kafkaSpout);

	// Kafka as a bolt -- sending messages to the output topic
	KafkaBolt<Object, Object> bolt = new KafkaBolt<>().withTopicSelector(new DefaultTopicSelector(Conf.outputTopic))
			.withTupleToKafkaMapper(new TransactionTupleToKafkaMapper());
	builder.setBolt(IDs.kafkaBolt, bolt).shuffleGrouping(IDs.userAccountBolt);

	// submit topolody to local cluster
	new LocalCluster().submitTopology(IDs.kafkaAccountsTopology, topologyConfig(), builder.createTopology());

	// wait a while, then simulate random transaction stream to Kafka
	Sleep.seconds(5);
	KafkaProduceExample.start(2000);

}
 
Example 48
Project: simple-kafka-storm-java   File: KafkaStormTridentExample.java   Source Code and License 5 votes vote down vote up
public static void main(String... args) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {

		// starting to build topology
		TridentTopology topology = new TridentTopology();

		// Kafka as an opaque trident spout
		OpaqueTridentKafkaSpout spout = new OpaqueTridentKafkaSpoutBuilder(Conf.zookeeper, Conf.inputTopic).build();
		Stream stream = topology.newStream(kafkaSpout, spout);

		// mapping transaction messages to pairs: (person,amount)
		Stream atomicTransactions = stream.each(strF, Functions.mapToPersonAmount, personAmountF);

		// bolt to println data
		atomicTransactions.each(personAmountF, Functions.printlnFunction, emptyF);

		// aggregating transactions and mapping to Kafka messages
		Stream transactionsGroupped = atomicTransactions.groupBy(personF)
				.persistentAggregate(new MemoryMapState.Factory(), amountF, new Sum(), sumF).newValuesStream()
				.each(personSumF, Functions.mapToKafkaMessage, keyMessageF);

		// Kafka as a bolt -- producing to outputTopic
		TridentKafkaStateFactory stateFactory = new TridentKafkaStateFactory() //
				.withKafkaTopicSelector(new DefaultTopicSelector(Conf.outputTopic)) //
				.withTridentTupleToKafkaMapper(new FieldNameBasedTupleToKafkaMapper<String, String>(key, message));
		transactionsGroupped.partitionPersist(stateFactory, keyMessageF, new TridentKafkaUpdater(), emptyF);

		// submitting topology to local cluster
		new LocalCluster().submitTopology(kafkaAccountsTopology, topologyConfig(), topology.build());

		// waiting a while, then running Kafka producer
		Sleep.seconds(5);
		KafkaProduceExample.start(20);

	}
 
Example 49
Project: resa   File: DectationTopology.java   Source Code and License 5 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Config conf = readConfig(new File(args[1]));

    if (conf == null) {
        throw new RuntimeException("cannot find conf file " + args[1]);
    }
    ResaConfig resaConfig = ResaConfig.create();
    resaConfig.putAll(conf);
    StormTopology topology = createTopology(conf);
    if (args[0].equals("[local]")) {
        resaConfig.setDebug(false);
        LocalCluster localCluster = new LocalCluster();
        localCluster.submitTopology("local", resaConfig, topology);
    } else {
        if (ConfigUtil.getBoolean(conf, "vd.metric.resa", false)) {
            resaConfig.addDrsSupport();
            resaConfig.put(ResaConfig.REBALANCE_WAITING_SECS, 0);
            System.out.println("ResaMetricsCollector is registered");
        }

        if (ConfigUtil.getBoolean(conf, "vd.metric.redis", true)) {
            resaConfig.registerMetricsConsumer(RedisMetricsCollector.class);
            System.out.println("RedisMetricsCollector is registered");
        }
        StormSubmitter.submitTopology(args[0], resaConfig, topology);
    }
}
 
Example 50
Project: piggybank-squeal   File: Main.java   Source Code and License 5 votes vote down vote up
public void launch(String jarFile) throws AlreadyAliveException, InvalidTopologyException, IOException {
	if (pc.getProperties().getProperty(RUN_DIRECT_KEY, "false").equalsIgnoreCase("true")) {
		String topology_name = pc.getProperties().getProperty(TOPOLOGY_NAME_KEY, "PigStorm-" + pc.getLastAlias());
		runTestCluster(topology_name);
	} else {			
		// Execute "storm jar <jarfile> <this.classname>";
		String exec = "storm jar " + jarFile + " " + this.getClass().getCanonicalName();
		System.out.println("Running: " + exec);
		Process p = Runtime.getRuntime().exec(exec);
		BufferedReader sout = new BufferedReader(new InputStreamReader(p.getInputStream()));
        BufferedReader serr = new BufferedReader(new InputStreamReader(p.getErrorStream()));
        
        // Pull any stdin/stdout
        String line;
        while ((line = sout.readLine()) != null) {
        	System.out.println(line);
        }
        while ((line = serr.readLine()) != null) {
        	System.err.println(line);
        }
        
        int ret;
		try {
			ret = p.waitFor();
		} catch (InterruptedException e) {
			throw new RuntimeException(e);
		}
        if (ret != 0) {
        	throw new RuntimeException("storm jar returned with non-zero status: " + ret);
        }
	}
}
 
Example 51
Project: piggybank-squeal   File: Main.java   Source Code and License 5 votes vote down vote up
public void submitTopology(String topology_name) throws AlreadyAliveException, InvalidTopologyException {
	Config conf = new Config();
	
	String extraConf = pc.getProperties().getProperty(EXTRA_CONF_KEY, null);
	if (extraConf != null) {
		System.out.println("Loading additional configuration properties from: " + extraConf);
		// Load the configuration file.
		Yaml yaml = new Yaml();
		FileReader fr;
		try {
			fr = new FileReader(extraConf);
			Map<String, Object> m = (Map<String, Object>) yaml.load(fr);
			conf.putAll(m);
			fr.close();
		} catch (IOException e) {
			throw new RuntimeException(e);
		}	
	}
	
	int workers = Integer.parseInt(pc.getProperties().getProperty(WORKERS_COUNT_KEY, "4"));
	conf.setNumWorkers(workers);
	int ackers = Integer.parseInt(pc.getProperties().getProperty(ACKERS_COUNT_KEY, "1"));
	conf.setNumAckers(ackers);
	
	// Register a Serializer for any Writable.
	registerSerializer(conf);
	
	passPigContextProperties(conf);
	
	StormSubmitter.submitTopology(topology_name, conf, getTopology());
}
 
Example 52
Project: piggybank-squeal   File: Main.java   Source Code and License 5 votes vote down vote up
public void runMain(String[] args) throws IOException, AlreadyAliveException, InvalidTopologyException {
	/* Create the Pig context */
	pc = (PigContext) getStuff("pigContext");
	initFromPigContext(pc);
	
	String topology_name = pc.getProperties().getProperty(TOPOLOGY_NAME_KEY, "PigStorm-" + pc.getLastAlias());
	
	if (pc.getProperties().getProperty(RUN_TEST_CLUSTER_KEY, "false").equalsIgnoreCase("true")) {
		runTestCluster(topology_name);
		log.info("Exitting forcefully due to non-terminated threads...");
		System.exit(0);
	} else {
		submitTopology(topology_name);
	}
}
 
Example 53
Project: flume2storm   File: ExampleTopology.java   Source Code and License 5 votes vote down vote up
public static void main(String[] args) throws F2SConfigurationException, ConfigurationException, AlreadyAliveException, InvalidTopologyException {
  Configuration configuration = new PropertiesConfiguration("properties");
  ExampleStringEmitter emitter = new ExampleStringEmitter();
  FlumeSpout<KryoNetConnectionParameters,KryoNetServiceProvider> spout = new FlumeSpout<KryoNetConnectionParameters,KryoNetServiceProvider>(emitter, configuration);
  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout("spout", spout);
  LoggerBolt bolt = new LoggerBolt();
  builder.setBolt("bolt", bolt).shuffleGrouping("spout");
  
  Config stormConf = new Config();
  stormConf.setDebug(true);
  stormConf.setNumWorkers(3);
  StormSubmitter.submitTopology("ExampleTopology", stormConf, builder.createTopology());
}
 
Example 54
Project: recsys-online   File: Recsys.java   Source Code and License 5 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException,
			InvalidTopologyException {
		logger.info("begin to running recsys.");
		BrokerHosts brokerHosts = new ZkHosts(Constants.kafka_zk_address);
		SpoutConfig spoutConfig = new SpoutConfig(brokerHosts, Constants.kafka_topic, 	Constants.kafka_zk_root, Constants.kafka_id);

		Config conf = new Config();
		Map<String, String> map = new HashMap<String, String>();
		map.put("metadata.broker.list", Constants.kakfa_broker_list);
		map.put("serializer.class", "kafka.serializer.StringEncoder");
		conf.put("kafka.broker.properties", map);
//		conf.put("topic", "topic2");

		spoutConfig.scheme = new SchemeAsMultiScheme(new MessageScheme());
		TopologyBuilder builder = new TopologyBuilder();
		builder.setSpout("spout", new KafkaSpout(spoutConfig));
		builder.setBolt("bolt", new HBaseStoreBolt()).shuffleGrouping("spout");
//		builder.setBolt("kafkabolt", new KafkaBolt<String, Integer>()).shuffleGrouping("bolt");

		if (!islocal) {
			conf.setNumWorkers(3);
			StormSubmitter.submitTopology(Constants.storm_topology_name, conf, builder.createTopology());
		} else {
			LocalCluster cluster = new LocalCluster();
			cluster.submitTopology(Constants.storm_topology_name, conf, builder.createTopology());
			Utils.sleep(100000);
			cluster.killTopology(Constants.storm_topology_name);
			cluster.shutdown();
		}
		logger.info("run recsys finish.");
	}
 
Example 55
Project: ingestive   File: MockedLocalCluster.java   Source Code and License 5 votes vote down vote up
@Override
public void submitTopology(String topologyName, Map conf,
		StormTopology topology) throws AlreadyAliveException,
		InvalidTopologyException {
	// TODO Auto-generated method stub

}
 
Example 56
Project: ingestive   File: MockedLocalCluster.java   Source Code and License 5 votes vote down vote up
@Override
public void submitTopologyWithOpts(String topologyName, Map conf,
		StormTopology topology, SubmitOptions submitOpts)
		throws AlreadyAliveException, InvalidTopologyException {
	// TODO Auto-generated method stub

}
 
Example 57
Project: storm-benchmark   File: Runner.java   Source Code and License 5 votes vote down vote up
public static void run(String name)
        throws ClassNotFoundException, IllegalAccessException,
        InstantiationException, AlreadyAliveException, InvalidTopologyException {
  if (name.startsWith("storm.benchmark.benchmarks")) {
    LOG.info("running benchmark " + name);
    runBenchmark((IBenchmark) getApplicationFromName(name));
  } else if (name.startsWith("storm.benchmark.tools.producer")) {
    LOG.info("running producer " + name);
    runProducer((IProducer) getApplicationFromName(name));
  } else {
    throw new RuntimeException(name + " is neither benchmark nor producer");
  }
}
 
Example 58
Project: storm-benchmark   File: Runner.java   Source Code and License 5 votes vote down vote up
public static void runBenchmark(IBenchmark benchmark)
        throws AlreadyAliveException, InvalidTopologyException,
        ClassNotFoundException, IllegalAccessException, InstantiationException {
  runApplication(benchmark);
  if (isMetricsEnabled()) {
    IMetricsCollector collector = benchmark.getMetricsCollector(config, topology);
    collector.run();
  }
}
 
Example 59
Project: storm-benchmark   File: Runner.java   Source Code and License 5 votes vote down vote up
private static void runApplication(IApplication app)
        throws AlreadyAliveException, InvalidTopologyException {
  config.putAll(Utils.readStormConfig());
  String name = (String) config.get(Config.TOPOLOGY_NAME);
  topology = app.getTopology(config);
  StormSubmitter.submitTopology(name, config, topology);
}
 
Example 60
Project: flink-perf   File: KafkaThroughput.java   Source Code and License 5 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, UnknownHostException, InterruptedException {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	TopologyBuilder builder = new TopologyBuilder();
	BrokerHosts hosts = new ZkHosts(pt.getRequired("zookeeper"));
	SpoutConfig spoutConfig = new SpoutConfig(hosts, pt.getRequired("topic"), "/" + pt.getRequired("topic"), UUID.randomUUID().toString());
	spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
	KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
	builder.setSpout("source", kafkaSpout, pt.getInt("sourceParallelism"));

	builder.setBolt("sink", new Throughput.Sink(pt), pt.getInt("sinkParallelism")).noneGrouping("source");

	Config conf = new Config();
	conf.setDebug(false);

	if (!pt.has("local")) {
		conf.setNumWorkers(pt.getInt("par", 2));

		StormSubmitter.submitTopologyWithProgressBar("kafka-spout-"+pt.get("name", "no_name"), conf, builder.createTopology());
	} else {
		conf.setMaxTaskParallelism(pt.getInt("par", 2));

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("kafka-spout", conf, builder.createTopology());

		Thread.sleep(300000);

		cluster.shutdown();
	}
}