Java Code Examples for backtype.storm.Config.setFallBackOnJavaSerialization()

The following are Jave code examples for showing how to use setFallBackOnJavaSerialization() of the backtype.storm.Config class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
Example 1
Project: openimaj   File: StormPlayground.java   Source Code and License Vote up 6 votes
public static void main(String[] args) {
	final Config conf = new Config();
	conf.setDebug(false);
	conf.setNumWorkers(2);
	conf.setMaxSpoutPending(1);
	conf.setFallBackOnJavaSerialization(false);
	conf.setSkipMissingKryoRegistrations(false);
	final LocalCluster cluster = new LocalCluster();
	final TopologyBuilder builder = new TopologyBuilder();
	builder.setSpout("randomSpout1", new RandomFieldSpout(2, 0, 0, 1)); // (nfields,seed,min,max)
	builder.setSpout("randomSpout2", new RandomFieldSpout(2, 10, 0, 1)); // (nfields,seed,min,max)
	JoinBolt.connectNewBolt(builder);
	final StormTopology topology = builder.createTopology();
	cluster.submitTopology("playTopology", conf, topology);
	Utils.sleep(10000);
	cluster.killTopology("playTopology");
	cluster.shutdown();

}
 
Example 2
Project: jstorm-0.9.6.3-   File: SequenceTopologyTool.java   Source Code and License Vote up 4 votes
public StormTopology buildTopology()
{
	Config conf = getConf();
	TopologyBuilder builder = new TopologyBuilder();

	int spout_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
	int bolt_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

	builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
			new SequenceSpout(), spout_Parallelism_hint);

	boolean isEnableSplit = JStormUtils.parseBoolean(
			conf.get("enable.split"), false);

	if (isEnableSplit == false) {
		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).localFirstGrouping(
				SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
	} else {

		builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
				new SplitRecord(), bolt_Parallelism_hint)
				.localOrShuffleGrouping(
						SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

		builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.TRADE_STREAM_ID);
		builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.CUSTOMER_STREAM_ID);

		builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
				new MergeRecord(), bolt_Parallelism_hint)
				.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
						new Fields("ID"))
				.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
						new Fields("ID"));

		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).noneGrouping(
				SequenceTopologyDef.MERGE_BOLT_NAME);
	}

	boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
			false);
	if (kryoEnable == true) {
		System.out.println("Use Kryo ");
		boolean useJavaSer = JStormUtils.parseBoolean(
				conf.get("fall.back.on.java.serialization"), true);

		Config.setFallBackOnJavaSerialization(conf, useJavaSer);

		Config.registerSerialization(conf, TradeCustomer.class);
		Config.registerSerialization(conf, Pair.class);
	}
	int ackerNum = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
	Config.setNumAckers(conf, ackerNum);

	int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
			20);
	conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	return  builder.createTopology();	
}
 
Example 3
Project: jstorm-0.9.6.3-   File: SequenceTopology.java   Source Code and License Vote up 4 votes
@SuppressWarnings("unchecked")
public static void SetBuilder(TopologyBuilder builder, Map conf) {

	int spout_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
	int bolt_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

	builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
			new SequenceSpout(), spout_Parallelism_hint);

	boolean isEnableSplit = JStormUtils.parseBoolean(
			conf.get("enable.split"), false);

	if (isEnableSplit == false) {
		BoltDeclarer boltDeclarer = builder.setBolt(
				SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(),
				bolt_Parallelism_hint);

		// localFirstGrouping is only for jstorm
		// boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
		boltDeclarer
				.localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME)
				.addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
	} else {

		builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
				new SplitRecord(), bolt_Parallelism_hint)
				.localOrShuffleGrouping(
						SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

		builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.TRADE_STREAM_ID);
		builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.CUSTOMER_STREAM_ID);

		builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
				new MergeRecord(), bolt_Parallelism_hint)
				.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
						new Fields("ID"))
				.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
						new Fields("ID"));

		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).noneGrouping(
				SequenceTopologyDef.MERGE_BOLT_NAME);
	}

	boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
			false);
	if (kryoEnable == true) {
		System.out.println("Use Kryo ");
		boolean useJavaSer = JStormUtils.parseBoolean(
				conf.get("fall.back.on.java.serialization"), true);

		Config.setFallBackOnJavaSerialization(conf, useJavaSer);

		Config.registerSerialization(conf, TradeCustomer.class);
		Config.registerSerialization(conf, Pair.class);
	}

	// conf.put(Config.TOPOLOGY_DEBUG, false);
	// conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
	// conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);

	int ackerNum = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
	Config.setNumAckers(conf, ackerNum);
	// conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
	// conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
	// conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);

	int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
			20);
	conf.put(Config.TOPOLOGY_WORKERS, workerNum);

}
 
Example 4
Project: learn_jstorm   File: SequenceTopologyTool.java   Source Code and License Vote up 4 votes
public StormTopology buildTopology()
{
	Config conf = getConf();
	TopologyBuilder builder = new TopologyBuilder();

	int spout_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
	int bolt_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

	builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
			new SequenceSpout(), spout_Parallelism_hint);

	boolean isEnableSplit = JStormUtils.parseBoolean(
			conf.get("enable.split"), false);

	if (isEnableSplit == false) {
		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).localFirstGrouping(
				SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
	} else {

		builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
				new SplitRecord(), bolt_Parallelism_hint)
				.localOrShuffleGrouping(
						SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

		builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.TRADE_STREAM_ID);
		builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.CUSTOMER_STREAM_ID);

		builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
				new MergeRecord(), bolt_Parallelism_hint)
				.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
						new Fields("ID"))
				.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
						new Fields("ID"));

		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).noneGrouping(
				SequenceTopologyDef.MERGE_BOLT_NAME);
	}

	boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
			false);
	if (kryoEnable == true) {
		System.out.println("Use Kryo ");
		boolean useJavaSer = JStormUtils.parseBoolean(
				conf.get("fall.back.on.java.serialization"), true);

		Config.setFallBackOnJavaSerialization(conf, useJavaSer);

		Config.registerSerialization(conf, TradeCustomer.class);
		Config.registerSerialization(conf, Pair.class);
	}
	int ackerNum = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
	Config.setNumAckers(conf, ackerNum);

	int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
			20);
	conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	return  builder.createTopology();	
}
 
Example 5
Project: learn_jstorm   File: SequenceTopology.java   Source Code and License Vote up 4 votes
public static void SetBuilder(TopologyBuilder builder, Map conf) {

		int spout_Parallelism_hint = JStormUtils.parseInt(
				conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
		int bolt_Parallelism_hint = JStormUtils.parseInt(
				conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

		builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
				new SequenceSpout(), spout_Parallelism_hint);

		boolean isEnableSplit = JStormUtils.parseBoolean(
				conf.get("enable.split"), false);

		if (isEnableSplit == false) {
			BoltDeclarer boltDeclarer = builder.setBolt(
					SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(),
					bolt_Parallelism_hint);

			// localFirstGrouping is only for jstorm
			// boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
			boltDeclarer
					.localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME)
					.addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
		} else {

			builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
					new SplitRecord(), bolt_Parallelism_hint)
					.localOrShuffleGrouping(
							SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

			builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
					new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
					SequenceTopologyDef.SPLIT_BOLT_NAME,
					SequenceTopologyDef.TRADE_STREAM_ID);
			builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
					new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
					SequenceTopologyDef.SPLIT_BOLT_NAME,
					SequenceTopologyDef.CUSTOMER_STREAM_ID);

			builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
					new MergeRecord(), bolt_Parallelism_hint)
					.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
							new Fields("ID"))
					.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
							new Fields("ID"));

			builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
					new TotalCount(), bolt_Parallelism_hint).noneGrouping(
					SequenceTopologyDef.MERGE_BOLT_NAME);
		}

		boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
				false);
		if (kryoEnable == true) {
			System.out.println("Use Kryo ");
			boolean useJavaSer = JStormUtils.parseBoolean(
					conf.get("fall.back.on.java.serialization"), true);

			Config.setFallBackOnJavaSerialization(conf, useJavaSer);

			Config.registerSerialization(conf, TradeCustomer.class);
			Config.registerSerialization(conf, Pair.class);
		}

		// conf.put(Config.TOPOLOGY_DEBUG, false);
		// conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
		// conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);

		int ackerNum = JStormUtils.parseInt(
				conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
		Config.setNumAckers(conf, ackerNum);
		// conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
		// conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
		// conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);

		int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
				20);
		conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	}
 
Example 6
Project: jstrom   File: SequenceTopologyTool.java   Source Code and License Vote up 4 votes
public StormTopology buildTopology()
{
	Config conf = getConf();
	TopologyBuilder builder = new TopologyBuilder();

	int spout_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
	int bolt_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

	builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
			new SequenceSpout(), spout_Parallelism_hint);

	boolean isEnableSplit = JStormUtils.parseBoolean(
			conf.get("enable.split"), false);

	if (isEnableSplit == false) {
		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).localFirstGrouping(
				SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
	} else {

		builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
				new SplitRecord(), bolt_Parallelism_hint)
				.localOrShuffleGrouping(
						SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

		builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.TRADE_STREAM_ID);
		builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.CUSTOMER_STREAM_ID);

		builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
				new MergeRecord(), bolt_Parallelism_hint)
				.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
						new Fields("ID"))
				.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
						new Fields("ID"));

		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).noneGrouping(
				SequenceTopologyDef.MERGE_BOLT_NAME);
	}

	boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
			false);
	if (kryoEnable == true) {
		System.out.println("Use Kryo ");
		boolean useJavaSer = JStormUtils.parseBoolean(
				conf.get("fall.back.on.java.serialization"), true);

		Config.setFallBackOnJavaSerialization(conf, useJavaSer);

		Config.registerSerialization(conf, TradeCustomer.class);
		Config.registerSerialization(conf, Pair.class);
	}
	int ackerNum = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
	Config.setNumAckers(conf, ackerNum);

	int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
			20);
	conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	return  builder.createTopology();	
}
 
Example 7
Project: jstrom   File: SequenceTopology.java   Source Code and License Vote up 4 votes
public static void SetBuilder(TopologyBuilder builder, Map conf) {

		int spout_Parallelism_hint = JStormUtils.parseInt(
				conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
		int bolt_Parallelism_hint = JStormUtils.parseInt(
				conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

		builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
				new SequenceSpout(), spout_Parallelism_hint);

		boolean isEnableSplit = JStormUtils.parseBoolean(
				conf.get("enable.split"), false);

		if (isEnableSplit == false) {
			BoltDeclarer boltDeclarer = builder.setBolt(
					SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(),
					bolt_Parallelism_hint);

			// localFirstGrouping is only for jstorm
			// boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
			boltDeclarer
					.shuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME)
					.addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
		} else {

			builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
					new SplitRecord(), bolt_Parallelism_hint)
					.localOrShuffleGrouping(
							SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

			builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
					new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
					SequenceTopologyDef.SPLIT_BOLT_NAME,
					SequenceTopologyDef.TRADE_STREAM_ID);
			builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
					new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
					SequenceTopologyDef.SPLIT_BOLT_NAME,
					SequenceTopologyDef.CUSTOMER_STREAM_ID);

			builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
					new MergeRecord(), bolt_Parallelism_hint)
					.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
							new Fields("ID"))
					.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
							new Fields("ID"));

			builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
					new TotalCount(), bolt_Parallelism_hint).noneGrouping(
					SequenceTopologyDef.MERGE_BOLT_NAME);
		}

		boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
				false);
		if (kryoEnable == true) {
			System.out.println("Use Kryo ");
			boolean useJavaSer = JStormUtils.parseBoolean(
					conf.get("fall.back.on.java.serialization"), true);

			Config.setFallBackOnJavaSerialization(conf, useJavaSer);

			Config.registerSerialization(conf, TradeCustomer.class);
			Config.registerSerialization(conf, Pair.class);
		}

		// conf.put(Config.TOPOLOGY_DEBUG, false);
		// conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
		// conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);

		int ackerNum = JStormUtils.parseInt(
				conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
		Config.setNumAckers(conf, ackerNum);
		// conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
		// conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
		// conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);

		int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
				20);
		conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	}