Java Code Examples for backtype.storm.Config.setFallBackOnJavaSerialization()

The following are Jave code examples for showing how to use setFallBackOnJavaSerialization() of the backtype.storm.Config class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
+ Save this method
Example 1
Project: openimaj   File: StormPlayground.java   View Source Code Vote up 6 votes
public static void main(String[] args) {
	final Config conf = new Config();
	conf.setDebug(false);
	conf.setNumWorkers(2);
	conf.setMaxSpoutPending(1);
	conf.setFallBackOnJavaSerialization(false);
	conf.setSkipMissingKryoRegistrations(false);
	final LocalCluster cluster = new LocalCluster();
	final TopologyBuilder builder = new TopologyBuilder();
	builder.setSpout("randomSpout1", new RandomFieldSpout(2, 0, 0, 1)); // (nfields,seed,min,max)
	builder.setSpout("randomSpout2", new RandomFieldSpout(2, 10, 0, 1)); // (nfields,seed,min,max)
	JoinBolt.connectNewBolt(builder);
	final StormTopology topology = builder.createTopology();
	cluster.submitTopology("playTopology", conf, topology);
	Utils.sleep(10000);
	cluster.killTopology("playTopology");
	cluster.shutdown();

}
 
Example 2
Project: jstorm-0.9.6.3-   File: SequenceTopologyTool.java   View Source Code Vote up 4 votes
public StormTopology buildTopology()
{
	Config conf = getConf();
	TopologyBuilder builder = new TopologyBuilder();

	int spout_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
	int bolt_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

	builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
			new SequenceSpout(), spout_Parallelism_hint);

	boolean isEnableSplit = JStormUtils.parseBoolean(
			conf.get("enable.split"), false);

	if (isEnableSplit == false) {
		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).localFirstGrouping(
				SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
	} else {

		builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
				new SplitRecord(), bolt_Parallelism_hint)
				.localOrShuffleGrouping(
						SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

		builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.TRADE_STREAM_ID);
		builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.CUSTOMER_STREAM_ID);

		builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
				new MergeRecord(), bolt_Parallelism_hint)
				.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
						new Fields("ID"))
				.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
						new Fields("ID"));

		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).noneGrouping(
				SequenceTopologyDef.MERGE_BOLT_NAME);
	}

	boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
			false);
	if (kryoEnable == true) {
		System.out.println("Use Kryo ");
		boolean useJavaSer = JStormUtils.parseBoolean(
				conf.get("fall.back.on.java.serialization"), true);

		Config.setFallBackOnJavaSerialization(conf, useJavaSer);

		Config.registerSerialization(conf, TradeCustomer.class);
		Config.registerSerialization(conf, Pair.class);
	}
	int ackerNum = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
	Config.setNumAckers(conf, ackerNum);

	int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
			20);
	conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	return  builder.createTopology();	
}
 
Example 3
Project: jstorm-0.9.6.3-   File: SequenceTopology.java   View Source Code Vote up 4 votes
@SuppressWarnings("unchecked")
public static void SetBuilder(TopologyBuilder builder, Map conf) {

	int spout_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
	int bolt_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

	builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
			new SequenceSpout(), spout_Parallelism_hint);

	boolean isEnableSplit = JStormUtils.parseBoolean(
			conf.get("enable.split"), false);

	if (isEnableSplit == false) {
		BoltDeclarer boltDeclarer = builder.setBolt(
				SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(),
				bolt_Parallelism_hint);

		// localFirstGrouping is only for jstorm
		// boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
		boltDeclarer
				.localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME)
				.addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
	} else {

		builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
				new SplitRecord(), bolt_Parallelism_hint)
				.localOrShuffleGrouping(
						SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

		builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.TRADE_STREAM_ID);
		builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.CUSTOMER_STREAM_ID);

		builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
				new MergeRecord(), bolt_Parallelism_hint)
				.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
						new Fields("ID"))
				.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
						new Fields("ID"));

		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).noneGrouping(
				SequenceTopologyDef.MERGE_BOLT_NAME);
	}

	boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
			false);
	if (kryoEnable == true) {
		System.out.println("Use Kryo ");
		boolean useJavaSer = JStormUtils.parseBoolean(
				conf.get("fall.back.on.java.serialization"), true);

		Config.setFallBackOnJavaSerialization(conf, useJavaSer);

		Config.registerSerialization(conf, TradeCustomer.class);
		Config.registerSerialization(conf, Pair.class);
	}

	// conf.put(Config.TOPOLOGY_DEBUG, false);
	// conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
	// conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);

	int ackerNum = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
	Config.setNumAckers(conf, ackerNum);
	// conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
	// conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
	// conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);

	int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
			20);
	conf.put(Config.TOPOLOGY_WORKERS, workerNum);

}
 
Example 4
Project: learn_jstorm   File: SequenceTopologyTool.java   View Source Code Vote up 4 votes
public StormTopology buildTopology()
{
	Config conf = getConf();
	TopologyBuilder builder = new TopologyBuilder();

	int spout_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
	int bolt_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

	builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
			new SequenceSpout(), spout_Parallelism_hint);

	boolean isEnableSplit = JStormUtils.parseBoolean(
			conf.get("enable.split"), false);

	if (isEnableSplit == false) {
		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).localFirstGrouping(
				SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
	} else {

		builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
				new SplitRecord(), bolt_Parallelism_hint)
				.localOrShuffleGrouping(
						SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

		builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.TRADE_STREAM_ID);
		builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.CUSTOMER_STREAM_ID);

		builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
				new MergeRecord(), bolt_Parallelism_hint)
				.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
						new Fields("ID"))
				.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
						new Fields("ID"));

		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).noneGrouping(
				SequenceTopologyDef.MERGE_BOLT_NAME);
	}

	boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
			false);
	if (kryoEnable == true) {
		System.out.println("Use Kryo ");
		boolean useJavaSer = JStormUtils.parseBoolean(
				conf.get("fall.back.on.java.serialization"), true);

		Config.setFallBackOnJavaSerialization(conf, useJavaSer);

		Config.registerSerialization(conf, TradeCustomer.class);
		Config.registerSerialization(conf, Pair.class);
	}
	int ackerNum = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
	Config.setNumAckers(conf, ackerNum);

	int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
			20);
	conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	return  builder.createTopology();	
}
 
Example 5
Project: learn_jstorm   File: SequenceTopology.java   View Source Code Vote up 4 votes
public static void SetBuilder(TopologyBuilder builder, Map conf) {

		int spout_Parallelism_hint = JStormUtils.parseInt(
				conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
		int bolt_Parallelism_hint = JStormUtils.parseInt(
				conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

		builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
				new SequenceSpout(), spout_Parallelism_hint);

		boolean isEnableSplit = JStormUtils.parseBoolean(
				conf.get("enable.split"), false);

		if (isEnableSplit == false) {
			BoltDeclarer boltDeclarer = builder.setBolt(
					SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(),
					bolt_Parallelism_hint);

			// localFirstGrouping is only for jstorm
			// boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
			boltDeclarer
					.localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME)
					.addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
		} else {

			builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
					new SplitRecord(), bolt_Parallelism_hint)
					.localOrShuffleGrouping(
							SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

			builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
					new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
					SequenceTopologyDef.SPLIT_BOLT_NAME,
					SequenceTopologyDef.TRADE_STREAM_ID);
			builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
					new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
					SequenceTopologyDef.SPLIT_BOLT_NAME,
					SequenceTopologyDef.CUSTOMER_STREAM_ID);

			builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
					new MergeRecord(), bolt_Parallelism_hint)
					.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
							new Fields("ID"))
					.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
							new Fields("ID"));

			builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
					new TotalCount(), bolt_Parallelism_hint).noneGrouping(
					SequenceTopologyDef.MERGE_BOLT_NAME);
		}

		boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
				false);
		if (kryoEnable == true) {
			System.out.println("Use Kryo ");
			boolean useJavaSer = JStormUtils.parseBoolean(
					conf.get("fall.back.on.java.serialization"), true);

			Config.setFallBackOnJavaSerialization(conf, useJavaSer);

			Config.registerSerialization(conf, TradeCustomer.class);
			Config.registerSerialization(conf, Pair.class);
		}

		// conf.put(Config.TOPOLOGY_DEBUG, false);
		// conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
		// conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);

		int ackerNum = JStormUtils.parseInt(
				conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
		Config.setNumAckers(conf, ackerNum);
		// conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
		// conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
		// conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);

		int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
				20);
		conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	}
 
Example 6
Project: jstrom   File: SequenceTopologyTool.java   View Source Code Vote up 4 votes
public StormTopology buildTopology()
{
	Config conf = getConf();
	TopologyBuilder builder = new TopologyBuilder();

	int spout_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
	int bolt_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

	builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
			new SequenceSpout(), spout_Parallelism_hint);

	boolean isEnableSplit = JStormUtils.parseBoolean(
			conf.get("enable.split"), false);

	if (isEnableSplit == false) {
		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).localFirstGrouping(
				SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
	} else {

		builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
				new SplitRecord(), bolt_Parallelism_hint)
				.localOrShuffleGrouping(
						SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

		builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.TRADE_STREAM_ID);
		builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.CUSTOMER_STREAM_ID);

		builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
				new MergeRecord(), bolt_Parallelism_hint)
				.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
						new Fields("ID"))
				.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
						new Fields("ID"));

		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).noneGrouping(
				SequenceTopologyDef.MERGE_BOLT_NAME);
	}

	boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
			false);
	if (kryoEnable == true) {
		System.out.println("Use Kryo ");
		boolean useJavaSer = JStormUtils.parseBoolean(
				conf.get("fall.back.on.java.serialization"), true);

		Config.setFallBackOnJavaSerialization(conf, useJavaSer);

		Config.registerSerialization(conf, TradeCustomer.class);
		Config.registerSerialization(conf, Pair.class);
	}
	int ackerNum = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
	Config.setNumAckers(conf, ackerNum);

	int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
			20);
	conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	return  builder.createTopology();	
}
 
Example 7
Project: jstrom   File: SequenceTopology.java   View Source Code Vote up 4 votes
public static void SetBuilder(TopologyBuilder builder, Map conf) {

		int spout_Parallelism_hint = JStormUtils.parseInt(
				conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
		int bolt_Parallelism_hint = JStormUtils.parseInt(
				conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

		builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
				new SequenceSpout(), spout_Parallelism_hint);

		boolean isEnableSplit = JStormUtils.parseBoolean(
				conf.get("enable.split"), false);

		if (isEnableSplit == false) {
			BoltDeclarer boltDeclarer = builder.setBolt(
					SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(),
					bolt_Parallelism_hint);

			// localFirstGrouping is only for jstorm
			// boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
			boltDeclarer
					.shuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME)
					.addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
		} else {

			builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
					new SplitRecord(), bolt_Parallelism_hint)
					.localOrShuffleGrouping(
							SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

			builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
					new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
					SequenceTopologyDef.SPLIT_BOLT_NAME,
					SequenceTopologyDef.TRADE_STREAM_ID);
			builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
					new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
					SequenceTopologyDef.SPLIT_BOLT_NAME,
					SequenceTopologyDef.CUSTOMER_STREAM_ID);

			builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
					new MergeRecord(), bolt_Parallelism_hint)
					.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
							new Fields("ID"))
					.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
							new Fields("ID"));

			builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
					new TotalCount(), bolt_Parallelism_hint).noneGrouping(
					SequenceTopologyDef.MERGE_BOLT_NAME);
		}

		boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
				false);
		if (kryoEnable == true) {
			System.out.println("Use Kryo ");
			boolean useJavaSer = JStormUtils.parseBoolean(
					conf.get("fall.back.on.java.serialization"), true);

			Config.setFallBackOnJavaSerialization(conf, useJavaSer);

			Config.registerSerialization(conf, TradeCustomer.class);
			Config.registerSerialization(conf, Pair.class);
		}

		// conf.put(Config.TOPOLOGY_DEBUG, false);
		// conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
		// conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);

		int ackerNum = JStormUtils.parseInt(
				conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
		Config.setNumAckers(conf, ackerNum);
		// conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
		// conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
		// conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);

		int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
				20);
		conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	}