Java Code Examples for backtype.storm.Config.registerSerialization()

The following are Jave code examples for showing how to use registerSerialization() of the backtype.storm.Config class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
Example 1
Project: alfresco-apache-storm-demo   File: ConfigurableTopology.java   Source Code and License Vote up 6 votes
protected int submit(String name, Config conf, TopologyBuilder builder) {

        // register Metadata for serialization with FieldsSerializer
        Config.registerSerialization(conf, Metadata.class);

        if (isLocal) {
            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology(name, conf, builder.createTopology());
            if (ttl != -1) {
                Utils.sleep(ttl * 1000);
                cluster.shutdown();
            }
        }

        else {
            try {
                StormSubmitter.submitTopology(name, conf,
                        builder.createTopology());
            } catch (Exception e) {
                e.printStackTrace();
                return -1;
            }
        }
        return 0;
    }
 
Example 2
Project: jstorm-0.9.6.3-   File: MasterBatchCoordinator.java   Source Code and License Vote up 5 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    ret.registerSerialization(TransactionAttempt.class);
    return ret;
}
 
Example 3
Project: jstorm-0.9.6.3-   File: RichSpoutBatchTriggerer.java   Source Code and License Vote up 5 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Map<String, Object> conf = _delegate.getComponentConfiguration();
    if(conf==null) conf = new HashMap();
    else conf = new HashMap(conf);
    Config.registerSerialization(conf, RichSpoutBatchId.class, RichSpoutBatchIdSerializer.class);
    return conf;
}
 
Example 4
Project: Tstream   File: RichSpoutBatchTriggerer.java   Source Code and License Vote up 5 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Map<String, Object> conf = _delegate.getComponentConfiguration();
    if(conf==null) conf = new HashMap();
    else conf = new HashMap(conf);
    Config.registerSerialization(conf, RichSpoutBatchId.class, RichSpoutBatchIdSerializer.class);
    return conf;
}
 
Example 5
Project: Infrastructure   File: TestTopology.java   Source Code and License Vote up 5 votes
@Override
public TopologyOutput createMainTopology() {
    Config config = new Config();
    config.setMessageTimeoutSecs(100);
    Config.registerSerialization(config, DataItem.class, DataItemSerializer.class);
    RecordingTopologyBuilder builder = new RecordingTopologyBuilder(options);
    builder.setSpout("IntermediarySpout", new TestIntermediarySpout("IntermediarySpout", TOPOLOGY_NAME, 
            "IntermediarySpoutStreamId"), 1).setNumTasks(1);
    BoltDeclarer endBolt = builder.setBolt("EndBolt", new TestEndBolt("EndBolt", TOPOLOGY_NAME
            , "EndBoltStreamId"), 1);
    endBolt.shuffleGrouping("IntermediarySpout", "IntermediarySpoutStreamId");
    return new TopologyOutput(config, builder, 1);
}
 
Example 6
Project: learn_jstorm   File: MasterBatchCoordinator.java   Source Code and License Vote up 5 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    ret.registerSerialization(TransactionAttempt.class);
    return ret;
}
 
Example 7
Project: Tstream   File: MasterBatchCoordinator.java   Source Code and License Vote up 5 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    ret.registerSerialization(TransactionAttempt.class);
    return ret;
}
 
Example 8
Project: Infrastructure   File: KryoTupleSerializerTest.java   Source Code and License Vote up 5 votes
/**
 * Test.
 */
@SuppressWarnings("rawtypes")
@Test
public void test() {
    //create a data item
    DataItem dataItem = new DataItem(1, "data");
    List<Object> tupleValues = new ArrayList<Object>();
    tupleValues.add(dataItem);
    
    //create a general tuple
    IGeneralTuple generalTuple = new GeneralTuple(tupleValues);
    //asserts the type
    Assert.assertEquals(true, generalTuple.isGeneralTuple());
    
    //create a switch tuple
    ISwitchTuple switchTuple = new SwitchTuple(1, tupleValues);
    //asserts the type
    Assert.assertEquals(false, switchTuple.isGeneralTuple());
    
    //get the storm kryo map
    Map conf = StormTestUtils.createStormKryoConf();
    //register the custom serializer
    Config.registerSerialization(conf, DataItem.class, DataItemSerializer.class);
    
    //serialize a general tuple
    KryoGeneralTupleSerializer genSer = new KryoGeneralTupleSerializer(conf);        
    assertSerialization(dataItem, generalTuple, genSer);
    
    //serialize a switch tuple
    KryoSwitchTupleSerializer swiSer = new KryoSwitchTupleSerializer(conf);        
    assertSerialization(dataItem, switchTuple, swiSer);

}
 
Example 9
Project: alfresco-apache-storm-demo   File: TestMetadataSerialization.java   Source Code and License Vote up 5 votes
@Test
public void testSerialization() throws IOException {
    Map conf = Utils.readDefaultConfig();
    Config.registerSerialization(conf, Metadata.class);

    KryoValuesSerializer kvs = new KryoValuesSerializer(conf);
    Metadata md = new Metadata();
    byte[] content = kvs.serializeObject(md);

    KryoValuesDeserializer kvd = new KryoValuesDeserializer(conf);
    Metadata md2 = (Metadata) kvd.deserializeObject(content);

    // TODO compare md1 and md2
}
 
Example 10
Project: jstrom   File: MasterBatchCoordinator.java   Source Code and License Vote up 5 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    ret.registerSerialization(TransactionAttempt.class);
    return ret;
}
 
Example 11
Project: jstrom   File: RichSpoutBatchTriggerer.java   Source Code and License Vote up 5 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Map<String, Object> conf = _delegate.getComponentConfiguration();
    if (conf == null)
        conf = new HashMap();
    else
        conf = new HashMap(conf);
    Config.registerSerialization(conf, RichSpoutBatchId.class, RichSpoutBatchIdSerializer.class);
    return conf;
}
 
Example 12
Project: jstorm-0.9.6.3-   File: SequenceTopologyTool.java   Source Code and License Vote up 4 votes
public StormTopology buildTopology()
{
	Config conf = getConf();
	TopologyBuilder builder = new TopologyBuilder();

	int spout_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
	int bolt_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

	builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
			new SequenceSpout(), spout_Parallelism_hint);

	boolean isEnableSplit = JStormUtils.parseBoolean(
			conf.get("enable.split"), false);

	if (isEnableSplit == false) {
		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).localFirstGrouping(
				SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
	} else {

		builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
				new SplitRecord(), bolt_Parallelism_hint)
				.localOrShuffleGrouping(
						SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

		builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.TRADE_STREAM_ID);
		builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.CUSTOMER_STREAM_ID);

		builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
				new MergeRecord(), bolt_Parallelism_hint)
				.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
						new Fields("ID"))
				.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
						new Fields("ID"));

		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).noneGrouping(
				SequenceTopologyDef.MERGE_BOLT_NAME);
	}

	boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
			false);
	if (kryoEnable == true) {
		System.out.println("Use Kryo ");
		boolean useJavaSer = JStormUtils.parseBoolean(
				conf.get("fall.back.on.java.serialization"), true);

		Config.setFallBackOnJavaSerialization(conf, useJavaSer);

		Config.registerSerialization(conf, TradeCustomer.class);
		Config.registerSerialization(conf, Pair.class);
	}
	int ackerNum = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
	Config.setNumAckers(conf, ackerNum);

	int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
			20);
	conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	return  builder.createTopology();	
}
 
Example 13
Project: jstorm-0.9.6.3-   File: SequenceTopology.java   Source Code and License Vote up 4 votes
@SuppressWarnings("unchecked")
public static void SetBuilder(TopologyBuilder builder, Map conf) {

	int spout_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
	int bolt_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

	builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
			new SequenceSpout(), spout_Parallelism_hint);

	boolean isEnableSplit = JStormUtils.parseBoolean(
			conf.get("enable.split"), false);

	if (isEnableSplit == false) {
		BoltDeclarer boltDeclarer = builder.setBolt(
				SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(),
				bolt_Parallelism_hint);

		// localFirstGrouping is only for jstorm
		// boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
		boltDeclarer
				.localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME)
				.addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
	} else {

		builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
				new SplitRecord(), bolt_Parallelism_hint)
				.localOrShuffleGrouping(
						SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

		builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.TRADE_STREAM_ID);
		builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.CUSTOMER_STREAM_ID);

		builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
				new MergeRecord(), bolt_Parallelism_hint)
				.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
						new Fields("ID"))
				.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
						new Fields("ID"));

		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).noneGrouping(
				SequenceTopologyDef.MERGE_BOLT_NAME);
	}

	boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
			false);
	if (kryoEnable == true) {
		System.out.println("Use Kryo ");
		boolean useJavaSer = JStormUtils.parseBoolean(
				conf.get("fall.back.on.java.serialization"), true);

		Config.setFallBackOnJavaSerialization(conf, useJavaSer);

		Config.registerSerialization(conf, TradeCustomer.class);
		Config.registerSerialization(conf, Pair.class);
	}

	// conf.put(Config.TOPOLOGY_DEBUG, false);
	// conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
	// conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);

	int ackerNum = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
	Config.setNumAckers(conf, ackerNum);
	// conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
	// conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
	// conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);

	int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
			20);
	conf.put(Config.TOPOLOGY_WORKERS, workerNum);

}
 
Example 14
Project: jstorm-0.9.6.3-   File: OpaqueMemoryTransactionalSpout.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
	Config conf = new Config();
	conf.registerSerialization(MemoryTransactionalSpoutMeta.class);
	return conf;
}
 
Example 15
Project: Tstream   File: OpaqueMemoryTransactionalSpout.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
	Config conf = new Config();
	conf.registerSerialization(MemoryTransactionalSpoutMeta.class);
	return conf;
}
 
Example 16
Project: learn_jstorm   File: OpaqueMemoryTransactionalSpout.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
	Config conf = new Config();
	conf.registerSerialization(MemoryTransactionalSpoutMeta.class);
	return conf;
}
 
Example 17
Project: jstrom   File: SequenceTopologyTool.java   Source Code and License Vote up 4 votes
public StormTopology buildTopology()
{
	Config conf = getConf();
	TopologyBuilder builder = new TopologyBuilder();

	int spout_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
	int bolt_Parallelism_hint = JStormUtils.parseInt(
			conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

	builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
			new SequenceSpout(), spout_Parallelism_hint);

	boolean isEnableSplit = JStormUtils.parseBoolean(
			conf.get("enable.split"), false);

	if (isEnableSplit == false) {
		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).localFirstGrouping(
				SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
	} else {

		builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
				new SplitRecord(), bolt_Parallelism_hint)
				.localOrShuffleGrouping(
						SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

		builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.TRADE_STREAM_ID);
		builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
				new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
				SequenceTopologyDef.SPLIT_BOLT_NAME,
				SequenceTopologyDef.CUSTOMER_STREAM_ID);

		builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
				new MergeRecord(), bolt_Parallelism_hint)
				.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
						new Fields("ID"))
				.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
						new Fields("ID"));

		builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
				new TotalCount(), bolt_Parallelism_hint).noneGrouping(
				SequenceTopologyDef.MERGE_BOLT_NAME);
	}

	boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
			false);
	if (kryoEnable == true) {
		System.out.println("Use Kryo ");
		boolean useJavaSer = JStormUtils.parseBoolean(
				conf.get("fall.back.on.java.serialization"), true);

		Config.setFallBackOnJavaSerialization(conf, useJavaSer);

		Config.registerSerialization(conf, TradeCustomer.class);
		Config.registerSerialization(conf, Pair.class);
	}
	int ackerNum = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
	Config.setNumAckers(conf, ackerNum);

	int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
			20);
	conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	return  builder.createTopology();	
}
 
Example 18
Project: jstrom   File: SequenceTopology.java   Source Code and License Vote up 4 votes
public static void SetBuilder(TopologyBuilder builder, Map conf) {

		int spout_Parallelism_hint = JStormUtils.parseInt(
				conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
		int bolt_Parallelism_hint = JStormUtils.parseInt(
				conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);

		builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME,
				new SequenceSpout(), spout_Parallelism_hint);

		boolean isEnableSplit = JStormUtils.parseBoolean(
				conf.get("enable.split"), false);

		if (isEnableSplit == false) {
			BoltDeclarer boltDeclarer = builder.setBolt(
					SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(),
					bolt_Parallelism_hint);

			// localFirstGrouping is only for jstorm
			// boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
			boltDeclarer
					.shuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME)
					.addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
		} else {

			builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME,
					new SplitRecord(), bolt_Parallelism_hint)
					.localOrShuffleGrouping(
							SequenceTopologyDef.SEQUENCE_SPOUT_NAME);

			builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME,
					new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
					SequenceTopologyDef.SPLIT_BOLT_NAME,
					SequenceTopologyDef.TRADE_STREAM_ID);
			builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
					new PairCount(), bolt_Parallelism_hint).shuffleGrouping(
					SequenceTopologyDef.SPLIT_BOLT_NAME,
					SequenceTopologyDef.CUSTOMER_STREAM_ID);

			builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME,
					new MergeRecord(), bolt_Parallelism_hint)
					.fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME,
							new Fields("ID"))
					.fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME,
							new Fields("ID"));

			builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME,
					new TotalCount(), bolt_Parallelism_hint).noneGrouping(
					SequenceTopologyDef.MERGE_BOLT_NAME);
		}

		boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"),
				false);
		if (kryoEnable == true) {
			System.out.println("Use Kryo ");
			boolean useJavaSer = JStormUtils.parseBoolean(
					conf.get("fall.back.on.java.serialization"), true);

			Config.setFallBackOnJavaSerialization(conf, useJavaSer);

			Config.registerSerialization(conf, TradeCustomer.class);
			Config.registerSerialization(conf, Pair.class);
		}

		// conf.put(Config.TOPOLOGY_DEBUG, false);
		// conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
		// conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);

		int ackerNum = JStormUtils.parseInt(
				conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
		Config.setNumAckers(conf, ackerNum);
		// conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
		// conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
		// conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);

		int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS),
				20);
		conf.put(Config.TOPOLOGY_WORKERS, workerNum);

	}
 
Example 19
Project: jstrom   File: OpaqueMemoryTransactionalSpout.java   Source Code and License Vote up 4 votes
@Override
public Map<String, Object> getComponentConfiguration() {
    Config conf = new Config();
    conf.registerSerialization(MemoryTransactionalSpoutMeta.class);
    return conf;
}
 
Example 20
Project: aeolus   File: AbstractBatchCollector.java   Source Code and License Vote up 2 votes
/**
 * Registers the classes {@link Batch Batch.class} and {@link BatchColumn BatchColumn.class} for serialization and
 * deserialization.
 * 
 * @param stormConfig
 *            The storm config the which the classes should be registered to.
 */
public static void registerKryoClasses(Config stormConfig) {
	stormConfig.registerSerialization(Batch.class);
	stormConfig.registerSerialization(BatchColumn.class);
}