Java Code Examples for backtype.storm.Config#registerSerialization()

The following examples show how to use backtype.storm.Config#registerSerialization() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: SwitchBoltIT.java From flowmix with Apache License 2.0

6 votes

@Test
public void test_timeDiffActivated_countEviction() throws InterruptedException {
    Flow flow = new FlowBuilder()
            .id("flow")
            .flowDefs()
            .stream("stream1")
            .stopGate().open(Policy.TIME_DELTA_LT, 1000).close(Policy.TIME, 5).evict(Policy.COUNT, 5).end()
            .endStream()   // send ALL results to stream2 and not to standard output
            .endDefs()
            .createFlow();

    StormTopology topology = buildTopology(flow, 50);
    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, topology);

    Thread.sleep(5000);

    assertEquals(5, MockSinkBolt.getEvents().size());
}

Example 2

Source File: SwitchBoltIT.java From flowmix with Apache License 2.0

6 votes

@Test
public void test_timeDiffActivated_timeEviction() throws InterruptedException {
    Flow flow = new FlowBuilder()
            .id("flow")
            .flowDefs()
            .stream("stream1")
            .stopGate().open(Policy.TIME_DELTA_LT, 5).close(Policy.TIME, 1).evict(Policy.TIME, 1).end()
            .endStream()   // send ALL results to stream2 and not to standard output
            .endDefs()
            .createFlow();

    StormTopology topology = buildTopology(flow, 50);
    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, topology);

    Thread.sleep(7000);

    int size = MockSinkBolt.getEvents().size();
    System.out.println("SIZE: " + size);
    assertTrue(size >= 50 && size <= 65);
}

Example 3

Source File: ExampleRunner.java From flowmix with Apache License 2.0

6 votes

public void run() {

    StormTopology topology = new FlowmixBuilder()
        .setFlowLoader(new SimpleFlowLoaderSpout(provider.getFlows(), 60000))
        .setEventsLoader(new MockEventGeneratorSpout(getMockEvents(), 10))
        .setOutputBolt(new PrinterBolt())
        .setParallelismHint(6)
      .create()
    .createTopology();

    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.setMaxSpoutPending(5000);
    conf.setDebug(false);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("example-topology", conf, topology);
  }

Example 4

Source File: SelectorBoltIT.java From flowmix with Apache License 2.0

5 votes

@Test
public void testSelection_basic() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
        .select().fields("key1", "key2").end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertTrue(MockSinkBolt.getEvents().size() > 0);

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("key1"));
    assertNotNull(event.get("key2"));
    assertNull(event.get("key3"));
    assertNull(event.get("key4"));
    assertNull(event.get("key5"));
  }
}

Example 5

Source File: SelectorBoltIT.java From flowmix with Apache License 2.0

5 votes

@Test
public void testSelection_fieldsDontExistDontReturn() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .select().fields("key7").end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertEquals(0, MockSinkBolt.getEvents().size());
}

Example 6

Source File: FilterBoltIT.java From flowmix with Apache License 2.0

5 votes

@Test
public void testFilter_nothingPasses() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
          .filter().filter(new CriteriaFilter(criteriaFromNode(new QueryBuilder().eq("key3", "val50").build()))).end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertEquals(0, MockSinkBolt.getEvents().size());
}

Example 7

Source File: FilterBoltIT.java From flowmix with Apache License 2.0

5 votes

@Test
public void testFilter_eventsPass() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
          .filter().filter(new CriteriaFilter(criteriaFromNode(new QueryBuilder().eq("key1", "val1").build()))).end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertTrue(MockSinkBolt.getEvents().size() > 0);

}

Example 8

Source File: RichSpoutBatchTriggerer.java From jstorm with Apache License 2.0

5 votes

@Override
public Map<String, Object> getComponentConfiguration() {
    Map<String, Object> conf = _delegate.getComponentConfiguration();
    if(conf==null) conf = new HashMap<>();
    else conf = new HashMap<>(conf);
    Config.registerSerialization(conf, RichSpoutBatchId.class, RichSpoutBatchIdSerializer.class);
    return conf;
}

Example 9

Source File: MasterBatchCoordinator.java From jstorm with Apache License 2.0

5 votes

@Override
public Map<String, Object> getComponentConfiguration() {
    Config ret = new Config();
    ret.setMaxTaskParallelism(1);
    ret.registerSerialization(TransactionAttempt.class);
    return ret;
}

Example 10

Source File: AvroUtils.java From jstorm with Apache License 2.0

5 votes

/**
 * A helper method to extract avro serialization configurations from the topology configuration and register
 * specific kryo serializers as necessary.  A default serializer will be provided if none is specified in the
 * configuration.  "avro.serializer" should specify the complete class name of the serializer, e.g.
 * "org.apache.stgorm.hdfs.avro.GenericAvroSerializer"
 *
 * @param conf The topology configuration
 * @throws ClassNotFoundException If the specified serializer cannot be located.
 */
public static void addAvroKryoSerializations(Config conf) throws ClassNotFoundException {
    final Class serializerClass;
    if (conf.containsKey("avro.serializer")) {
        serializerClass = Class.forName((String)conf.get("avro.serializer"));
    }
    else {
        serializerClass = GenericAvroSerializer.class;
    }
    conf.registerSerialization(GenericData.Record.class, serializerClass);
    conf.setSkipMissingKryoRegistrations(false);
}

Example 11

Source File: MemoryTransactionalSpout.java From jstorm with Apache License 2.0

4 votes

@Override
public Map<String, Object> getComponentConfiguration() {
    Config conf = new Config();
    conf.registerSerialization(MemoryTransactionalSpoutMeta.class);
    return conf;
}

Example 12

Source File: OpaqueMemoryTransactionalSpout.java From jstorm with Apache License 2.0

4 votes

@Override
public Map<String, Object> getComponentConfiguration() {
    Config conf = new Config();
    conf.registerSerialization(MemoryTransactionalSpoutMeta.class);
    return conf;
}

Example 13

Source File: SequenceTopology.java From jstorm with Apache License 2.0

4 votes

public static void SetBuilder(TopologyBuilder builder, Map conf) {
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    
    builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, new SequenceSpout(), spout_Parallelism_hint);
    
    boolean isEnableSplit = JStormUtils.parseBoolean(conf.get("enable.split"), false);
    
    if (!isEnableSplit) {
        BoltDeclarer boltDeclarer = builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(),
                bolt_Parallelism_hint);
                
        // localFirstGrouping is only for jstorm
        // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
        boltDeclarer.shuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME)
                .allGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, SequenceTopologyDef.CONTROL_STREAM_ID)
                .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 3);
    } else {
        
        builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME, new SplitRecord(), bolt_Parallelism_hint)
                .localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
                
        builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME, new PairCount(), bolt_Parallelism_hint)
                .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new PairCount(), bolt_Parallelism_hint)
                .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
                
        builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME, new MergeRecord(), bolt_Parallelism_hint)
                .fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME, new Fields("ID"))
                .fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new Fields("ID"));
                
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint)
                .noneGrouping(SequenceTopologyDef.MERGE_BOLT_NAME);
    }
    
    boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"), false);
    if (kryoEnable) {
        System.out.println("Use Kryo ");
        boolean useJavaSer = JStormUtils.parseBoolean(conf.get("fall.back.on.java.serialization"), true);
        
        Config.setFallBackOnJavaSerialization(conf, useJavaSer);
        
        Config.registerSerialization(conf, TradeCustomer.class, TradeCustomerSerializer.class);
        Config.registerSerialization(conf, Pair.class, PairSerializer.class);
    }
    
    // conf.put(Config.TOPOLOGY_DEBUG, false);
    // conf.put(ConfigExtension.TOPOLOGY_DEBUG_RECV_TUPLE, false);
    // conf.put(Config.STORM_LOCAL_MODE_ZMQ, false);
    
    int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
    Config.setNumAckers(conf, ackerNum);
    // conf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 6);
    // conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 20);
    // conf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 1);
    
    int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS), 20);
    conf.put(Config.TOPOLOGY_WORKERS, workerNum);
    
}

Example 14

Source File: SequenceTopologyTool.java From jstorm with Apache License 2.0

4 votes

public StormTopology buildTopology() {
    Config conf = getConf();
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    
    builder.setSpout(SequenceTopologyDef.SEQUENCE_SPOUT_NAME, new SequenceSpout(), spout_Parallelism_hint);
    
    boolean isEnableSplit = JStormUtils.parseBoolean(conf.get("enable.split"), false);
    
    if (!isEnableSplit) {
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint)
                .localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    } else {
        
        builder.setBolt(SequenceTopologyDef.SPLIT_BOLT_NAME, new SplitRecord(), bolt_Parallelism_hint)
                .localOrShuffleGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
                
        builder.setBolt(SequenceTopologyDef.TRADE_BOLT_NAME, new PairCount(), bolt_Parallelism_hint)
                .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.TRADE_STREAM_ID);
        builder.setBolt(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new PairCount(), bolt_Parallelism_hint)
                .shuffleGrouping(SequenceTopologyDef.SPLIT_BOLT_NAME, SequenceTopologyDef.CUSTOMER_STREAM_ID);
                
        builder.setBolt(SequenceTopologyDef.MERGE_BOLT_NAME, new MergeRecord(), bolt_Parallelism_hint)
                .fieldsGrouping(SequenceTopologyDef.TRADE_BOLT_NAME, new Fields("ID"))
                .fieldsGrouping(SequenceTopologyDef.CUSTOMER_BOLT_NAME, new Fields("ID"));
                
        builder.setBolt(SequenceTopologyDef.TOTAL_BOLT_NAME, new TotalCount(), bolt_Parallelism_hint)
                .noneGrouping(SequenceTopologyDef.MERGE_BOLT_NAME);
    }
    
    boolean kryoEnable = JStormUtils.parseBoolean(conf.get("kryo.enable"), false);
    if (kryoEnable) {
        System.out.println("Use Kryo ");
        boolean useJavaSer = JStormUtils.parseBoolean(conf.get("fall.back.on.java.serialization"), true);
        
        Config.setFallBackOnJavaSerialization(conf, useJavaSer);
        
        Config.registerSerialization(conf, TradeCustomer.class);
        Config.registerSerialization(conf, Pair.class);
    }
    int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), 1);
    Config.setNumAckers(conf, ackerNum);
    
    int workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS), 20);
    conf.put(Config.TOPOLOGY_WORKERS, workerNum);
    
    return builder.createTopology();
}

Example 15

Source File: AggregatorBoltIT.java From flowmix with Apache License 2.0

4 votes

@Test
public void test_countTrigger_countEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.COUNT, 5)
          .evict(Policy.COUNT, 2)
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);

  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents().size());
  assertTrue(MockSinkBolt.getEvents().size() > 130);
  assertTrue(MockSinkBolt.getEvents().size() < 160);

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() == 2);
  }
}

Example 16

Source File: AggregatorBoltIT.java From flowmix with Apache License 2.0

4 votes

@Test
public void test_countTrigger_timeEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.COUNT, 600)       // trigger every 500 events received
          .evict(Policy.TIME, 1)            // only keep last 1 second in the window
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);

  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }


  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertEquals(1, MockSinkBolt.getEvents().size());

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() > 90);
    assertTrue(event.<Long>get("aCount").getValue() < 100);
  }
}

Example 17

Source File: AggregatorBoltIT.java From flowmix with Apache License 2.0

4 votes

@Test
public void test_timeTrigger_countEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.TIME, 2)
          .evict(Policy.COUNT, 10)
          .clearOnTrigger()
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);


  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertEquals(4, MockSinkBolt.getEvents().size());

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() == 10);
  }
}

Example 18

Source File: AggregatorBoltIT.java From flowmix with Apache License 2.0

4 votes

@Test
public void test_timeTrigger_timeEvict() {

    Flow flow = new FlowBuilder()
        .id("myflow")
        .flowDefs()
            .stream("stream1")
                .partition().fields("key3").end()
                .aggregate().aggregator(CountAggregator.class)
                  .config(OUTPUT_FIELD, "aCount")
                  .trigger(Policy.TIME, 5)
                  .evict(Policy.TIME, 10)
                  .clearOnTrigger()
                .end()
            .endStream()
        .endDefs()
    .createFlow();

    StormTopology topology = buildTopology(flow, 10);
    Config conf = new Config();
    conf.registerSerialization(Event.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);
    conf.setNumWorkers(1);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology(getTopologyName(), conf, topology);

    try {
      Thread.sleep(25000);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }

    cluster.shutdown();

    System.out.println(MockSinkBolt.getEvents().size());
    assertEquals(4, MockSinkBolt.getEvents().size());

    for(Event event : MockSinkBolt.getEvents()) {
      assertNotNull(event.get("aCount"));
      assertTrue(event.<Long>get("aCount").getValue() > 350);
      assertTrue(event.<Long>get("aCount").getValue() < 500);
    }
}