Java Code Examples for backtype.storm.Config#setSkipMissingKryoRegistrations()

The following examples show how to use backtype.storm.Config#setSkipMissingKryoRegistrations() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: flowmix   File: ExampleRunner.java    License: Apache License 2.0 6 votes vote down vote up
public void run() {

    StormTopology topology = new FlowmixBuilder()
        .setFlowLoader(new SimpleFlowLoaderSpout(provider.getFlows(), 60000))
        .setEventsLoader(new MockEventGeneratorSpout(getMockEvents(), 10))
        .setOutputBolt(new PrinterBolt())
        .setParallelismHint(6)
      .create()
    .createTopology();

    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.setMaxSpoutPending(5000);
    conf.setDebug(false);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("example-topology", conf, topology);
  }
 
Example 2
Source Project: flowmix   File: SwitchBoltIT.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void test_timeDiffActivated_countEviction() throws InterruptedException {
    Flow flow = new FlowBuilder()
            .id("flow")
            .flowDefs()
            .stream("stream1")
            .stopGate().open(Policy.TIME_DELTA_LT, 1000).close(Policy.TIME, 5).evict(Policy.COUNT, 5).end()
            .endStream()   // send ALL results to stream2 and not to standard output
            .endDefs()
            .createFlow();

    StormTopology topology = buildTopology(flow, 50);
    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, topology);

    Thread.sleep(5000);

    assertEquals(5, MockSinkBolt.getEvents().size());
}
 
Example 3
Source Project: flowmix   File: SwitchBoltIT.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void test_timeDiffActivated_timeEviction() throws InterruptedException {
    Flow flow = new FlowBuilder()
            .id("flow")
            .flowDefs()
            .stream("stream1")
            .stopGate().open(Policy.TIME_DELTA_LT, 5).close(Policy.TIME, 1).evict(Policy.TIME, 1).end()
            .endStream()   // send ALL results to stream2 and not to standard output
            .endDefs()
            .createFlow();

    StormTopology topology = buildTopology(flow, 50);
    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, topology);

    Thread.sleep(7000);

    int size = MockSinkBolt.getEvents().size();
    System.out.println("SIZE: " + size);
    assertTrue(size >= 50 && size <= 65);
}
 
Example 4
Source Project: flowmix   File: SelectorBoltIT.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSelection_basic() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
        .select().fields("key1", "key2").end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertTrue(MockSinkBolt.getEvents().size() > 0);

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("key1"));
    assertNotNull(event.get("key2"));
    assertNull(event.get("key3"));
    assertNull(event.get("key4"));
    assertNull(event.get("key5"));
  }
}
 
Example 5
Source Project: flowmix   File: SelectorBoltIT.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSelection_fieldsDontExistDontReturn() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .select().fields("key7").end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertEquals(0, MockSinkBolt.getEvents().size());
}
 
Example 6
Source Project: flowmix   File: FilterBoltIT.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFilter_nothingPasses() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
          .filter().filter(new CriteriaFilter(criteriaFromNode(new QueryBuilder().eq("key3", "val50").build()))).end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertEquals(0, MockSinkBolt.getEvents().size());
}
 
Example 7
Source Project: flowmix   File: FilterBoltIT.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFilter_eventsPass() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
          .filter().filter(new CriteriaFilter(criteriaFromNode(new QueryBuilder().eq("key1", "val1").build()))).end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertTrue(MockSinkBolt.getEvents().size() > 0);

}
 
Example 8
Source Project: jstorm   File: AvroUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * A helper method to extract avro serialization configurations from the topology configuration and register
 * specific kryo serializers as necessary.  A default serializer will be provided if none is specified in the
 * configuration.  "avro.serializer" should specify the complete class name of the serializer, e.g.
 * "org.apache.stgorm.hdfs.avro.GenericAvroSerializer"
 *
 * @param conf The topology configuration
 * @throws ClassNotFoundException If the specified serializer cannot be located.
 */
public static void addAvroKryoSerializations(Config conf) throws ClassNotFoundException {
    final Class serializerClass;
    if (conf.containsKey("avro.serializer")) {
        serializerClass = Class.forName((String)conf.get("avro.serializer"));
    }
    else {
        serializerClass = GenericAvroSerializer.class;
    }
    conf.registerSerialization(GenericData.Record.class, serializerClass);
    conf.setSkipMissingKryoRegistrations(false);
}
 
Example 9
Source Project: flowmix   File: AggregatorBoltIT.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void test_timeTrigger_timeEvict() {

    Flow flow = new FlowBuilder()
        .id("myflow")
        .flowDefs()
            .stream("stream1")
                .partition().fields("key3").end()
                .aggregate().aggregator(CountAggregator.class)
                  .config(OUTPUT_FIELD, "aCount")
                  .trigger(Policy.TIME, 5)
                  .evict(Policy.TIME, 10)
                  .clearOnTrigger()
                .end()
            .endStream()
        .endDefs()
    .createFlow();

    StormTopology topology = buildTopology(flow, 10);
    Config conf = new Config();
    conf.registerSerialization(Event.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);
    conf.setNumWorkers(1);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology(getTopologyName(), conf, topology);

    try {
      Thread.sleep(25000);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }

    cluster.shutdown();

    System.out.println(MockSinkBolt.getEvents().size());
    assertEquals(4, MockSinkBolt.getEvents().size());

    for(Event event : MockSinkBolt.getEvents()) {
      assertNotNull(event.get("aCount"));
      assertTrue(event.<Long>get("aCount").getValue() > 350);
      assertTrue(event.<Long>get("aCount").getValue() < 500);
    }
}
 
Example 10
Source Project: flowmix   File: AggregatorBoltIT.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void test_timeTrigger_countEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.TIME, 2)
          .evict(Policy.COUNT, 10)
          .clearOnTrigger()
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);


  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertEquals(4, MockSinkBolt.getEvents().size());

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() == 10);
  }
}
 
Example 11
Source Project: flowmix   File: AggregatorBoltIT.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void test_countTrigger_timeEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.COUNT, 600)       // trigger every 500 events received
          .evict(Policy.TIME, 1)            // only keep last 1 second in the window
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);

  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }


  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertEquals(1, MockSinkBolt.getEvents().size());

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() > 90);
    assertTrue(event.<Long>get("aCount").getValue() < 100);
  }
}
 
Example 12
Source Project: flowmix   File: AggregatorBoltIT.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void test_countTrigger_countEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.COUNT, 5)
          .evict(Policy.COUNT, 2)
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);

  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents().size());
  assertTrue(MockSinkBolt.getEvents().size() > 130);
  assertTrue(MockSinkBolt.getEvents().size() < 160);

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() == 2);
  }
}