Java Code Examples for backtype.storm.Config#setSkipMissingKryoRegistrations()

The following examples show how to use backtype.storm.Config#setSkipMissingKryoRegistrations() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ExampleRunner.java From flowmix with Apache License 2.0

6 votes

public void run() {

    StormTopology topology = new FlowmixBuilder()
        .setFlowLoader(new SimpleFlowLoaderSpout(provider.getFlows(), 60000))
        .setEventsLoader(new MockEventGeneratorSpout(getMockEvents(), 10))
        .setOutputBolt(new PrinterBolt())
        .setParallelismHint(6)
      .create()
    .createTopology();

    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.setMaxSpoutPending(5000);
    conf.setDebug(false);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("example-topology", conf, topology);
  }

Example 2

Source File: SwitchBoltIT.java From flowmix with Apache License 2.0

6 votes

@Test
public void test_timeDiffActivated_countEviction() throws InterruptedException {
    Flow flow = new FlowBuilder()
            .id("flow")
            .flowDefs()
            .stream("stream1")
            .stopGate().open(Policy.TIME_DELTA_LT, 1000).close(Policy.TIME, 5).evict(Policy.COUNT, 5).end()
            .endStream()   // send ALL results to stream2 and not to standard output
            .endDefs()
            .createFlow();

    StormTopology topology = buildTopology(flow, 50);
    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, topology);

    Thread.sleep(5000);

    assertEquals(5, MockSinkBolt.getEvents().size());
}

Example 3

Source File: SwitchBoltIT.java From flowmix with Apache License 2.0

6 votes

@Test
public void test_timeDiffActivated_timeEviction() throws InterruptedException {
    Flow flow = new FlowBuilder()
            .id("flow")
            .flowDefs()
            .stream("stream1")
            .stopGate().open(Policy.TIME_DELTA_LT, 5).close(Policy.TIME, 1).evict(Policy.TIME, 1).end()
            .endStream()   // send ALL results to stream2 and not to standard output
            .endDefs()
            .createFlow();

    StormTopology topology = buildTopology(flow, 50);
    Config conf = new Config();
    conf.setNumWorkers(20);
    conf.registerSerialization(BaseEvent.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("test", conf, topology);

    Thread.sleep(7000);

    int size = MockSinkBolt.getEvents().size();
    System.out.println("SIZE: " + size);
    assertTrue(size >= 50 && size <= 65);
}

Example 4

Source File: SelectorBoltIT.java From flowmix with Apache License 2.0

5 votes

@Test
public void testSelection_basic() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
        .select().fields("key1", "key2").end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertTrue(MockSinkBolt.getEvents().size() > 0);

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("key1"));
    assertNotNull(event.get("key2"));
    assertNull(event.get("key3"));
    assertNull(event.get("key4"));
    assertNull(event.get("key5"));
  }
}

Example 5

Source File: SelectorBoltIT.java From flowmix with Apache License 2.0

5 votes

@Test
public void testSelection_fieldsDontExistDontReturn() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .select().fields("key7").end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertEquals(0, MockSinkBolt.getEvents().size());
}

Example 6

Source File: FilterBoltIT.java From flowmix with Apache License 2.0

5 votes

@Test
public void testFilter_nothingPasses() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
          .filter().filter(new CriteriaFilter(criteriaFromNode(new QueryBuilder().eq("key3", "val50").build()))).end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertEquals(0, MockSinkBolt.getEvents().size());
}

Example 7

Source File: FilterBoltIT.java From flowmix with Apache License 2.0

5 votes

@Test
public void testFilter_eventsPass() {

  Flow flow = new FlowBuilder()
    .id("myflow")
    .flowDefs()
      .stream("stream1")
          .filter().filter(new CriteriaFilter(criteriaFromNode(new QueryBuilder().eq("key1", "val1").build()))).end()
      .endStream()
    .endDefs()
  .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(20);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("test", conf, topology);

  try {
    Thread.sleep(3000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  assertTrue(MockSinkBolt.getEvents().size() > 0);

}

Example 8

Source File: AvroUtils.java From jstorm with Apache License 2.0

5 votes

/**
 * A helper method to extract avro serialization configurations from the topology configuration and register
 * specific kryo serializers as necessary.  A default serializer will be provided if none is specified in the
 * configuration.  "avro.serializer" should specify the complete class name of the serializer, e.g.
 * "org.apache.stgorm.hdfs.avro.GenericAvroSerializer"
 *
 * @param conf The topology configuration
 * @throws ClassNotFoundException If the specified serializer cannot be located.
 */
public static void addAvroKryoSerializations(Config conf) throws ClassNotFoundException {
    final Class serializerClass;
    if (conf.containsKey("avro.serializer")) {
        serializerClass = Class.forName((String)conf.get("avro.serializer"));
    }
    else {
        serializerClass = GenericAvroSerializer.class;
    }
    conf.registerSerialization(GenericData.Record.class, serializerClass);
    conf.setSkipMissingKryoRegistrations(false);
}

Example 9

Source File: AggregatorBoltIT.java From flowmix with Apache License 2.0

4 votes

@Test
public void test_timeTrigger_timeEvict() {

    Flow flow = new FlowBuilder()
        .id("myflow")
        .flowDefs()
            .stream("stream1")
                .partition().fields("key3").end()
                .aggregate().aggregator(CountAggregator.class)
                  .config(OUTPUT_FIELD, "aCount")
                  .trigger(Policy.TIME, 5)
                  .evict(Policy.TIME, 10)
                  .clearOnTrigger()
                .end()
            .endStream()
        .endDefs()
    .createFlow();

    StormTopology topology = buildTopology(flow, 10);
    Config conf = new Config();
    conf.registerSerialization(Event.class, EventSerializer.class);
    conf.setSkipMissingKryoRegistrations(false);
    conf.setNumWorkers(1);

    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology(getTopologyName(), conf, topology);

    try {
      Thread.sleep(25000);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }

    cluster.shutdown();

    System.out.println(MockSinkBolt.getEvents().size());
    assertEquals(4, MockSinkBolt.getEvents().size());

    for(Event event : MockSinkBolt.getEvents()) {
      assertNotNull(event.get("aCount"));
      assertTrue(event.<Long>get("aCount").getValue() > 350);
      assertTrue(event.<Long>get("aCount").getValue() < 500);
    }
}

Example 10

Source File: AggregatorBoltIT.java From flowmix with Apache License 2.0

4 votes

@Test
public void test_timeTrigger_countEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.TIME, 2)
          .evict(Policy.COUNT, 10)
          .clearOnTrigger()
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);


  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertEquals(4, MockSinkBolt.getEvents().size());

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() == 10);
  }
}

Example 11

Source File: AggregatorBoltIT.java From flowmix with Apache License 2.0

4 votes

@Test
public void test_countTrigger_timeEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.COUNT, 600)       // trigger every 500 events received
          .evict(Policy.TIME, 1)            // only keep last 1 second in the window
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);

  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }


  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents());
  assertEquals(1, MockSinkBolt.getEvents().size());

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() > 90);
    assertTrue(event.<Long>get("aCount").getValue() < 100);
  }
}

Example 12

Source File: AggregatorBoltIT.java From flowmix with Apache License 2.0

4 votes

@Test
public void test_countTrigger_countEvict() {

  Flow flow = new FlowBuilder()
          .id("myflow")
          .flowDefs()
          .stream("stream1")
          .partition().fields("key3").end()
          .aggregate().aggregator(CountAggregator.class)
          .config(OUTPUT_FIELD, "aCount")
          .trigger(Policy.COUNT, 5)
          .evict(Policy.COUNT, 2)
          .end()
          .endStream()
          .endDefs()
          .createFlow();

  StormTopology topology = buildTopology(flow, 10);
  Config conf = new Config();
  conf.registerSerialization(Event.class, EventSerializer.class);
  conf.setSkipMissingKryoRegistrations(false);
  conf.setNumWorkers(1);

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology(getTopologyName(), conf, topology);

  try {
    Thread.sleep(10000);
  } catch (InterruptedException e) {
    e.printStackTrace();
  }

  cluster.shutdown();
  System.out.println(MockSinkBolt.getEvents().size());
  assertTrue(MockSinkBolt.getEvents().size() > 130);
  assertTrue(MockSinkBolt.getEvents().size() < 160);

  for(Event event : MockSinkBolt.getEvents()) {
    assertNotNull(event.get("aCount"));
    assertTrue(event.<Long>get("aCount").getValue() == 2);
  }
}