org.apache.samza.serializers.IntegerSerde Java Examples

The following examples show how to use org.apache.samza.serializers.IntegerSerde. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: TestTransformingInputDescriptor.java From samza with Apache License 2.0

6 votes

@Test
public void testAPIUsage() {
  // does not assert anything, but acts as a compile-time check on expected descriptor type parameters
  // and validates that the method calls can be chained.
  ExampleTransformingSystemDescriptor imeTransformingSystem =
      new ExampleTransformingSystemDescriptor("imeTransformingSystem")
          .withSystemConfigs(Collections.emptyMap());
  ExampleTransformingInputDescriptor<Long> input1 = imeTransformingSystem.getInputDescriptor("input1", new IntegerSerde());
  ExampleTransformingOutputDescriptor<Integer> output1 = imeTransformingSystem.getOutputDescriptor("output1", new IntegerSerde());

  input1
      .shouldBootstrap()
      .withOffsetDefault(SystemStreamMetadata.OffsetType.NEWEST)
      .withPriority(1)
      .shouldResetOffset()
      .withStreamConfigs(Collections.emptyMap());

  output1
      .withStreamConfigs(Collections.emptyMap());
}

Example #2

Source File: TestSchedulerFunction.java From samza with Apache License 2.0

6 votes

@Test
public void testImmediateTimer() {
  final InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
  final InMemoryInputDescriptor<Integer> imid = isd.getInputDescriptor("test-input", new IntegerSerde());

  StreamApplication app = new StreamApplication() {
    @Override
    public void describe(StreamApplicationDescriptor appDescriptor) {

      appDescriptor.getInputStream(imid)
          .map(new TestFunction());
    }
  };

  TestRunner
      .of(app)
      .addInputStream(imid, Arrays.asList(1, 2, 3, 4, 5))
      .run(Duration.ofSeconds(1));

  assertTrue(timerFired.get());
}

Example #3

Source File: TestLocalTableEndToEnd.java From samza with Apache License 2.0

6 votes

@Override
public void describe(StreamApplicationDescriptor appDesc) {
  Table<KV<Integer, Profile>> table = appDesc.getTable(
      new InMemoryTableDescriptor("t1", KVSerde.of(new IntegerSerde(), new ProfileJsonSerde())));
  DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test");
  GenericInputDescriptor<Profile> profileISD = ksd.getInputDescriptor("Profile", new NoOpSerde<>());
  appDesc.getInputStream(profileISD)
      .map(m -> new KV(m.getMemberId(), m))
      .sendTo(table);

  GenericInputDescriptor<PageView> pageViewISD = ksd.getInputDescriptor("PageView", new NoOpSerde<>());
  appDesc.getInputStream(pageViewISD)
      .map(pv -> {
        received.add(pv);
        return pv;
      })
      .partitionBy(PageView::getMemberId, v -> v, KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()), "p1")
      .join(table, new PageViewToProfileJoinFunction())
      .sink((m, collector, coordinator) -> joined.add(m));
}

Example #4

Source File: RepartitionWindowApp.java From samza with Apache License 2.0

6 votes

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
  KVSerde<String, PageView> inputSerde = KVSerde.of(new StringSerde("UTF-8"), new JsonSerdeV2<>(PageView.class));
  KVSerde<String, String> outputSerde = KVSerde.of(new StringSerde(), new StringSerde());
  KafkaSystemDescriptor ksd = new KafkaSystemDescriptor(SYSTEM);
  KafkaInputDescriptor<KV<String, PageView>> id = ksd.getInputDescriptor(INPUT_TOPIC, inputSerde);
  KafkaOutputDescriptor<KV<String, String>> od = ksd.getOutputDescriptor(OUTPUT_TOPIC, outputSerde);

  appDescriptor.getInputStream(id)
      .map(KV::getValue)
      .partitionBy(PageView::getUserId, m -> m, inputSerde, "p1")
      .window(Windows.keyedSessionWindow(m -> m.getKey(), Duration.ofSeconds(3), () -> 0, (m, c) -> c + 1, new StringSerde("UTF-8"), new IntegerSerde()), "w1")
      .map(wp -> KV.of(wp.getKey().getKey().toString(), String.valueOf(wp.getMessage())))
      .sendTo(appDescriptor.getOutputStream(od));

}

Example #5

Source File: TumblingWindowApp.java From samza with Apache License 2.0

6 votes

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
  JsonSerdeV2<PageView> inputSerde = new JsonSerdeV2<>(PageView.class);
  KVSerde<String, Integer> outputSerde = KVSerde.of(new StringSerde(), new IntegerSerde());
  KafkaSystemDescriptor ksd = new KafkaSystemDescriptor(SYSTEM);
  KafkaInputDescriptor<PageView> id = ksd.getInputDescriptor(INPUT_TOPIC, inputSerde);
  KafkaOutputDescriptor<KV<String, Integer>> od = ksd.getOutputDescriptor(OUTPUT_TOPIC, outputSerde);

  MessageStream<PageView> pageViews = appDescriptor.getInputStream(id);
  OutputStream<KV<String, Integer>> outputStream = appDescriptor.getOutputStream(od);

  pageViews
      .filter(m -> !FILTER_KEY.equals(m.getUserId()))
      .window(Windows.keyedTumblingWindow(PageView::getUserId, Duration.ofSeconds(3),
          new StringSerde(), new JsonSerdeV2<>(PageView.class)), "tumblingWindow")
      .map(m -> KV.of(m.getKey().getKey(), m.getMessage().size()))
      .sendTo(outputStream);

}

Example #6

Source File: SessionWindowApp.java From samza with Apache License 2.0

6 votes

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
  JsonSerdeV2<PageView> inputSerde = new JsonSerdeV2<>(PageView.class);
  KVSerde<String, Integer> outputSerde = KVSerde.of(new StringSerde(), new IntegerSerde());
  KafkaSystemDescriptor ksd = new KafkaSystemDescriptor(SYSTEM);
  KafkaInputDescriptor<PageView> id = ksd.getInputDescriptor(INPUT_TOPIC, inputSerde);
  KafkaOutputDescriptor<KV<String, Integer>> od = ksd.getOutputDescriptor(OUTPUT_TOPIC, outputSerde);

  MessageStream<PageView> pageViews = appDescriptor.getInputStream(id);
  OutputStream<KV<String, Integer>> outputStream = appDescriptor.getOutputStream(od);

  pageViews
      .filter(m -> !FILTER_KEY.equals(m.getUserId()))
      .window(Windows.keyedSessionWindow(PageView::getUserId, Duration.ofSeconds(3),
          new StringSerde(), new JsonSerdeV2<>(PageView.class)), "sessionWindow")
      .map(m -> KV.of(m.getKey().getKey(), m.getMessage().size()))
      .sendTo(outputStream);
}

Example #7

Source File: WindowExample.java From samza with Apache License 2.0

6 votes

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
  KafkaSystemDescriptor trackingSystem = new KafkaSystemDescriptor("tracking");

  KafkaInputDescriptor<PageViewEvent> inputStreamDescriptor =
      trackingSystem.getInputDescriptor("pageViewEvent", new JsonSerdeV2<>(PageViewEvent.class));
  KafkaOutputDescriptor<Integer> outputStreamDescriptor =
      trackingSystem.getOutputDescriptor("pageViewEventPerMember", new IntegerSerde());

  SupplierFunction<Integer> initialValue = () -> 0;
  FoldLeftFunction<PageViewEvent, Integer> counter = (m, c) -> c == null ? 1 : c + 1;
  MessageStream<PageViewEvent> inputStream = appDescriptor.getInputStream(inputStreamDescriptor);
  OutputStream<Integer> outputStream = appDescriptor.getOutputStream(outputStreamDescriptor);

  // create a tumbling window that outputs the number of message collected every 10 minutes.
  // also emit early results if either the number of messages collected reaches 30000, or if no new messages arrive
  // for 1 minute.
  inputStream
      .window(Windows.tumblingWindow(Duration.ofMinutes(10), initialValue, counter, new IntegerSerde())
          .setLateTrigger(Triggers.any(Triggers.count(30000),
              Triggers.timeSinceLastMessage(Duration.ofMinutes(1)))), "window")
      .map(WindowPane::getMessage)
      .sendTo(outputStream);
}

Example #8

Source File: TestTableConfigGenerator.java From samza with Apache License 2.0

6 votes

@Test
public void testWithSerdes() {
  List<TableDescriptor> descriptors = Arrays.asList(
      new MockLocalTableDescriptor("t1", KVSerde.of(new StringSerde(), new IntegerSerde())),
      new MockLocalTableDescriptor("t2", KVSerde.of(new StringSerde(), new IntegerSerde()))
  );
  Config jobConfig = new MapConfig(TableConfigGenerator.generateSerdeConfig(descriptors));
  JavaTableConfig javaTableConfig = new JavaTableConfig(jobConfig);
  assertNotNull(javaTableConfig.getKeySerde("t1"));
  assertNotNull(javaTableConfig.getMsgSerde("t1"));
  assertNotNull(javaTableConfig.getKeySerde("t2"));
  assertNotNull(javaTableConfig.getMsgSerde("t2"));

  MapConfig tableConfig = new MapConfig(TableConfigGenerator.generate(jobConfig, descriptors));
  javaTableConfig = new JavaTableConfig(tableConfig);
  assertNotNull(javaTableConfig.getTableProviderFactory("t1"));
  assertNotNull(javaTableConfig.getTableProviderFactory("t2"));
}

Example #9