org.apache.samza.serializers.NoOpSerde Java Examples

@Override
public void describe(StreamApplicationDescriptor appDesc) {
  Table<KV<Integer, Profile>> table = appDesc.getTable(
      new InMemoryTableDescriptor("t1", KVSerde.of(new IntegerSerde(), new ProfileJsonSerde())));
  DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test");
  GenericInputDescriptor<Profile> profileISD = ksd.getInputDescriptor("Profile", new NoOpSerde<>());
  appDesc.getInputStream(profileISD)
      .map(m -> new KV(m.getMemberId(), m))
      .sendTo(table);

  GenericInputDescriptor<PageView> pageViewISD = ksd.getInputDescriptor("PageView", new NoOpSerde<>());
  appDesc.getInputStream(pageViewISD)
      .map(pv -> {
        received.add(pv);
        return pv;
      })
      .partitionBy(PageView::getMemberId, v -> v, KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()), "p1")
      .join(table, new PageViewToProfileJoinFunction())
      .sink((m, collector, coordinator) -> joined.add(m));
}
 

@Test
public void testGetInputStreamWithValueSerde() {

  String streamId = "test-stream-1";
  Serde mockValueSerde = mock(Serde.class);
  GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
  GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mockValueSerde);
  StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
    appDesc.getInputStream(isd);
  }, getConfig());

  InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId);
  assertEquals(OpCode.INPUT, inputOpSpec.getOpCode());
  assertEquals(streamId, inputOpSpec.getStreamId());
  assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId));
  assertTrue(inputOpSpec.getKeySerde() instanceof NoOpSerde);
  assertEquals(mockValueSerde, inputOpSpec.getValueSerde());
}
 

@Test
public void testGetOutputStreamWithValueSerde() {
  String streamId = "test-stream-1";
  Serde mockValueSerde = mock(Serde.class);
  GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass");
  GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mockValueSerde);

  StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> {
    appDesc.getOutputStream(osd);
  }, getConfig());

  OutputStreamImpl<TestMessageEnvelope> outputStreamImpl = streamAppDesc.getOutputStreams().get(streamId);
  assertEquals(streamId, outputStreamImpl.getStreamId());
  assertEquals(osd, streamAppDesc.getOutputDescriptors().get(streamId));
  assertTrue(outputStreamImpl.getKeySerde() instanceof NoOpSerde);
  assertEquals(mockValueSerde, outputStreamImpl.getValueSerde());
}
 

@Test
public void testConfigGeneration() {
  String systemName = "kinesis";
  String streamName = "Seine";
  KinesisSystemDescriptor sd = new KinesisSystemDescriptor(systemName);
  Map<String, String> cliConfig = new HashMap<>();
  cliConfig.put("key1", "value1");
  KinesisInputDescriptor<KV<String, byte[]>> id = sd.getInputDescriptor(streamName, new NoOpSerde<byte[]>())
      .withRegion("Paris")
      .withAccessKey("accessKey")
      .withSecretKey("secretKey")
      .withKCLConfig(cliConfig);

  Map<String, String> generatedConfig = id.toConfig();
  Assert.assertEquals(5, generatedConfig.size());

  Assert.assertEquals(systemName, generatedConfig.get("streams.Seine.samza.system"));
  Assert.assertEquals("Paris",
      generatedConfig.get(String.format(KinesisConfig.CONFIG_STREAM_REGION, systemName, streamName)));
  Assert.assertEquals("accessKey",
      generatedConfig.get(String.format(KinesisConfig.CONFIG_STREAM_ACCESS_KEY, systemName, streamName)));
  Assert.assertEquals("secretKey",
      generatedConfig.get(String.format(KinesisConfig.CONFIG_STREAM_SECRET_KEY, systemName, streamName)));
  Assert.assertEquals("value1", generatedConfig.get(
      String.format(KinesisConfig.CONFIG_STREAM_KINESIS_CLIENT_LIB_CONFIG, systemName, streamName) + "key1"));
}
 

@Test
public void testGetIntermediateStreamWithValueSerde() {
  String streamId = "stream-1";
  StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig());

  Serde mockValueSerde = mock(Serde.class);
  IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl =
      streamAppDesc.getIntermediateStream(streamId, mockValueSerde, false);

  assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec());
  assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream());
  assertEquals(streamId, intermediateStreamImpl.getStreamId());
  assertTrue(intermediateStreamImpl.getOutputStream().getKeySerde() instanceof NoOpSerde);
  assertEquals(mockValueSerde, intermediateStreamImpl.getOutputStream().getValueSerde());
  assertTrue(((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getKeySerde() instanceof NoOpSerde);
  assertEquals(mockValueSerde, ((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getValueSerde());
}
 

/**
 * Job should fail because it times out too soon
 */
@Test(expected = SamzaException.class)
public void testSamzaJobTimeoutFailureForAsyncTask() {
  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("async-test");

  InMemoryInputDescriptor<Integer> imid = isd
      .getInputDescriptor("ints", new NoOpSerde<>());

  InMemoryOutputDescriptor imod = isd
      .getOutputDescriptor("ints-out", new NoOpSerde<>());

  TestRunner
      .of(MyAsyncStreamTask.class)
      .addInputStream(imid, Arrays.asList(1, 2, 3, 4))
      .addOutputStream(imod, 1)
      .run(Duration.ofMillis(1));
}
 

@Test
public void testWikipediaFeedTask() throws Exception {
  String[] wikipediaFeedSamples = new String[] { "{\"channel\":\"#en.wikipedia\",\"raw\":\"[[Fear Is the Key (song)]]  https://en.wikipedia.org/w/index.php?diff=865574761&oldid=861177329 * Sam Sailor * (+46) Redirecting to [[Fear of the Dark (Iron Maiden album)]] ([[User:Sam Sailor/Scripts/Sagittarius+|♐]])\",\"time\":1540408899419,\"source\":\"rc-pmtpa\"}" };

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("kafka");

  InMemoryInputDescriptor rawWikiEvents = isd
      .getInputDescriptor("wikipedia-raw", new NoOpSerde<>());

  InMemoryOutputDescriptor<WikipediaFeedEvent> outputStreamDesc = isd
      .getOutputDescriptor("wikipedia-edits", new NoOpSerde<>());

  TestRunner
      .of(new WikipediaParserTaskApplication())
      .addInputStream(rawWikiEvents, parseJSONToMap(wikipediaFeedSamples))
      .addOutputStream(outputStreamDesc, 1)
      .run(Duration.ofSeconds(2));

  Assert.assertEquals(1
      , TestRunner.consumeStream(outputStreamDesc, Duration.ofSeconds(1)).get(0).size());
}
 

@Test
public void testAsyncTaskWithMultiplePartitionMultithreaded() throws Exception {
  Map<Integer, List<KV>> inputPartitionData = new HashMap<>();
  Map<Integer, List<Integer>> expectedOutputPartitionData = new HashMap<>();
  genData(inputPartitionData, expectedOutputPartitionData);

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("async-test");

  InMemoryInputDescriptor<KV> imid = isd
      .getInputDescriptor("ints", new NoOpSerde<>());

  InMemoryOutputDescriptor imod = isd
      .getOutputDescriptor("ints-out", new NoOpSerde<>());

  TestRunner
      .of(MyAsyncStreamTask.class)
      .addInputStream(imid, inputPartitionData)
      .addOutputStream(imod, 5)
      .addConfig("task.max.concurrency", "4")
      .run(Duration.ofSeconds(2));

  StreamAssert.containsInAnyOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000));
}
 

@Test
public void testTumblingWindowExample() {
  List<PageView> pageViewEvents = TestUtils.genSamplePageViewData();

  InMemorySystemDescriptor inMemorySystem = new InMemorySystemDescriptor("kafka");

  InMemoryInputDescriptor<KV<String, PageView>> pageViewInputDescriptor =
      inMemorySystem.getInputDescriptor("pageview-tumbling-input", new NoOpSerde<KV<String, PageView>>());

  InMemoryOutputDescriptor<KV<String, UserPageViews>> userPageViewOutputDescriptor =
      inMemorySystem.getOutputDescriptor("pageview-tumbling-output", new NoOpSerde<KV<String, UserPageViews>>());

  TestRunner
      .of(new TumblingWindowExample())
      .addInputStream(pageViewInputDescriptor, pageViewEvents)
      .addOutputStream(userPageViewOutputDescriptor, 1)
      .run(Duration.ofMinutes(1));

  Assert.assertTrue(TestRunner.consumeStream(userPageViewOutputDescriptor, Duration.ofMillis(1000)).get(0).size() > 1);
}
 

@Test
public void testSessionWindowExample() {
  List<PageView> pageViewEvents = TestUtils.genSamplePageViewData();

  InMemorySystemDescriptor inMemorySystem = new InMemorySystemDescriptor("kafka");

  InMemoryInputDescriptor<KV<String, PageView>> pageViewInputDescriptor =
      inMemorySystem.getInputDescriptor("pageview-session-input", new NoOpSerde<KV<String, PageView>>());

  InMemoryOutputDescriptor<KV<String, UserPageViews>> userPageViewOutputDescriptor =
      inMemorySystem.getOutputDescriptor("pageview-session-output", new NoOpSerde<KV<String, UserPageViews>>());

  TestRunner
      .of(new SessionWindowExample())
      .addInputStream(pageViewInputDescriptor, pageViewEvents)
      .addOutputStream(userPageViewOutputDescriptor, 1)
      .run(Duration.ofMinutes(1));

  Assert.assertEquals(2, TestRunner.consumeStream(userPageViewOutputDescriptor, Duration.ofMillis(1000)).get(0).size());
}
 

/** The dummy stream created will only be used in Beam tests. */
private static InputDescriptor<OpMessage<String>, ?> createDummyStreamDescriptor(String id) {
  final GenericSystemDescriptor dummySystem =
      new GenericSystemDescriptor(id, InMemorySystemFactory.class.getName());
  final GenericInputDescriptor<OpMessage<String>> dummyInput =
      dummySystem.getInputDescriptor(id, new NoOpSerde<>());
  dummyInput.withOffsetDefault(SystemStreamMetadata.OffsetType.OLDEST);
  final Config config = new MapConfig(dummyInput.toConfig(), dummySystem.toConfig());
  final SystemFactory factory = new InMemorySystemFactory();
  final StreamSpec dummyStreamSpec = new StreamSpec(id, id, id, 1);
  factory.getAdmin(id, config).createStream(dummyStreamSpec);

  final SystemProducer producer = factory.getProducer(id, config, null);
  final SystemStream sysStream = new SystemStream(id, id);
  final Consumer<Object> sendFn =
      (msg) -> {
        producer.send(id, new OutgoingMessageEnvelope(sysStream, 0, null, msg));
      };
  final WindowedValue<String> windowedValue =
      WindowedValue.timestampedValueInGlobalWindow("dummy", new Instant());

  sendFn.accept(OpMessage.ofElement(windowedValue));
  sendFn.accept(new WatermarkMessage(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis()));
  sendFn.accept(new EndOfStreamMessage(null));
  return dummyInput;
}
 

@Override
public void translatePortable(
    PipelineNode.PTransformNode transform,
    QueryablePipeline pipeline,
    PortableTranslationContext ctx) {

  final String outputId = ctx.getOutputId(transform);
  final GenericSystemDescriptor systemDescriptor =
      new GenericSystemDescriptor(outputId, SamzaImpulseSystemFactory.class.getName());

  // The KvCoder is needed here for Samza not to crop the key.
  final Serde<KV<?, OpMessage<byte[]>>> kvSerde = KVSerde.of(new NoOpSerde(), new NoOpSerde<>());
  final GenericInputDescriptor<KV<?, OpMessage<byte[]>>> inputDescriptor =
      systemDescriptor.getInputDescriptor(outputId, kvSerde);

  ctx.registerInputMessageStream(outputId, inputDescriptor);
}
 

@Test
public void testFilterExample() {
  List<PageView> rawPageViewEvents = new ArrayList<>();
  rawPageViewEvents.add(new PageView("google.com", "user1", "india"));
  rawPageViewEvents.add(new PageView("facebook.com", "invalidUserId", "france"));
  rawPageViewEvents.add(new PageView("yahoo.com", "user2", "china"));

  InMemorySystemDescriptor inMemorySystem = new InMemorySystemDescriptor("kafka");

  InMemoryInputDescriptor<PageView> badPageViewEvents =
      inMemorySystem.getInputDescriptor("pageview-filter-input", new NoOpSerde<PageView>());

  InMemoryOutputDescriptor<PageView> goodPageViewEvents =
      inMemorySystem.getOutputDescriptor("pageview-filter-output", new NoOpSerde<PageView>());

  TestRunner
      .of(new FilterExample())
      .addInputStream(badPageViewEvents, rawPageViewEvents)
      .addOutputStream(goodPageViewEvents, 1)
      .run(Duration.ofMillis(1500));

  Assert.assertEquals(TestRunner.consumeStream(goodPageViewEvents, Duration.ofMillis(1000)).get(0).size(), 2);
}
 

@Test
public void testAsyncTaskWithSinglePartitionUsingStreamAssert() throws Exception {
  List<Integer> inputList = Arrays.asList(1, 2, 3, 4, 5);
  List<Integer> outputList = Arrays.asList(50, 10, 20, 30, 40);

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("async-test");

  InMemoryInputDescriptor<Integer> imid = isd
      .getInputDescriptor("ints", new NoOpSerde<Integer>());

  InMemoryOutputDescriptor imod = isd
      .getOutputDescriptor("ints-out", new NoOpSerde<>());

  TestRunner
      .of(MyAsyncStreamTask.class)
      .addInputStream(imid, inputList)
      .addOutputStream(imod, 1)
      .run(Duration.ofSeconds(2));

  StreamAssert.containsInAnyOrder(outputList, imod, Duration.ofMillis(1000));
}
 

@Test
public void testAsyncTaskWithSinglePartition() throws Exception {
  List<Integer> inputList = Arrays.asList(1, 2, 3, 4, 5);
  List<Integer> outputList = Arrays.asList(10, 20, 30, 40, 50);

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("async-test");

  InMemoryInputDescriptor<Integer> imid = isd
      .getInputDescriptor("ints", new NoOpSerde<Integer>());

  InMemoryOutputDescriptor imod = isd
      .getOutputDescriptor("ints-out", new NoOpSerde<>());

  TestRunner
      .of(MyAsyncStreamTask.class)
      .addInputStream(imid, inputList)
      .addOutputStream(imod, 1)
      .run(Duration.ofSeconds(2));

  Assert.assertThat(TestRunner.consumeStream(imod, Duration.ofMillis(1000)).get(0),
      IsIterableContainingInOrder.contains(outputList.toArray()));
}
 

@Test
public void testHighLevelApi() throws Exception {
  Random random = new Random();
  int count = 10;
  List<PageView> pageViews = new ArrayList<>();
  for (int memberId = 0; memberId < count; memberId++) {
    String pagekey = PAGEKEYS[random.nextInt(PAGEKEYS.length - 1)];
    PageView pv = new PageView(pagekey, memberId);
    pageViews.add(pv);
  }

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
  InMemoryInputDescriptor<PageView> imid = isd.getInputDescriptor("PageView", new NoOpSerde<PageView>());
  InMemoryOutputDescriptor<PageView> imod = isd.getOutputDescriptor("Output", new NoOpSerde<PageView>());

  TestRunner
      .of(new PageViewRepartitionApplication())
      .addInputStream(imid, pageViews)
      .addOutputStream(imod, 10)
      .run(Duration.ofMillis(1500));

  Assert.assertEquals(TestRunner.consumeStream(imod, Duration.ofMillis(1000)).get(random.nextInt(count)).size(), 1);
}
 

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
  Config config = appDescriptor.getConfig();
  KafkaSystemDescriptor kafkaSystemDescriptor = new KafkaSystemDescriptor(TEST_SYSTEM);
  KafkaOutputDescriptor<PageView>
      outputDescriptor = kafkaSystemDescriptor.getOutputDescriptor(NON_GUEST_PAGE_VIEW_STREAM, new NoOpSerde<>());
  OutputStream<PageView> nonGuestPageViewStream = appDescriptor.getOutputStream(outputDescriptor);

  Predicate<PageView> failProcess = (Predicate<PageView> & Serializable) (ignored) -> config.getBoolean(FAIL_PROCESS, false);
  Predicate<PageView> failDownstreamOperator = (Predicate<PageView> & Serializable) (ignored) -> config.getBoolean(FAIL_DOWNSTREAM_OPERATOR, false);
  Supplier<Long> processJitter = (Supplier<Long> & Serializable) () -> config.getLong(PROCESS_JITTER, 100);

  appDescriptor.getInputStream(kafkaSystemDescriptor.getInputDescriptor(PAGE_VIEW_STREAM, new NoOpSerde<PageView>()))
      .flatMapAsync(pageView -> filterGuestPageViews(pageView, failProcess, processJitter))
      .filter(pageView -> filterLoginPageViews(pageView, failDownstreamOperator))
      .sendTo(nonGuestPageViewStream);
}
 

private List<PageView> runTest(List<PageView> pageViews, Map<String, String> configs) {
  configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, PAGE_VIEW_STREAM), TEST_SYSTEM);

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor(TEST_SYSTEM);
  InMemoryInputDescriptor<PageView> pageViewStreamDesc = isd
      .getInputDescriptor(PAGE_VIEW_STREAM, new NoOpSerde<>());


  InMemoryOutputDescriptor<PageView> outputStreamDesc = isd
      .getOutputDescriptor(NON_GUEST_PAGE_VIEW_STREAM, new NoOpSerde<>());

  TestRunner
      .of(new AsyncFlatMapExample())
      .addInputStream(pageViewStreamDesc, pageViews)
      .addOutputStream(outputStreamDesc, 1)
      .addConfig(new MapConfig(configs))
      .run(Duration.ofMillis(50000));

  Map<Integer, List<PageView>> result = TestRunner.consumeStream(outputStreamDesc, Duration.ofMillis(1000));
  List<PageView> results = result.values().stream()
      .flatMap(List::stream)
      .collect(Collectors.toList());

  return results;
}
 

/**
 * Null page key is passed in input data which should fail filter logic
 */
@Test(expected = SamzaException.class)
public void testSamzaJobFailureForStreamApplication() {
  int count = 10;
  List<TestData.PageView> pageviews = new ArrayList<>();
  for (int memberId = 0; memberId < count; memberId++) {
    pageviews.add(new TestData.PageView(null, memberId));
  }

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");
  InMemoryInputDescriptor<PageView> imid = isd.getInputDescriptor("PageView", new NoOpSerde<PageView>());
  InMemoryOutputDescriptor<PageView> imod = isd.getOutputDescriptor("Output", new NoOpSerde<PageView>());

  TestRunner.of(new PageViewFilterApplication())
      .addInputStream(imid, pageviews)
      .addOutputStream(imod, 10)
      .run(Duration.ofMillis(1000));
}
 

@Test
public void testStatefulTaskWithLocalTable() {
  List<PageView> pageViews = Arrays.asList(TestTableData.generatePageViews(10));
  List<Profile> profiles = Arrays.asList(TestTableData.generateProfiles(10));

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");

  InMemoryInputDescriptor<TestTableData.PageView> pageViewStreamDesc = isd
      .getInputDescriptor("PageView", new NoOpSerde<TestTableData.PageView>());

  InMemoryInputDescriptor<TestTableData.Profile> profileStreamDesc = isd
      .getInputDescriptor("Profile", new NoOpSerde<TestTableData.Profile>())
      .shouldBootstrap();

  InMemoryOutputDescriptor<TestTableData.EnrichedPageView> outputStreamDesc = isd
      .getOutputDescriptor("EnrichedPageView", new NoOpSerde<>());

  TestRunner
      .of(new JoinTaskApplication())
      .addInputStream(pageViewStreamDesc, pageViews)
      .addInputStream(profileStreamDesc, profiles)
      .addOutputStream(outputStreamDesc, 1)
      .run(Duration.ofSeconds(2));

  Assert.assertEquals(10, TestRunner.consumeStream(outputStreamDesc, Duration.ofSeconds(1)).get(0).size());
}
 

@Test
public void testSyncTaskWithSinglePartition() throws Exception {
  List<Integer> inputList = Arrays.asList(1, 2, 3, 4, 5);
  List<Integer> outputList = Arrays.asList(10, 20, 30, 40, 50);

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");

  InMemoryInputDescriptor<Integer> imid = isd
      .getInputDescriptor("input", new NoOpSerde<Integer>());

  InMemoryOutputDescriptor<Integer> imod = isd
      .getOutputDescriptor("output", new NoOpSerde<Integer>());

  TestRunner
      .of(MyStreamTestTask.class)
      .addInputStream(imid, inputList)
      .addOutputStream(imod, 1)
      .addExternalContext(new TestContext(10))
      .run(Duration.ofSeconds(1));

  Assert.assertThat(TestRunner.consumeStream(imod, Duration.ofMillis(1000)).get(0),
      IsIterableContainingInOrder.contains(outputList.toArray()));

}
 

/**
 * Samza job logic expects integers, but doubles are passed here which results in failure
 */
@Test(expected = SamzaException.class)
public void testSamzaJobFailureForSyncTask() {
  List<Double> inputList = Arrays.asList(1.2, 2.3, 3.33, 4.5);

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");

  InMemoryInputDescriptor<Double> imid = isd
      .getInputDescriptor("doubles", new NoOpSerde<Double>());

  InMemoryOutputDescriptor imod = isd
      .getOutputDescriptor("output", new NoOpSerde<>());

  TestRunner
      .of(MyStreamTestTask.class)
      .addInputStream(imid, inputList)
      .addOutputStream(imod, 1)
      .addExternalContext(new TestContext(10))
      .run(Duration.ofSeconds(1));
}
 

@Test
public void testSyncTaskWithSinglePartitionMultithreaded() throws Exception {
  List<Integer> inputList = Arrays.asList(1, 2, 3, 4, 5);
  List<Integer> outputList = Arrays.asList(10, 20, 30, 40, 50);

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");

  InMemoryInputDescriptor<Integer> imid = isd
      .getInputDescriptor("input", new NoOpSerde<Integer>());

  InMemoryOutputDescriptor<Integer> imod = isd
      .getOutputDescriptor("output", new NoOpSerde<Integer>());

  TestRunner
      .of(MyStreamTestTask.class)
      .addInputStream(imid, inputList)
      .addOutputStream(imod, 1)
      .addConfig("job.container.thread.pool.size", "4")
      .addExternalContext(new TestContext(10))
      .run(Duration.ofSeconds(1));

  StreamAssert.containsInOrder(outputList, imod, Duration.ofMillis(1000));
}
 

@Test
public void testSyncTaskWithMultiplePartition() throws Exception {
  Map<Integer, List<KV>> inputPartitionData = new HashMap<>();
  Map<Integer, List<Integer>> expectedOutputPartitionData = new HashMap<>();
  genData(inputPartitionData, expectedOutputPartitionData);

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");

  InMemoryInputDescriptor<KV> imid = isd
      .getInputDescriptor("input", new NoOpSerde<KV>());

  InMemoryOutputDescriptor<Integer> imod = isd
      .getOutputDescriptor("output", new NoOpSerde<Integer>());

  TestRunner
      .of(MyStreamTestTask.class)
      .addInputStream(imid, inputPartitionData)
      .addOutputStream(imod, 5)
      .addExternalContext(new TestContext(10))
      .run(Duration.ofSeconds(2));

  StreamAssert.containsInOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000));
}
 

void syncTaskWithMultiplePartitionMultithreadedHelper(Map<Integer, List<KV>> inputPartitionData,
    Map<Integer, List<Integer>> expectedOutputPartitionData) throws Exception {

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");

  InMemoryInputDescriptor<KV> imid = isd
      .getInputDescriptor("input", new NoOpSerde<KV>());

  InMemoryOutputDescriptor<Integer> imod = isd
      .getOutputDescriptor("output", new NoOpSerde<Integer>());

  TestRunner
      .of(MyStreamTestTask.class)
      .addInputStream(imid, inputPartitionData)
      .addOutputStream(imod, 5)
      .addConfig("job.container.thread.pool.size", "4")
      .addExternalContext(new TestContext(10))
      .run(Duration.ofSeconds(2));

  StreamAssert.containsInOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000));
}
 

@Test
public void testAsyncTaskWithMultiplePartition() throws Exception {
  Map<Integer, List<KV>> inputPartitionData = new HashMap<>();
  Map<Integer, List<Integer>> expectedOutputPartitionData = new HashMap<>();
  genData(inputPartitionData, expectedOutputPartitionData);

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("async-test");

  InMemoryInputDescriptor<KV> imid = isd
      .getInputDescriptor("ints", new NoOpSerde<KV>());
  InMemoryOutputDescriptor imod = isd
      .getOutputDescriptor("ints-out", new NoOpSerde<>());

  TestRunner
      .of(MyAsyncStreamTask.class)
      .addInputStream(imid, inputPartitionData)
      .addOutputStream(imod, 5)
      .run(Duration.ofSeconds(2));

  StreamAssert.containsInOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000));
}
 

@Override
public void describe(TaskApplicationDescriptor appDescriptor) {
  KafkaSystemDescriptor ksd = new KafkaSystemDescriptor("test");
  KafkaInputDescriptor<Profile> profileISD = ksd.getInputDescriptor("Profile", new JsonSerdeV2<>());
  KafkaInputDescriptor<PageView> pageViewISD = ksd.getInputDescriptor("PageView", new JsonSerdeV2<>());
  KafkaOutputDescriptor<EnrichedPageView> enrichedPageViewOSD =
      ksd.getOutputDescriptor("EnrichedPageView", new NoOpSerde<>());
  appDescriptor
      .withInputStream(profileISD)
      .withInputStream(pageViewISD)
      .withOutputStream(enrichedPageViewOSD)
      .withTable(new InMemoryTableDescriptor("profile-view-store",
          KVSerde.of(new IntegerSerde(), new TestTableData.ProfileJsonSerde())))
      .withTaskFactory((StreamTaskFactory) () -> new StatefulStreamTask());
}
 

@Test
public void testStatefulJoinWithLocalTable() {
  Random random = new Random();
  List<KV<String, TestTableData.PageView>> pageViews = Arrays.asList(TestTableData.generatePageViews(10))
      .stream()
      .map(x -> KV.of(PAGEKEYS[random.nextInt(PAGEKEYS.length)], x))
      .collect(Collectors.toList());
  List<KV<String, TestTableData.Profile>> profiles = Arrays.asList(TestTableData.generateProfiles(10))
      .stream()
      .map(x -> KV.of(PAGEKEYS[random.nextInt(PAGEKEYS.length)], x))
      .collect(Collectors.toList());

  InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test");

  InMemoryInputDescriptor<KV<String, TestTableData.PageView>> pageViewStreamDesc = isd
      .getInputDescriptor("PageView", new NoOpSerde<KV<String, TestTableData.PageView>>());

  InMemoryInputDescriptor<KV<String, TestTableData.Profile>> profileStreamDesc = isd
      .getInputDescriptor("Profile", new NoOpSerde<KV<String, TestTableData.Profile>>())
      .shouldBootstrap();

  InMemoryOutputDescriptor<TestTableData.EnrichedPageView> outputStreamDesc = isd
      .getOutputDescriptor("EnrichedPageView", new NoOpSerde<>());

  InMemoryOutputDescriptor<String> joinKeysDescriptor = isd
      .getOutputDescriptor("JoinPageKeys", new NoOpSerde<>());

  TestRunner
      .of(new PageViewProfileViewJoinApplication())
      .addInputStream(pageViewStreamDesc, pageViews)
      .addInputStream(profileStreamDesc, profiles)
      .addOutputStream(outputStreamDesc, 1)
      .addOutputStream(joinKeysDescriptor, 1)
      .run(Duration.ofSeconds(2));


  Assert.assertEquals(10, TestRunner.consumeStream(outputStreamDesc, Duration.ofSeconds(1)).get(0).size());
  Assert.assertEquals(10, TestRunner.consumeStream(joinKeysDescriptor, Duration.ofSeconds(1)).get(0).size());
}
 

@Override
public void describe(StreamApplicationDescriptor appDescriptor) {
  Table<KV<Integer, TestTableData.Profile>> table = appDescriptor.getTable(getTableDescriptor());
  KafkaSystemDescriptor sd =
      new KafkaSystemDescriptor("test");
  appDescriptor.getInputStream(sd.getInputDescriptor(PAGEVIEW_STREAM, new NoOpSerde<TestTableData.PageView>()))
      .partitionBy(TestTableData.PageView::getMemberId, v -> v, KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()), "partition-page-view")
      .join(table, new PageViewToProfileJoinFunction())
      .sendTo(appDescriptor.getOutputStream(sd.getOutputDescriptor(ENRICHED_PAGEVIEW_STREAM, new NoOpSerde<>())));
}
 

@Override
public void describe(TaskApplicationDescriptor appDescriptor) {
  DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test");
  GenericInputDescriptor<TestTableData.PageView> pageViewISD = ksd.getInputDescriptor("PageView", new NoOpSerde<>());
  appDescriptor
      .withInputStream(pageViewISD)
      .withTaskFactory((StreamTaskFactory) () -> new MyStreamTask());
}