org.apache.samza.serializers.NoOpSerde Java Examples
The following examples show how to use
org.apache.samza.serializers.NoOpSerde.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestLocalTableEndToEnd.java From samza with Apache License 2.0 | 6 votes |
@Override public void describe(StreamApplicationDescriptor appDesc) { Table<KV<Integer, Profile>> table = appDesc.getTable( new InMemoryTableDescriptor("t1", KVSerde.of(new IntegerSerde(), new ProfileJsonSerde()))); DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test"); GenericInputDescriptor<Profile> profileISD = ksd.getInputDescriptor("Profile", new NoOpSerde<>()); appDesc.getInputStream(profileISD) .map(m -> new KV(m.getMemberId(), m)) .sendTo(table); GenericInputDescriptor<PageView> pageViewISD = ksd.getInputDescriptor("PageView", new NoOpSerde<>()); appDesc.getInputStream(pageViewISD) .map(pv -> { received.add(pv); return pv; }) .partitionBy(PageView::getMemberId, v -> v, KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()), "p1") .join(table, new PageViewToProfileJoinFunction()) .sink((m, collector, coordinator) -> joined.add(m)); }
Example #2
Source File: TestStreamApplicationDescriptorImpl.java From samza with Apache License 2.0 | 6 votes |
@Test public void testGetInputStreamWithValueSerde() { String streamId = "test-stream-1"; Serde mockValueSerde = mock(Serde.class); GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericInputDescriptor isd = sd.getInputDescriptor(streamId, mockValueSerde); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getInputStream(isd); }, getConfig()); InputOperatorSpec inputOpSpec = streamAppDesc.getInputOperators().get(streamId); assertEquals(OpCode.INPUT, inputOpSpec.getOpCode()); assertEquals(streamId, inputOpSpec.getStreamId()); assertEquals(isd, streamAppDesc.getInputDescriptors().get(streamId)); assertTrue(inputOpSpec.getKeySerde() instanceof NoOpSerde); assertEquals(mockValueSerde, inputOpSpec.getValueSerde()); }
Example #3
Source File: TestStreamApplicationDescriptorImpl.java From samza with Apache License 2.0 | 6 votes |
@Test public void testGetOutputStreamWithValueSerde() { String streamId = "test-stream-1"; Serde mockValueSerde = mock(Serde.class); GenericSystemDescriptor sd = new GenericSystemDescriptor("mockSystem", "mockSystemFactoryClass"); GenericOutputDescriptor osd = sd.getOutputDescriptor(streamId, mockValueSerde); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { appDesc.getOutputStream(osd); }, getConfig()); OutputStreamImpl<TestMessageEnvelope> outputStreamImpl = streamAppDesc.getOutputStreams().get(streamId); assertEquals(streamId, outputStreamImpl.getStreamId()); assertEquals(osd, streamAppDesc.getOutputDescriptors().get(streamId)); assertTrue(outputStreamImpl.getKeySerde() instanceof NoOpSerde); assertEquals(mockValueSerde, outputStreamImpl.getValueSerde()); }
Example #4
Source File: TestKinesisInputDescriptor.java From samza with Apache License 2.0 | 6 votes |
@Test public void testConfigGeneration() { String systemName = "kinesis"; String streamName = "Seine"; KinesisSystemDescriptor sd = new KinesisSystemDescriptor(systemName); Map<String, String> cliConfig = new HashMap<>(); cliConfig.put("key1", "value1"); KinesisInputDescriptor<KV<String, byte[]>> id = sd.getInputDescriptor(streamName, new NoOpSerde<byte[]>()) .withRegion("Paris") .withAccessKey("accessKey") .withSecretKey("secretKey") .withKCLConfig(cliConfig); Map<String, String> generatedConfig = id.toConfig(); Assert.assertEquals(5, generatedConfig.size()); Assert.assertEquals(systemName, generatedConfig.get("streams.Seine.samza.system")); Assert.assertEquals("Paris", generatedConfig.get(String.format(KinesisConfig.CONFIG_STREAM_REGION, systemName, streamName))); Assert.assertEquals("accessKey", generatedConfig.get(String.format(KinesisConfig.CONFIG_STREAM_ACCESS_KEY, systemName, streamName))); Assert.assertEquals("secretKey", generatedConfig.get(String.format(KinesisConfig.CONFIG_STREAM_SECRET_KEY, systemName, streamName))); Assert.assertEquals("value1", generatedConfig.get( String.format(KinesisConfig.CONFIG_STREAM_KINESIS_CLIENT_LIB_CONFIG, systemName, streamName) + "key1")); }
Example #5
Source File: TestStreamApplicationDescriptorImpl.java From samza with Apache License 2.0 | 6 votes |
@Test public void testGetIntermediateStreamWithValueSerde() { String streamId = "stream-1"; StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { }, getConfig()); Serde mockValueSerde = mock(Serde.class); IntermediateMessageStreamImpl<TestMessageEnvelope> intermediateStreamImpl = streamAppDesc.getIntermediateStream(streamId, mockValueSerde, false); assertEquals(streamAppDesc.getInputOperators().get(streamId), intermediateStreamImpl.getOperatorSpec()); assertEquals(streamAppDesc.getOutputStreams().get(streamId), intermediateStreamImpl.getOutputStream()); assertEquals(streamId, intermediateStreamImpl.getStreamId()); assertTrue(intermediateStreamImpl.getOutputStream().getKeySerde() instanceof NoOpSerde); assertEquals(mockValueSerde, intermediateStreamImpl.getOutputStream().getValueSerde()); assertTrue(((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getKeySerde() instanceof NoOpSerde); assertEquals(mockValueSerde, ((InputOperatorSpec) (OperatorSpec) intermediateStreamImpl.getOperatorSpec()).getValueSerde()); }
Example #6
Source File: AsyncStreamTaskIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
/** * Job should fail because it times out too soon */ @Test(expected = SamzaException.class) public void testSamzaJobTimeoutFailureForAsyncTask() { InMemorySystemDescriptor isd = new InMemorySystemDescriptor("async-test"); InMemoryInputDescriptor<Integer> imid = isd .getInputDescriptor("ints", new NoOpSerde<>()); InMemoryOutputDescriptor imod = isd .getOutputDescriptor("ints-out", new NoOpSerde<>()); TestRunner .of(MyAsyncStreamTask.class) .addInputStream(imid, Arrays.asList(1, 2, 3, 4)) .addOutputStream(imod, 1) .run(Duration.ofMillis(1)); }
Example #7
Source File: TestWikipediaTask.java From samza-hello-samza with Apache License 2.0 | 6 votes |
@Test public void testWikipediaFeedTask() throws Exception { String[] wikipediaFeedSamples = new String[] { "{\"channel\":\"#en.wikipedia\",\"raw\":\"[[Fear Is the Key (song)]] https://en.wikipedia.org/w/index.php?diff=865574761&oldid=861177329 * Sam Sailor * (+46) Redirecting to [[Fear of the Dark (Iron Maiden album)]] ([[User:Sam Sailor/Scripts/Sagittarius+|♐]])\",\"time\":1540408899419,\"source\":\"rc-pmtpa\"}" }; InMemorySystemDescriptor isd = new InMemorySystemDescriptor("kafka"); InMemoryInputDescriptor rawWikiEvents = isd .getInputDescriptor("wikipedia-raw", new NoOpSerde<>()); InMemoryOutputDescriptor<WikipediaFeedEvent> outputStreamDesc = isd .getOutputDescriptor("wikipedia-edits", new NoOpSerde<>()); TestRunner .of(new WikipediaParserTaskApplication()) .addInputStream(rawWikiEvents, parseJSONToMap(wikipediaFeedSamples)) .addOutputStream(outputStreamDesc, 1) .run(Duration.ofSeconds(2)); Assert.assertEquals(1 , TestRunner.consumeStream(outputStreamDesc, Duration.ofSeconds(1)).get(0).size()); }
Example #8
Source File: AsyncStreamTaskIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
@Test public void testAsyncTaskWithMultiplePartitionMultithreaded() throws Exception { Map<Integer, List<KV>> inputPartitionData = new HashMap<>(); Map<Integer, List<Integer>> expectedOutputPartitionData = new HashMap<>(); genData(inputPartitionData, expectedOutputPartitionData); InMemorySystemDescriptor isd = new InMemorySystemDescriptor("async-test"); InMemoryInputDescriptor<KV> imid = isd .getInputDescriptor("ints", new NoOpSerde<>()); InMemoryOutputDescriptor imod = isd .getOutputDescriptor("ints-out", new NoOpSerde<>()); TestRunner .of(MyAsyncStreamTask.class) .addInputStream(imid, inputPartitionData) .addOutputStream(imod, 5) .addConfig("task.max.concurrency", "4") .run(Duration.ofSeconds(2)); StreamAssert.containsInAnyOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000)); }
Example #9
Source File: TestSamzaCookBookExamples.java From samza-hello-samza with Apache License 2.0 | 6 votes |
@Test public void testTumblingWindowExample() { List<PageView> pageViewEvents = TestUtils.genSamplePageViewData(); InMemorySystemDescriptor inMemorySystem = new InMemorySystemDescriptor("kafka"); InMemoryInputDescriptor<KV<String, PageView>> pageViewInputDescriptor = inMemorySystem.getInputDescriptor("pageview-tumbling-input", new NoOpSerde<KV<String, PageView>>()); InMemoryOutputDescriptor<KV<String, UserPageViews>> userPageViewOutputDescriptor = inMemorySystem.getOutputDescriptor("pageview-tumbling-output", new NoOpSerde<KV<String, UserPageViews>>()); TestRunner .of(new TumblingWindowExample()) .addInputStream(pageViewInputDescriptor, pageViewEvents) .addOutputStream(userPageViewOutputDescriptor, 1) .run(Duration.ofMinutes(1)); Assert.assertTrue(TestRunner.consumeStream(userPageViewOutputDescriptor, Duration.ofMillis(1000)).get(0).size() > 1); }
Example #10
Source File: TestSamzaCookBookExamples.java From samza-hello-samza with Apache License 2.0 | 6 votes |
@Test public void testSessionWindowExample() { List<PageView> pageViewEvents = TestUtils.genSamplePageViewData(); InMemorySystemDescriptor inMemorySystem = new InMemorySystemDescriptor("kafka"); InMemoryInputDescriptor<KV<String, PageView>> pageViewInputDescriptor = inMemorySystem.getInputDescriptor("pageview-session-input", new NoOpSerde<KV<String, PageView>>()); InMemoryOutputDescriptor<KV<String, UserPageViews>> userPageViewOutputDescriptor = inMemorySystem.getOutputDescriptor("pageview-session-output", new NoOpSerde<KV<String, UserPageViews>>()); TestRunner .of(new SessionWindowExample()) .addInputStream(pageViewInputDescriptor, pageViewEvents) .addOutputStream(userPageViewOutputDescriptor, 1) .run(Duration.ofMinutes(1)); Assert.assertEquals(2, TestRunner.consumeStream(userPageViewOutputDescriptor, Duration.ofMillis(1000)).get(0).size()); }
Example #11
Source File: TranslationContext.java From beam with Apache License 2.0 | 6 votes |
/** The dummy stream created will only be used in Beam tests. */ private static InputDescriptor<OpMessage<String>, ?> createDummyStreamDescriptor(String id) { final GenericSystemDescriptor dummySystem = new GenericSystemDescriptor(id, InMemorySystemFactory.class.getName()); final GenericInputDescriptor<OpMessage<String>> dummyInput = dummySystem.getInputDescriptor(id, new NoOpSerde<>()); dummyInput.withOffsetDefault(SystemStreamMetadata.OffsetType.OLDEST); final Config config = new MapConfig(dummyInput.toConfig(), dummySystem.toConfig()); final SystemFactory factory = new InMemorySystemFactory(); final StreamSpec dummyStreamSpec = new StreamSpec(id, id, id, 1); factory.getAdmin(id, config).createStream(dummyStreamSpec); final SystemProducer producer = factory.getProducer(id, config, null); final SystemStream sysStream = new SystemStream(id, id); final Consumer<Object> sendFn = (msg) -> { producer.send(id, new OutgoingMessageEnvelope(sysStream, 0, null, msg)); }; final WindowedValue<String> windowedValue = WindowedValue.timestampedValueInGlobalWindow("dummy", new Instant()); sendFn.accept(OpMessage.ofElement(windowedValue)); sendFn.accept(new WatermarkMessage(BoundedWindow.TIMESTAMP_MAX_VALUE.getMillis())); sendFn.accept(new EndOfStreamMessage(null)); return dummyInput; }
Example #12
Source File: ImpulseTranslator.java From beam with Apache License 2.0 | 6 votes |
@Override public void translatePortable( PipelineNode.PTransformNode transform, QueryablePipeline pipeline, PortableTranslationContext ctx) { final String outputId = ctx.getOutputId(transform); final GenericSystemDescriptor systemDescriptor = new GenericSystemDescriptor(outputId, SamzaImpulseSystemFactory.class.getName()); // The KvCoder is needed here for Samza not to crop the key. final Serde<KV<?, OpMessage<byte[]>>> kvSerde = KVSerde.of(new NoOpSerde(), new NoOpSerde<>()); final GenericInputDescriptor<KV<?, OpMessage<byte[]>>> inputDescriptor = systemDescriptor.getInputDescriptor(outputId, kvSerde); ctx.registerInputMessageStream(outputId, inputDescriptor); }
Example #13
Source File: TestSamzaCookBookExamples.java From samza-hello-samza with Apache License 2.0 | 6 votes |
@Test public void testFilterExample() { List<PageView> rawPageViewEvents = new ArrayList<>(); rawPageViewEvents.add(new PageView("google.com", "user1", "india")); rawPageViewEvents.add(new PageView("facebook.com", "invalidUserId", "france")); rawPageViewEvents.add(new PageView("yahoo.com", "user2", "china")); InMemorySystemDescriptor inMemorySystem = new InMemorySystemDescriptor("kafka"); InMemoryInputDescriptor<PageView> badPageViewEvents = inMemorySystem.getInputDescriptor("pageview-filter-input", new NoOpSerde<PageView>()); InMemoryOutputDescriptor<PageView> goodPageViewEvents = inMemorySystem.getOutputDescriptor("pageview-filter-output", new NoOpSerde<PageView>()); TestRunner .of(new FilterExample()) .addInputStream(badPageViewEvents, rawPageViewEvents) .addOutputStream(goodPageViewEvents, 1) .run(Duration.ofMillis(1500)); Assert.assertEquals(TestRunner.consumeStream(goodPageViewEvents, Duration.ofMillis(1000)).get(0).size(), 2); }
Example #14
Source File: AsyncStreamTaskIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
@Test public void testAsyncTaskWithSinglePartitionUsingStreamAssert() throws Exception { List<Integer> inputList = Arrays.asList(1, 2, 3, 4, 5); List<Integer> outputList = Arrays.asList(50, 10, 20, 30, 40); InMemorySystemDescriptor isd = new InMemorySystemDescriptor("async-test"); InMemoryInputDescriptor<Integer> imid = isd .getInputDescriptor("ints", new NoOpSerde<Integer>()); InMemoryOutputDescriptor imod = isd .getOutputDescriptor("ints-out", new NoOpSerde<>()); TestRunner .of(MyAsyncStreamTask.class) .addInputStream(imid, inputList) .addOutputStream(imod, 1) .run(Duration.ofSeconds(2)); StreamAssert.containsInAnyOrder(outputList, imod, Duration.ofMillis(1000)); }
Example #15
Source File: AsyncStreamTaskIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
@Test public void testAsyncTaskWithSinglePartition() throws Exception { List<Integer> inputList = Arrays.asList(1, 2, 3, 4, 5); List<Integer> outputList = Arrays.asList(10, 20, 30, 40, 50); InMemorySystemDescriptor isd = new InMemorySystemDescriptor("async-test"); InMemoryInputDescriptor<Integer> imid = isd .getInputDescriptor("ints", new NoOpSerde<Integer>()); InMemoryOutputDescriptor imod = isd .getOutputDescriptor("ints-out", new NoOpSerde<>()); TestRunner .of(MyAsyncStreamTask.class) .addInputStream(imid, inputList) .addOutputStream(imod, 1) .run(Duration.ofSeconds(2)); Assert.assertThat(TestRunner.consumeStream(imod, Duration.ofMillis(1000)).get(0), IsIterableContainingInOrder.contains(outputList.toArray())); }
Example #16
Source File: StreamApplicationIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
@Test public void testHighLevelApi() throws Exception { Random random = new Random(); int count = 10; List<PageView> pageViews = new ArrayList<>(); for (int memberId = 0; memberId < count; memberId++) { String pagekey = PAGEKEYS[random.nextInt(PAGEKEYS.length - 1)]; PageView pv = new PageView(pagekey, memberId); pageViews.add(pv); } InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test"); InMemoryInputDescriptor<PageView> imid = isd.getInputDescriptor("PageView", new NoOpSerde<PageView>()); InMemoryOutputDescriptor<PageView> imod = isd.getOutputDescriptor("Output", new NoOpSerde<PageView>()); TestRunner .of(new PageViewRepartitionApplication()) .addInputStream(imid, pageViews) .addOutputStream(imod, 10) .run(Duration.ofMillis(1500)); Assert.assertEquals(TestRunner.consumeStream(imod, Duration.ofMillis(1000)).get(random.nextInt(count)).size(), 1); }
Example #17
Source File: TestAsyncFlatMap.java From samza with Apache License 2.0 | 6 votes |
@Override public void describe(StreamApplicationDescriptor appDescriptor) { Config config = appDescriptor.getConfig(); KafkaSystemDescriptor kafkaSystemDescriptor = new KafkaSystemDescriptor(TEST_SYSTEM); KafkaOutputDescriptor<PageView> outputDescriptor = kafkaSystemDescriptor.getOutputDescriptor(NON_GUEST_PAGE_VIEW_STREAM, new NoOpSerde<>()); OutputStream<PageView> nonGuestPageViewStream = appDescriptor.getOutputStream(outputDescriptor); Predicate<PageView> failProcess = (Predicate<PageView> & Serializable) (ignored) -> config.getBoolean(FAIL_PROCESS, false); Predicate<PageView> failDownstreamOperator = (Predicate<PageView> & Serializable) (ignored) -> config.getBoolean(FAIL_DOWNSTREAM_OPERATOR, false); Supplier<Long> processJitter = (Supplier<Long> & Serializable) () -> config.getLong(PROCESS_JITTER, 100); appDescriptor.getInputStream(kafkaSystemDescriptor.getInputDescriptor(PAGE_VIEW_STREAM, new NoOpSerde<PageView>())) .flatMapAsync(pageView -> filterGuestPageViews(pageView, failProcess, processJitter)) .filter(pageView -> filterLoginPageViews(pageView, failDownstreamOperator)) .sendTo(nonGuestPageViewStream); }
Example #18
Source File: TestAsyncFlatMap.java From samza with Apache License 2.0 | 6 votes |
private List<PageView> runTest(List<PageView> pageViews, Map<String, String> configs) { configs.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID, PAGE_VIEW_STREAM), TEST_SYSTEM); InMemorySystemDescriptor isd = new InMemorySystemDescriptor(TEST_SYSTEM); InMemoryInputDescriptor<PageView> pageViewStreamDesc = isd .getInputDescriptor(PAGE_VIEW_STREAM, new NoOpSerde<>()); InMemoryOutputDescriptor<PageView> outputStreamDesc = isd .getOutputDescriptor(NON_GUEST_PAGE_VIEW_STREAM, new NoOpSerde<>()); TestRunner .of(new AsyncFlatMapExample()) .addInputStream(pageViewStreamDesc, pageViews) .addOutputStream(outputStreamDesc, 1) .addConfig(new MapConfig(configs)) .run(Duration.ofMillis(50000)); Map<Integer, List<PageView>> result = TestRunner.consumeStream(outputStreamDesc, Duration.ofMillis(1000)); List<PageView> results = result.values().stream() .flatMap(List::stream) .collect(Collectors.toList()); return results; }
Example #19
Source File: StreamApplicationIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
/** * Null page key is passed in input data which should fail filter logic */ @Test(expected = SamzaException.class) public void testSamzaJobFailureForStreamApplication() { int count = 10; List<TestData.PageView> pageviews = new ArrayList<>(); for (int memberId = 0; memberId < count; memberId++) { pageviews.add(new TestData.PageView(null, memberId)); } InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test"); InMemoryInputDescriptor<PageView> imid = isd.getInputDescriptor("PageView", new NoOpSerde<PageView>()); InMemoryOutputDescriptor<PageView> imod = isd.getOutputDescriptor("Output", new NoOpSerde<PageView>()); TestRunner.of(new PageViewFilterApplication()) .addInputStream(imid, pageviews) .addOutputStream(imod, 10) .run(Duration.ofMillis(1000)); }
Example #20
Source File: StreamTaskIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
@Test public void testStatefulTaskWithLocalTable() { List<PageView> pageViews = Arrays.asList(TestTableData.generatePageViews(10)); List<Profile> profiles = Arrays.asList(TestTableData.generateProfiles(10)); InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test"); InMemoryInputDescriptor<TestTableData.PageView> pageViewStreamDesc = isd .getInputDescriptor("PageView", new NoOpSerde<TestTableData.PageView>()); InMemoryInputDescriptor<TestTableData.Profile> profileStreamDesc = isd .getInputDescriptor("Profile", new NoOpSerde<TestTableData.Profile>()) .shouldBootstrap(); InMemoryOutputDescriptor<TestTableData.EnrichedPageView> outputStreamDesc = isd .getOutputDescriptor("EnrichedPageView", new NoOpSerde<>()); TestRunner .of(new JoinTaskApplication()) .addInputStream(pageViewStreamDesc, pageViews) .addInputStream(profileStreamDesc, profiles) .addOutputStream(outputStreamDesc, 1) .run(Duration.ofSeconds(2)); Assert.assertEquals(10, TestRunner.consumeStream(outputStreamDesc, Duration.ofSeconds(1)).get(0).size()); }
Example #21
Source File: StreamTaskIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
@Test public void testSyncTaskWithSinglePartition() throws Exception { List<Integer> inputList = Arrays.asList(1, 2, 3, 4, 5); List<Integer> outputList = Arrays.asList(10, 20, 30, 40, 50); InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test"); InMemoryInputDescriptor<Integer> imid = isd .getInputDescriptor("input", new NoOpSerde<Integer>()); InMemoryOutputDescriptor<Integer> imod = isd .getOutputDescriptor("output", new NoOpSerde<Integer>()); TestRunner .of(MyStreamTestTask.class) .addInputStream(imid, inputList) .addOutputStream(imod, 1) .addExternalContext(new TestContext(10)) .run(Duration.ofSeconds(1)); Assert.assertThat(TestRunner.consumeStream(imod, Duration.ofMillis(1000)).get(0), IsIterableContainingInOrder.contains(outputList.toArray())); }
Example #22
Source File: StreamTaskIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
/** * Samza job logic expects integers, but doubles are passed here which results in failure */ @Test(expected = SamzaException.class) public void testSamzaJobFailureForSyncTask() { List<Double> inputList = Arrays.asList(1.2, 2.3, 3.33, 4.5); InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test"); InMemoryInputDescriptor<Double> imid = isd .getInputDescriptor("doubles", new NoOpSerde<Double>()); InMemoryOutputDescriptor imod = isd .getOutputDescriptor("output", new NoOpSerde<>()); TestRunner .of(MyStreamTestTask.class) .addInputStream(imid, inputList) .addOutputStream(imod, 1) .addExternalContext(new TestContext(10)) .run(Duration.ofSeconds(1)); }
Example #23
Source File: StreamTaskIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
@Test public void testSyncTaskWithSinglePartitionMultithreaded() throws Exception { List<Integer> inputList = Arrays.asList(1, 2, 3, 4, 5); List<Integer> outputList = Arrays.asList(10, 20, 30, 40, 50); InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test"); InMemoryInputDescriptor<Integer> imid = isd .getInputDescriptor("input", new NoOpSerde<Integer>()); InMemoryOutputDescriptor<Integer> imod = isd .getOutputDescriptor("output", new NoOpSerde<Integer>()); TestRunner .of(MyStreamTestTask.class) .addInputStream(imid, inputList) .addOutputStream(imod, 1) .addConfig("job.container.thread.pool.size", "4") .addExternalContext(new TestContext(10)) .run(Duration.ofSeconds(1)); StreamAssert.containsInOrder(outputList, imod, Duration.ofMillis(1000)); }
Example #24
Source File: StreamTaskIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
@Test public void testSyncTaskWithMultiplePartition() throws Exception { Map<Integer, List<KV>> inputPartitionData = new HashMap<>(); Map<Integer, List<Integer>> expectedOutputPartitionData = new HashMap<>(); genData(inputPartitionData, expectedOutputPartitionData); InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test"); InMemoryInputDescriptor<KV> imid = isd .getInputDescriptor("input", new NoOpSerde<KV>()); InMemoryOutputDescriptor<Integer> imod = isd .getOutputDescriptor("output", new NoOpSerde<Integer>()); TestRunner .of(MyStreamTestTask.class) .addInputStream(imid, inputPartitionData) .addOutputStream(imod, 5) .addExternalContext(new TestContext(10)) .run(Duration.ofSeconds(2)); StreamAssert.containsInOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000)); }
Example #25
Source File: StreamTaskIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
void syncTaskWithMultiplePartitionMultithreadedHelper(Map<Integer, List<KV>> inputPartitionData, Map<Integer, List<Integer>> expectedOutputPartitionData) throws Exception { InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test"); InMemoryInputDescriptor<KV> imid = isd .getInputDescriptor("input", new NoOpSerde<KV>()); InMemoryOutputDescriptor<Integer> imod = isd .getOutputDescriptor("output", new NoOpSerde<Integer>()); TestRunner .of(MyStreamTestTask.class) .addInputStream(imid, inputPartitionData) .addOutputStream(imod, 5) .addConfig("job.container.thread.pool.size", "4") .addExternalContext(new TestContext(10)) .run(Duration.ofSeconds(2)); StreamAssert.containsInOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000)); }
Example #26
Source File: AsyncStreamTaskIntegrationTest.java From samza with Apache License 2.0 | 6 votes |
@Test public void testAsyncTaskWithMultiplePartition() throws Exception { Map<Integer, List<KV>> inputPartitionData = new HashMap<>(); Map<Integer, List<Integer>> expectedOutputPartitionData = new HashMap<>(); genData(inputPartitionData, expectedOutputPartitionData); InMemorySystemDescriptor isd = new InMemorySystemDescriptor("async-test"); InMemoryInputDescriptor<KV> imid = isd .getInputDescriptor("ints", new NoOpSerde<KV>()); InMemoryOutputDescriptor imod = isd .getOutputDescriptor("ints-out", new NoOpSerde<>()); TestRunner .of(MyAsyncStreamTask.class) .addInputStream(imid, inputPartitionData) .addOutputStream(imod, 5) .run(Duration.ofSeconds(2)); StreamAssert.containsInOrder(expectedOutputPartitionData, imod, Duration.ofMillis(1000)); }
Example #27
Source File: StreamTaskIntegrationTest.java From samza with Apache License 2.0 | 5 votes |
@Override public void describe(TaskApplicationDescriptor appDescriptor) { KafkaSystemDescriptor ksd = new KafkaSystemDescriptor("test"); KafkaInputDescriptor<Profile> profileISD = ksd.getInputDescriptor("Profile", new JsonSerdeV2<>()); KafkaInputDescriptor<PageView> pageViewISD = ksd.getInputDescriptor("PageView", new JsonSerdeV2<>()); KafkaOutputDescriptor<EnrichedPageView> enrichedPageViewOSD = ksd.getOutputDescriptor("EnrichedPageView", new NoOpSerde<>()); appDescriptor .withInputStream(profileISD) .withInputStream(pageViewISD) .withOutputStream(enrichedPageViewOSD) .withTable(new InMemoryTableDescriptor("profile-view-store", KVSerde.of(new IntegerSerde(), new TestTableData.ProfileJsonSerde()))) .withTaskFactory((StreamTaskFactory) () -> new StatefulStreamTask()); }
Example #28
Source File: StreamApplicationIntegrationTest.java From samza with Apache License 2.0 | 5 votes |
@Test public void testStatefulJoinWithLocalTable() { Random random = new Random(); List<KV<String, TestTableData.PageView>> pageViews = Arrays.asList(TestTableData.generatePageViews(10)) .stream() .map(x -> KV.of(PAGEKEYS[random.nextInt(PAGEKEYS.length)], x)) .collect(Collectors.toList()); List<KV<String, TestTableData.Profile>> profiles = Arrays.asList(TestTableData.generateProfiles(10)) .stream() .map(x -> KV.of(PAGEKEYS[random.nextInt(PAGEKEYS.length)], x)) .collect(Collectors.toList()); InMemorySystemDescriptor isd = new InMemorySystemDescriptor("test"); InMemoryInputDescriptor<KV<String, TestTableData.PageView>> pageViewStreamDesc = isd .getInputDescriptor("PageView", new NoOpSerde<KV<String, TestTableData.PageView>>()); InMemoryInputDescriptor<KV<String, TestTableData.Profile>> profileStreamDesc = isd .getInputDescriptor("Profile", new NoOpSerde<KV<String, TestTableData.Profile>>()) .shouldBootstrap(); InMemoryOutputDescriptor<TestTableData.EnrichedPageView> outputStreamDesc = isd .getOutputDescriptor("EnrichedPageView", new NoOpSerde<>()); InMemoryOutputDescriptor<String> joinKeysDescriptor = isd .getOutputDescriptor("JoinPageKeys", new NoOpSerde<>()); TestRunner .of(new PageViewProfileViewJoinApplication()) .addInputStream(pageViewStreamDesc, pageViews) .addInputStream(profileStreamDesc, profiles) .addOutputStream(outputStreamDesc, 1) .addOutputStream(joinKeysDescriptor, 1) .run(Duration.ofSeconds(2)); Assert.assertEquals(10, TestRunner.consumeStream(outputStreamDesc, Duration.ofSeconds(1)).get(0).size()); Assert.assertEquals(10, TestRunner.consumeStream(joinKeysDescriptor, Duration.ofSeconds(1)).get(0).size()); }
Example #29
Source File: TestLocalTableWithSideInputsEndToEnd.java From samza with Apache License 2.0 | 5 votes |
@Override public void describe(StreamApplicationDescriptor appDescriptor) { Table<KV<Integer, TestTableData.Profile>> table = appDescriptor.getTable(getTableDescriptor()); KafkaSystemDescriptor sd = new KafkaSystemDescriptor("test"); appDescriptor.getInputStream(sd.getInputDescriptor(PAGEVIEW_STREAM, new NoOpSerde<TestTableData.PageView>())) .partitionBy(TestTableData.PageView::getMemberId, v -> v, KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()), "partition-page-view") .join(table, new PageViewToProfileJoinFunction()) .sendTo(appDescriptor.getOutputStream(sd.getOutputDescriptor(ENRICHED_PAGEVIEW_STREAM, new NoOpSerde<>()))); }
Example #30
Source File: TestLocalTableWithConfigRewriterEndToEnd.java From samza with Apache License 2.0 | 5 votes |
@Override public void describe(TaskApplicationDescriptor appDescriptor) { DelegatingSystemDescriptor ksd = new DelegatingSystemDescriptor("test"); GenericInputDescriptor<TestTableData.PageView> pageViewISD = ksd.getInputDescriptor("PageView", new NoOpSerde<>()); appDescriptor .withInputStream(pageViewISD) .withTaskFactory((StreamTaskFactory) () -> new MyStreamTask()); }