org.apache.flink.streaming.util.TestHarnessUtil Java Examples
The following examples show how to use
org.apache.flink.streaming.util.TestHarnessUtil.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CoBroadcastWithNonKeyedOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testMultiStateSupport() throws Exception { try ( TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness(new FunctionWithMultipleStates(), STATE_DESCRIPTOR, STATE_DESCRIPTOR_A) ) { testHarness.processElement2(new StreamRecord<>(5, 12L)); testHarness.processElement2(new StreamRecord<>(6, 13L)); testHarness.processElement1(new StreamRecord<>("9", 15L)); Queue<Object> expectedBr = new ConcurrentLinkedQueue<>(); expectedBr.add(new StreamRecord<>("9:key.6->6", 15L)); expectedBr.add(new StreamRecord<>("9:key.5->5", 15L)); expectedBr.add(new StreamRecord<>("9:5->value.5", 15L)); expectedBr.add(new StreamRecord<>("9:6->value.6", 15L)); TestHarnessUtil.assertOutputEquals("Wrong Side Output", expectedBr, testHarness.getOutput()); } }
Example #2
Source File: CoBroadcastWithNonKeyedOperatorTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testMultiStateSupport() throws Exception { try ( TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness(new FunctionWithMultipleStates(), STATE_DESCRIPTOR, STATE_DESCRIPTOR_A) ) { testHarness.processElement2(new StreamRecord<>(5, 12L)); testHarness.processElement2(new StreamRecord<>(6, 13L)); testHarness.processElement1(new StreamRecord<>("9", 15L)); Queue<Object> expectedBr = new ConcurrentLinkedQueue<>(); expectedBr.add(new StreamRecord<>("9:key.6->6", 15L)); expectedBr.add(new StreamRecord<>("9:key.5->5", 15L)); expectedBr.add(new StreamRecord<>("9:5->value.5", 15L)); expectedBr.add(new StreamRecord<>("9:6->value.6", 15L)); TestHarnessUtil.assertOutputEquals("Wrong Side Output", expectedBr, testHarness.getOutput()); } }
Example #3
Source File: GroupAlsoByWindowTest.java From flink-dataflow with Apache License 2.0 | 6 votes |
@Test public void testAfterWatermarkProgram() throws Exception { WindowingStrategy strategy = fixedWindowWithAfterWatermarkTriggerStrategy; long initialTime = 0L; OneInputStreamOperatorTestHarness<WindowedValue<KV<String, Integer>>, WindowedValue<KV<String, Integer>>> testHarness = createTestingOperatorAndState(strategy, initialTime); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>(makeWindowedValue(strategy, KV.of("key1", 6), new Instant(initialTime + 1), null, PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)), initialTime + 1)); expectedOutput.add(new Watermark(initialTime + 10000)); expectedOutput.add(new StreamRecord<>(makeWindowedValue(strategy, KV.of("key1", 11), new Instant(initialTime + 10000), null, PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)), initialTime + 10000)); expectedOutput.add(new StreamRecord<>(makeWindowedValue(strategy, KV.of("key2", 1), new Instant(initialTime + 19500), null, PaneInfo.createPane(true, true, PaneInfo.Timing.ON_TIME)), initialTime + 19500)); expectedOutput.add(new Watermark(initialTime + 20000)); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new ResultSortComparator()); testHarness.close(); }
Example #4
Source File: StreamMapTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testMap() throws Exception { StreamMap<Integer, String> operator = new StreamMap<Integer, String>(new Map()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<Integer, String>(operator); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>(); testHarness.open(); testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3)); expectedOutput.add(new StreamRecord<String>("+2", initialTime + 1)); expectedOutput.add(new StreamRecord<String>("+3", initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<String>("+4", initialTime + 3)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
Example #5
Source File: SourceStreamTaskTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * This test verifies that open() and close() are correctly called by the StreamTask. */ @Test @SuppressWarnings("unchecked") public void testOpenClose() throws Exception { final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>( SourceStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO); testHarness.setupOutputForSingletonOperatorChain(); StreamConfig streamConfig = testHarness.getStreamConfig(); StreamSource<String, ?> sourceOperator = new StreamSource<>(new OpenCloseTestSource()); streamConfig.setStreamOperator(sourceOperator); streamConfig.setOperatorID(new OperatorID()); testHarness.invoke(); testHarness.waitForTaskCompletion(); Assert.assertTrue("RichFunction methods where not called.", OpenCloseTestSource.closeCalled); List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput()); Assert.assertEquals(10, resultElements.size()); }
Example #6
Source File: String2SortMergeJoinOperatorTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testFullJoin() throws Exception { StreamOperator joinOperator = newOperator(FlinkJoinType.FULL, leftIsSmall); TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> testHarness = buildSortMergeJoin(joinOperator); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>(newRow("a", "02"))); expectedOutput.add(new StreamRecord<>(newRow("b", "14"))); expectedOutput.add(new StreamRecord<>(newRow("c", "2null"))); expectedOutput.add(new StreamRecord<>(newRow("d", "0null"))); testHarness.waitForTaskCompletion(); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, transformToBinary(testHarness.getOutput())); }
Example #7
Source File: LegacyKeyedProcessOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTimestampAndWatermarkQuerying() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.processElement(new StreamRecord<>(5, 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.processElement(new StreamRecord<>(6, 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
Example #8
Source File: KeyedProcessOperatorTest.java From flink with Apache License 2.0 | 5 votes |
/** * This also verifies that the timestamps ouf side-emitted records is correct. */ @Test public void testSideOutput() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new SideOutputProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>( operator, new IdentityKeySelector<>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(42, 17L /* timestamp */)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("IN:42", 17L /* timestamp */)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); ConcurrentLinkedQueue<StreamRecord<Integer>> expectedIntSideOutput = new ConcurrentLinkedQueue<>(); expectedIntSideOutput.add(new StreamRecord<>(42, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Integer>> intSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.", expectedIntSideOutput, intSideOutput); ConcurrentLinkedQueue<StreamRecord<Long>> expectedLongSideOutput = new ConcurrentLinkedQueue<>(); expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Long>> longSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.", expectedLongSideOutput, longSideOutput); testHarness.close(); }
Example #9
Source File: KeyedCoProcessOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testEventTimeTimers() throws Exception { KeyedCoProcessOperator<String, Integer, String, String> operator = new KeyedCoProcessOperator<>(new EventTimeTriggeringProcessFunction()); TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new KeyedTwoInputStreamOperatorTestHarness<>( operator, new IntToStringKeySelector<>(), new IdentityKeySelector<String>(), BasicTypeInfo.STRING_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement1(new StreamRecord<>(17, 42L)); testHarness.processElement2(new StreamRecord<>("18", 42L)); testHarness.processWatermark1(new Watermark(5)); testHarness.processWatermark2(new Watermark(5)); testHarness.processWatermark1(new Watermark(6)); testHarness.processWatermark2(new Watermark(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("INPUT1:17", 42L)); expectedOutput.add(new StreamRecord<>("INPUT2:18", 42L)); expectedOutput.add(new StreamRecord<>("17:1777", 5L)); expectedOutput.add(new Watermark(5L)); expectedOutput.add(new StreamRecord<>("18:1777", 6L)); expectedOutput.add(new Watermark(6L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
Example #10
Source File: KeyedCoProcessOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testProcessingTimeTimers() throws Exception { KeyedCoProcessOperator<String, Integer, String, String> operator = new KeyedCoProcessOperator<>(new ProcessingTimeTriggeringProcessFunction()); TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new KeyedTwoInputStreamOperatorTestHarness<>( operator, new IntToStringKeySelector<>(), new IdentityKeySelector<String>(), BasicTypeInfo.STRING_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement1(new StreamRecord<>(17)); testHarness.processElement2(new StreamRecord<>("18")); testHarness.setProcessingTime(5); testHarness.setProcessingTime(6); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("INPUT1:17")); expectedOutput.add(new StreamRecord<>("INPUT2:18")); expectedOutput.add(new StreamRecord<>("1777")); expectedOutput.add(new StreamRecord<>("1777")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
Example #11
Source File: ProcessOperatorTest.java From flink with Apache License 2.0 | 5 votes |
/** * This also verifies that the timestamps ouf side-emitted records is correct. */ @Test public void testSideOutput() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new SideOutputProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(42, 17L /* timestamp */)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("IN:42", 17L /* timestamp */)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); ConcurrentLinkedQueue<StreamRecord<Integer>> expectedIntSideOutput = new ConcurrentLinkedQueue<>(); expectedIntSideOutput.add(new StreamRecord<>(42, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Integer>> intSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.", expectedIntSideOutput, intSideOutput); ConcurrentLinkedQueue<StreamRecord<Long>> expectedLongSideOutput = new ConcurrentLinkedQueue<>(); expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Long>> longSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.", expectedLongSideOutput, longSideOutput); testHarness.close(); }
Example #12
Source File: KeyedProcessOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTimestampAndWatermarkQuerying() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new QueryingFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(17)); testHarness.processElement(new StreamRecord<>(5, 12L)); testHarness.processWatermark(new Watermark(42)); testHarness.processElement(new StreamRecord<>(6, 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5TIME:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6TIME:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
Example #13
Source File: KeyedCoProcessOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testTimestampAndProcessingTimeQuerying() throws Exception { KeyedCoProcessOperator<String, Integer, String, String> operator = new KeyedCoProcessOperator<>(new ProcessingTimeQueryingProcessFunction()); TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new KeyedTwoInputStreamOperatorTestHarness<>( operator, new IntToStringKeySelector<>(), new IdentityKeySelector<String>(), BasicTypeInfo.STRING_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(17); testHarness.processElement1(new StreamRecord<>(5)); testHarness.setProcessingTime(42); testHarness.processElement2(new StreamRecord<>("6")); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("5PT:17 TS:null")); expectedOutput.add(new StreamRecord<>("6PT:42 TS:null")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
Example #14
Source File: ProcessOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * This also verifies that the timestamps ouf side-emitted records is correct. */ @Test public void testSideOutput() throws Exception { ProcessOperator<Integer, String> operator = new ProcessOperator<>(new SideOutputProcessFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(42, 17L /* timestamp */)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("IN:42", 17L /* timestamp */)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); ConcurrentLinkedQueue<StreamRecord<Integer>> expectedIntSideOutput = new ConcurrentLinkedQueue<>(); expectedIntSideOutput.add(new StreamRecord<>(42, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Integer>> intSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.INTEGER_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.", expectedIntSideOutput, intSideOutput); ConcurrentLinkedQueue<StreamRecord<Long>> expectedLongSideOutput = new ConcurrentLinkedQueue<>(); expectedLongSideOutput.add(new StreamRecord<>(42L, 17L /* timestamp */)); ConcurrentLinkedQueue<StreamRecord<Long>> longSideOutput = testHarness.getSideOutput(SideOutputProcessFunction.LONG_OUTPUT_TAG); TestHarnessUtil.assertOutputEquals( "Side output was not correct.", expectedLongSideOutput, longSideOutput); testHarness.close(); }
Example #15
Source File: CoProcessOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTimestampAndWatermarkQuerying() throws Exception { CoProcessOperator<Integer, String, String> operator = new CoProcessOperator<>(new WatermarkQueryingProcessFunction()); TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new TwoInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processWatermark1(new Watermark(17)); testHarness.processWatermark2(new Watermark(17)); testHarness.processElement1(new StreamRecord<>(5, 12L)); testHarness.processWatermark1(new Watermark(42)); testHarness.processWatermark2(new Watermark(42)); testHarness.processElement2(new StreamRecord<>("6", 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5WM:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6WM:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
Example #16
Source File: StreamFlatMapTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testFlatMap() throws Exception { StreamFlatMap<Integer, Integer> operator = new StreamFlatMap<Integer, Integer>(new MyFlatMap()); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<Integer, Integer>(operator); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>(); testHarness.open(); testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3)); testHarness.processElement(new StreamRecord<Integer>(4, initialTime + 4)); testHarness.processElement(new StreamRecord<Integer>(5, initialTime + 5)); testHarness.processElement(new StreamRecord<Integer>(6, initialTime + 6)); testHarness.processElement(new StreamRecord<Integer>(7, initialTime + 7)); testHarness.processElement(new StreamRecord<Integer>(8, initialTime + 8)); expectedOutput.add(new StreamRecord<Integer>(2, initialTime + 2)); expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 4)); expectedOutput.add(new StreamRecord<Integer>(16, initialTime + 4)); expectedOutput.add(new StreamRecord<Integer>(6, initialTime + 6)); expectedOutput.add(new StreamRecord<Integer>(36, initialTime + 6)); expectedOutput.add(new StreamRecord<Integer>(8, initialTime + 8)); expectedOutput.add(new StreamRecord<Integer>(64, initialTime + 8)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
Example #17
Source File: LegacyKeyedCoProcessOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTimestampAndWatermarkQuerying() throws Exception { LegacyKeyedCoProcessOperator<String, Integer, String, String> operator = new LegacyKeyedCoProcessOperator<>(new WatermarkQueryingProcessFunction()); TwoInputStreamOperatorTestHarness<Integer, String, String> testHarness = new KeyedTwoInputStreamOperatorTestHarness<>( operator, new IntToStringKeySelector<>(), new IdentityKeySelector<String>(), BasicTypeInfo.STRING_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark1(new Watermark(17)); testHarness.processWatermark2(new Watermark(17)); testHarness.processElement1(new StreamRecord<>(5, 12L)); testHarness.processWatermark1(new Watermark(42)); testHarness.processWatermark2(new Watermark(42)); testHarness.processElement2(new StreamRecord<>("6", 13L)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(17L)); expectedOutput.add(new StreamRecord<>("5WM:17 TS:12", 12L)); expectedOutput.add(new Watermark(42L)); expectedOutput.add(new StreamRecord<>("6WM:42 TS:13", 13L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
Example #18
Source File: StreamFilterTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test @SuppressWarnings("unchecked") public void testFilter() throws Exception { StreamFilter<Integer> operator = new StreamFilter<Integer>(new MyFilter()); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<Integer, Integer>(operator); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>(); testHarness.open(); testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1)); testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2)); testHarness.processWatermark(new Watermark(initialTime + 2)); testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3)); testHarness.processElement(new StreamRecord<Integer>(4, initialTime + 4)); testHarness.processElement(new StreamRecord<Integer>(5, initialTime + 5)); testHarness.processElement(new StreamRecord<Integer>(6, initialTime + 6)); testHarness.processElement(new StreamRecord<Integer>(7, initialTime + 7)); expectedOutput.add(new StreamRecord<Integer>(2, initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 4)); expectedOutput.add(new StreamRecord<Integer>(6, initialTime + 6)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
Example #19
Source File: CoBroadcastWithKeyedOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSideOutput() throws Exception { try ( TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness( BasicTypeInfo.STRING_TYPE_INFO, new IdentityKeySelector<>(), new FunctionWithSideOutput()) ) { testHarness.processWatermark1(new Watermark(10L)); testHarness.processWatermark2(new Watermark(10L)); testHarness.processElement2(new StreamRecord<>(5, 12L)); testHarness.processWatermark1(new Watermark(40L)); testHarness.processWatermark2(new Watermark(40L)); testHarness.processElement1(new StreamRecord<>("6", 13L)); testHarness.processElement1(new StreamRecord<>("6", 15L)); testHarness.processWatermark1(new Watermark(50L)); testHarness.processWatermark2(new Watermark(50L)); Queue<StreamRecord<String>> expectedBr = new ConcurrentLinkedQueue<>(); expectedBr.add(new StreamRecord<>("BR:5 WM:10 TS:12", 12L)); Queue<StreamRecord<String>> expectedNonBr = new ConcurrentLinkedQueue<>(); expectedNonBr.add(new StreamRecord<>("NON-BR:6 WM:40 TS:13", 13L)); expectedNonBr.add(new StreamRecord<>("NON-BR:6 WM:40 TS:15", 15L)); TestHarnessUtil.assertOutputEquals( "Wrong Side Output", expectedBr, testHarness.getSideOutput(FunctionWithSideOutput.BROADCAST_TAG)); TestHarnessUtil.assertOutputEquals( "Wrong Side Output", expectedNonBr, testHarness.getSideOutput(FunctionWithSideOutput.NON_BROADCAST_TAG)); } }
Example #20
Source File: KeyedProcessOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Verifies that we don't have leakage between different keys. */ @Test public void testEventTimeTimerWithState() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(1)); testHarness.processElement(new StreamRecord<>(17, 0L)); // should set timer for 6 testHarness.processElement(new StreamRecord<>(13, 0L)); // should set timer for 6 testHarness.processWatermark(new Watermark(2)); testHarness.processElement(new StreamRecord<>(42, 1L)); // should set timer for 7 testHarness.processElement(new StreamRecord<>(13, 1L)); // should delete timer testHarness.processWatermark(new Watermark(6)); testHarness.processWatermark(new Watermark(7)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(1L)); expectedOutput.add(new StreamRecord<>("INPUT:17", 0L)); expectedOutput.add(new StreamRecord<>("INPUT:13", 0L)); expectedOutput.add(new Watermark(2L)); expectedOutput.add(new StreamRecord<>("INPUT:42", 1L)); expectedOutput.add(new StreamRecord<>("STATE:17", 6L)); expectedOutput.add(new Watermark(6L)); expectedOutput.add(new StreamRecord<>("STATE:42", 7L)); expectedOutput.add(new Watermark(7L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
Example #21
Source File: String2HashJoinOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testInnerHashJoin() throws Exception { init(false, false, true); testHarness.processElement(new StreamRecord<>(newRow("a", "0"), initialTime), 0, 0); testHarness.processElement(new StreamRecord<>(newRow("d", "0"), initialTime), 0, 0); testHarness.processElement(new StreamRecord<>(newRow("b", "1"), initialTime), 0, 1); testHarness.waitForInputProcessing(); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, transformToBinary(testHarness.getOutput())); testHarness.endInput(0, 0); testHarness.endInput(0, 1); testHarness.waitForInputProcessing(); testHarness.processElement(new StreamRecord<>(newRow("a", "2"), initialTime), 1, 1); expectedOutput.add(new StreamRecord<>(newRow("a", "02"))); testHarness.waitForInputProcessing(); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, transformToBinary(testHarness.getOutput())); testHarness.processElement(new StreamRecord<>(newRow("c", "2"), initialTime), 1, 1); testHarness.processElement(new StreamRecord<>(newRow("b", "4"), initialTime), 1, 0); expectedOutput.add(new StreamRecord<>(newRow("b", "14"))); testHarness.waitForInputProcessing(); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, transformToBinary(testHarness.getOutput())); testHarness.endInput(1, 0); testHarness.endInput(1, 1); testHarness.waitForTaskCompletion(); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, transformToBinary(testHarness.getOutput())); }
Example #22
Source File: LegacyKeyedProcessOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Verifies that we don't have leakage between different keys. */ @Test public void testProcessingTimeTimerWithState() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.PROCESSING_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(1); testHarness.processElement(new StreamRecord<>(17)); // should set timer for 6 testHarness.setProcessingTime(2); testHarness.processElement(new StreamRecord<>(42)); // should set timer for 7 testHarness.setProcessingTime(6); testHarness.setProcessingTime(7); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("INPUT:17")); expectedOutput.add(new StreamRecord<>("INPUT:42")); expectedOutput.add(new StreamRecord<>("STATE:17")); expectedOutput.add(new StreamRecord<>("STATE:42")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
Example #23
Source File: CoStreamMapTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testCoMap() throws Exception { CoStreamMap<Double, Integer, String> operator = new CoStreamMap<Double, Integer, String>(new MyCoMap()); TwoInputStreamOperatorTestHarness<Double, Integer, String> testHarness = new TwoInputStreamOperatorTestHarness<Double, Integer, String>(operator); long initialTime = 0L; ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>(); testHarness.open(); testHarness.processElement1(new StreamRecord<Double>(1.1d, initialTime + 1)); testHarness.processElement1(new StreamRecord<Double>(1.2d, initialTime + 2)); testHarness.processElement1(new StreamRecord<Double>(1.3d, initialTime + 3)); testHarness.processWatermark1(new Watermark(initialTime + 3)); testHarness.processElement1(new StreamRecord<Double>(1.4d, initialTime + 4)); testHarness.processElement1(new StreamRecord<Double>(1.5d, initialTime + 5)); testHarness.processElement2(new StreamRecord<Integer>(1, initialTime + 1)); testHarness.processElement2(new StreamRecord<Integer>(2, initialTime + 2)); testHarness.processWatermark2(new Watermark(initialTime + 2)); testHarness.processElement2(new StreamRecord<Integer>(3, initialTime + 3)); testHarness.processElement2(new StreamRecord<Integer>(4, initialTime + 4)); testHarness.processElement2(new StreamRecord<Integer>(5, initialTime + 5)); expectedOutput.add(new StreamRecord<String>("1.1", initialTime + 1)); expectedOutput.add(new StreamRecord<String>("1.2", initialTime + 2)); expectedOutput.add(new StreamRecord<String>("1.3", initialTime + 3)); expectedOutput.add(new StreamRecord<String>("1.4", initialTime + 4)); expectedOutput.add(new StreamRecord<String>("1.5", initialTime + 5)); expectedOutput.add(new StreamRecord<String>("1", initialTime + 1)); expectedOutput.add(new StreamRecord<String>("2", initialTime + 2)); expectedOutput.add(new Watermark(initialTime + 2)); expectedOutput.add(new StreamRecord<String>("3", initialTime + 3)); expectedOutput.add(new StreamRecord<String>("4", initialTime + 4)); expectedOutput.add(new StreamRecord<String>("5", initialTime + 5)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
Example #24
Source File: SourceStreamTaskTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testNotMarkingEndOfInputWhenTaskCancelled () throws Exception { final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>( SourceStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO); testHarness .setupOperatorChain( new OperatorID(), new StreamSource<>(new CancelTestSource( BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), "Hello"))) .chain( new OperatorID(), new TestBoundedOneInputStreamOperator("Operator1"), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig())) .finish(); StreamConfig streamConfig = testHarness.getStreamConfig(); streamConfig.setTimeCharacteristic(TimeCharacteristic.ProcessingTime); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.invoke(); CancelTestSource.getDataProcessing().await(); testHarness.getTask().cancel(); try { testHarness.waitForTaskCompletion(); } catch (Throwable t) { if (!ExceptionUtils.findThrowable(t, CancelTaskException.class).isPresent()) { throw t; } } expectedOutput.add(new StreamRecord<>("Hello")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }
Example #25
Source File: CoBroadcastWithKeyedOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSideOutput() throws Exception { try ( TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness( BasicTypeInfo.STRING_TYPE_INFO, new IdentityKeySelector<>(), new FunctionWithSideOutput()) ) { testHarness.processWatermark1(new Watermark(10L)); testHarness.processWatermark2(new Watermark(10L)); testHarness.processElement2(new StreamRecord<>(5, 12L)); testHarness.processWatermark1(new Watermark(40L)); testHarness.processWatermark2(new Watermark(40L)); testHarness.processElement1(new StreamRecord<>("6", 13L)); testHarness.processElement1(new StreamRecord<>("6", 15L)); testHarness.processWatermark1(new Watermark(50L)); testHarness.processWatermark2(new Watermark(50L)); Queue<StreamRecord<String>> expectedBr = new ConcurrentLinkedQueue<>(); expectedBr.add(new StreamRecord<>("BR:5 WM:10 TS:12", 12L)); Queue<StreamRecord<String>> expectedNonBr = new ConcurrentLinkedQueue<>(); expectedNonBr.add(new StreamRecord<>("NON-BR:6 WM:40 TS:13", 13L)); expectedNonBr.add(new StreamRecord<>("NON-BR:6 WM:40 TS:15", 15L)); TestHarnessUtil.assertOutputEquals( "Wrong Side Output", expectedBr, testHarness.getSideOutput(FunctionWithSideOutput.BROADCAST_TAG)); TestHarnessUtil.assertOutputEquals( "Wrong Side Output", expectedNonBr, testHarness.getSideOutput(FunctionWithSideOutput.NON_BROADCAST_TAG)); } }
Example #26
Source File: WindowOperatorTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testCleanupTimerWithEmptyListStateForSessionWindows() throws Exception { final int gapSize = 3; final long lateness = 10; ListStateDescriptor<Tuple2<String, Integer>> windowStateDesc = new ListStateDescriptor<>("window-contents", STRING_INT_TUPLE.createSerializer(new ExecutionConfig())); WindowOperator<String, Tuple2<String, Integer>, Iterable<Tuple2<String, Integer>>, Tuple2<String, Integer>, TimeWindow> operator = new WindowOperator<>( EventTimeSessionWindows.withGap(Time.seconds(gapSize)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), windowStateDesc, new InternalIterableWindowFunction<>(new PassThroughFunction()), EventTimeTrigger.create(), lateness, null /* late data output tag */); OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = createTestHarness(operator); testHarness.open(); ConcurrentLinkedQueue<Object> expected = new ConcurrentLinkedQueue<>(); testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), 1000)); testHarness.processWatermark(new Watermark(4998)); expected.add(new StreamRecord<>(new Tuple2<>("key2", 1), 3999)); expected.add(new Watermark(4998)); testHarness.processWatermark(new Watermark(14600)); expected.add(new Watermark(14600)); ConcurrentLinkedQueue<Object> actual = testHarness.getOutput(); TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expected, actual, new Tuple2ResultSortComparator()); testHarness.close(); }
Example #27
Source File: CoBroadcastWithKeyedOperatorTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testFunctionWithTimer() throws Exception { final String expectedKey = "6"; try ( TwoInputStreamOperatorTestHarness<String, Integer, String> testHarness = getInitializedTestHarness( BasicTypeInfo.STRING_TYPE_INFO, new IdentityKeySelector<>(), new FunctionWithTimerOnKeyed(41L, expectedKey)) ) { testHarness.processWatermark1(new Watermark(10L)); testHarness.processWatermark2(new Watermark(10L)); testHarness.processElement2(new StreamRecord<>(5, 12L)); testHarness.processWatermark1(new Watermark(40L)); testHarness.processWatermark2(new Watermark(40L)); testHarness.processElement1(new StreamRecord<>(expectedKey, 13L)); testHarness.processElement1(new StreamRecord<>(expectedKey, 15L)); testHarness.processWatermark1(new Watermark(50L)); testHarness.processWatermark2(new Watermark(50L)); Queue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(10L)); expectedOutput.add(new StreamRecord<>("BR:5 WM:10 TS:12", 12L)); expectedOutput.add(new Watermark(40L)); expectedOutput.add(new StreamRecord<>("NON-BR:6 WM:40 TS:13", 13L)); expectedOutput.add(new StreamRecord<>("NON-BR:6 WM:40 TS:15", 15L)); expectedOutput.add(new StreamRecord<>("TIMER:41", 41L)); expectedOutput.add(new Watermark(50L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); } }
Example #28
Source File: EventTimeOrderingOperatorTest.java From flink-connectors with Apache License 2.0 | 5 votes |
@Test public void testLateElements() throws Exception { testHarness.processWatermark(1L); assertEquals(1L, operator.lastWatermark); testHarness.processElement(record(K1, 0L)); testHarness.processElement(record(K1, 1L)); testHarness.processWatermark(2L); assertEquals(2L, operator.lastWatermark); Queue<Object> actual = testHarness.getOutput(); Queue<Object> expected = new ConcurrentLinkedQueue<>(); expected.add(watermark(1L)); expected.add(watermark(2L)); TestHarnessUtil.assertOutputEquals("Unexpected output", expected, actual); }
Example #29
Source File: KeyedProcessOperatorTest.java From flink with Apache License 2.0 | 5 votes |
/** * Verifies that we don't have leakage between different keys. */ @Test public void testEventTimeTimerWithState() throws Exception { KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new TriggeringStatefulFlatMapFunction(TimeDomain.EVENT_TIME)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processWatermark(new Watermark(1)); testHarness.processElement(new StreamRecord<>(17, 0L)); // should set timer for 6 testHarness.processElement(new StreamRecord<>(13, 0L)); // should set timer for 6 testHarness.processWatermark(new Watermark(2)); testHarness.processElement(new StreamRecord<>(42, 1L)); // should set timer for 7 testHarness.processElement(new StreamRecord<>(13, 1L)); // should delete timer testHarness.processWatermark(new Watermark(6)); testHarness.processWatermark(new Watermark(7)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new Watermark(1L)); expectedOutput.add(new StreamRecord<>("INPUT:17", 0L)); expectedOutput.add(new StreamRecord<>("INPUT:13", 0L)); expectedOutput.add(new Watermark(2L)); expectedOutput.add(new StreamRecord<>("INPUT:42", 1L)); expectedOutput.add(new StreamRecord<>("STATE:17", 6L)); expectedOutput.add(new Watermark(6L)); expectedOutput.add(new StreamRecord<>("STATE:42", 7L)); expectedOutput.add(new Watermark(7L)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
Example #30
Source File: StatefulOperatorChainedTaskTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private void processRecords(OneInputStreamTaskTestHarness<String, String> testHarness) throws Exception { ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); testHarness.processElement(new StreamRecord<>("10"), 0, 0); testHarness.processElement(new StreamRecord<>("20"), 0, 0); testHarness.processElement(new StreamRecord<>("30"), 0, 0); testHarness.waitForInputProcessing(); expectedOutput.add(new StreamRecord<>("10")); expectedOutput.add(new StreamRecord<>("20")); expectedOutput.add(new StreamRecord<>("30")); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); }