Java Code Examples for org.apache.flink.streaming.api.datastream.DataStream#process()

The following examples show how to use org.apache.flink.streaming.api.datastream.DataStream#process() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SideOutputITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSideOutputWithMultipleConsumersWithObjectReuse() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
Example 2
Source File: SideOutputITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Test ProcessFunction side output.
 */
@Test
public void testProcessFunctionSideOutput() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
	passThroughtStream.addSink(resultSink);
	see.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
Example 3
Source File: SideOutputITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSideOutputNameClash() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);

	expectedException.expect(UnsupportedOperationException.class);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
}
Example 4
Source File: SideOutputITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDifferentSideOutputTypes() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("string"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("int"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList(13, 13, 13, 13, 13), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
Example 5
Source File: SideOutputITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSideOutputWithMultipleConsumersWithObjectReuse() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
Example 6
Source File: SideOutputITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSideOutputWithMultipleConsumers() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
Example 7
Source File: BroadcastStateTransformationTest.java    From bravo with Apache License 2.0 5 votes vote down vote up 
public DataStream<String> constructTestPipeline(DataStream<String> source) {

		OutputTag<Integer> filtered = new OutputTag<>("filter", BasicTypeInfo.INT_TYPE_INFO);
		OutputTag<Integer> process = new OutputTag<>("process", BasicTypeInfo.INT_TYPE_INFO);

		SingleOutputStreamOperator<String> input = source.process(new ProcessFunction<String, String>() {
			private static final long serialVersionUID = 1L;

			@Override
			public void processElement(String s, Context ctx,
					Collector<String> out) throws Exception {

				if (s.startsWith("filter ")) {
					ctx.output(filtered, Integer.parseInt(s.substring(7)));
				} else if (s.startsWith("process ")) {
					ctx.output(process, Integer.parseInt(s.substring(8)));
				} else {
					throw new RuntimeException("oOoO");
				}

			}
		});

		BroadcastStream<Integer> broadcast = input.getSideOutput(filtered).broadcast(bcstate);

		return input.getSideOutput(process).keyBy(i -> i).connect(broadcast).process(new BroadcastProcessor(bcstate))
				.uid("stateful");
	}
Example 8
Source File: SideOutputITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Test ProcessFunction side output.
 */
@Test
public void testProcessFunctionSideOutput() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
	passThroughtStream.addSink(resultSink);
	see.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
Example 9
Source File: SideOutputITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSideOutputNameClash() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);

	expectedException.expect(UnsupportedOperationException.class);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
}
Example 10
Source File: SideOutputITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDifferentSideOutputTypes() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("string"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("int"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList(13, 13, 13, 13, 13), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
Example 11
Source File: SideOutputITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSideOutputWithMultipleConsumersWithObjectReuse() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
Example 12
Source File: SideOutputITCase.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSideOutputWithMultipleConsumers() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
Example 13
Source File: SideOutputITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Test ProcessFunction side output.
 */
@Test
public void testProcessFunctionSideOutput() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink);
	passThroughtStream.addSink(resultSink);
	see.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
Example 14
Source File: SideOutputITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSideOutputNameClash() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);

	expectedException.expect(UnsupportedOperationException.class);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
}
Example 15
Source File: SideOutputITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testDifferentSideOutputTypes() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("string"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("int"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableObjectReuse();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList(13, 13, 13, 13, 13), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
Example 16
Source File: SideOutputITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSideOutputWithMultipleConsumers() throws Exception {
	final OutputTag<String> sideOutputTag = new OutputTag<String>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<String> sideOutputResultSink2 = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	DataStream<Integer> dataStream = env.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag, "sideout-" + String.valueOf(value));
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink1);
	passThroughtStream.getSideOutput(sideOutputTag).addSink(sideOutputResultSink2);
	passThroughtStream.addSink(resultSink);
	env.execute();

	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink1.getSortedResult());
	assertEquals(Arrays.asList("sideout-1", "sideout-2", "sideout-3", "sideout-4", "sideout-5"), sideOutputResultSink2.getSortedResult());
	assertEquals(Arrays.asList(1, 2, 3, 4, 5), resultSink.getSortedResult());
}
Example 17
Source File: SideOutput.java    From blog_demos with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    // 并行度为1
    env.setParallelism(1);

    // 定义OutputTag
    final OutputTag<String> outputTag = new OutputTag<String>("side-output"){};

    // 创建一个List，里面有两个Tuple2元素
    List<Tuple2<String, Integer>> list = new ArrayList<>();
    list.add(new Tuple2("aaa", 1));
    list.add(new Tuple2("bbb", 2));
    list.add(new Tuple2("ccc", 3));

    //通过List创建DataStream
    DataStream<Tuple2<String, Integer>> fromCollectionDataStream = env.fromCollection(list);

    //所有元素都进入mainDataStream，f1字段为奇数的元素进入SideOutput
    SingleOutputStreamOperator<String> mainDataStream = fromCollectionDataStream
            .process(new ProcessFunction<Tuple2<String, Integer>, String>() {
                @Override
                public void processElement(Tuple2<String, Integer> value, Context ctx, Collector<String> out) throws Exception {

                    //进入主流程的下一个算子
                    out.collect("main, name : " + value.f0 + ", value : " + value.f1);

                    //f1字段为奇数的元素进入SideOutput
                    if(1 == value.f1 % 2) {
                        ctx.output(outputTag, "side, name : " + value.f0 + ", value : " + value.f1);
                    }
                }
            });

    // 禁止chanin，这样可以在页面上看清楚原始的DAG
    mainDataStream.disableChaining();

    // 取得旁路数据
    DataStream<String> sideDataStream = mainDataStream.getSideOutput(outputTag);


    mainDataStream.print();
    sideDataStream.print();

    env.execute("processfunction demo : sideoutput");
}
Example 18
Source File: SideOutputITCase.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test
public void testUnionOfTwoSideOutputs() throws Exception {
	TestListResultSink<Integer> evensResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> oddsResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> oddsUEvensResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> evensUOddsResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> oddsUOddsResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> evensUEvensResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> oddsUEvensExternalResultSink = new TestListResultSink<>();
	TestListResultSink<Integer> resultSink = new TestListResultSink<>();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	DataStream<Integer> input = env.fromElements(1, 2, 3, 4);

	OutputTag<Integer> oddTag = new OutputTag<Integer>("odds"){};
	OutputTag<Integer> evenTag = new OutputTag<Integer>("even"){};

	SingleOutputStreamOperator<Integer> passThroughStream =
		input.process(new ProcessFunction<Integer, Integer>() {
			@Override
			public void processElement(Integer value, Context ctx, Collector<Integer> out) throws Exception {
				if (value % 2 != 0) {
					ctx.output(oddTag, value);
				}
				else {
					ctx.output(evenTag, value);
				}
				out.collect(value);
			}
		});

	DataStream<Integer> evens = passThroughStream.getSideOutput(evenTag);
	DataStream<Integer> odds = passThroughStream.getSideOutput(oddTag);

	evens.addSink(evensResultSink);
	odds.addSink(oddsResultSink);
	passThroughStream.addSink(resultSink);

	odds.union(evens).addSink(oddsUEvensResultSink);
	evens.union(odds).addSink(evensUOddsResultSink);

	odds.union(odds).addSink(oddsUOddsResultSink);
	evens.union(evens).addSink(evensUEvensResultSink);

	odds.union(env.fromElements(2, 4)).addSink(oddsUEvensExternalResultSink);

	env.execute();

	assertEquals(
		Arrays.asList(1, 3),
		oddsResultSink.getSortedResult());

	assertEquals(
		Arrays.asList(2, 4),
		evensResultSink.getSortedResult());

	assertEquals(
		Arrays.asList(1, 2, 3, 4),
		resultSink.getSortedResult());

	assertEquals(
		Arrays.asList(1, 2, 3, 4),
		oddsUEvensResultSink.getSortedResult());

	assertEquals(
		Arrays.asList(1, 2, 3, 4),
		evensUOddsResultSink.getSortedResult());

	assertEquals(
		Arrays.asList(1, 1, 3, 3),
		oddsUOddsResultSink.getSortedResult());

	assertEquals(
		Arrays.asList(2, 2, 4, 4),
		evensUEvensResultSink.getSortedResult());

	assertEquals(
		Arrays.asList(1, 2, 3, 4),
		oddsUEvensExternalResultSink.getSortedResult());
}