Java Code Examples for org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator#print()

The following examples show how to use org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator#print() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Main.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
    Properties props = KafkaConfigUtil.buildKafkaProps(parameterTool);

    SingleOutputStreamOperator<MetricEvent> metricData = env.addSource(new FlinkKafkaConsumer011<>(
            parameterTool.get(METRICS_TOPIC),
            new SimpleStringSchema(),
            props)).setParallelism(1)
            .map(string -> GsonUtil.fromJson(string, MetricEvent.class));

    metricData.print();

    CheckPointUtil.setCheckpointConfig(env, parameterTool)
            .execute("zhisheng --- checkpoint config example");
}
Example 2
Source File: WordCountIntegrationTest.java    From tutorials with MIT License 6 votes vote down vote up 
@Test
public void givenStreamOfEvents_whenProcessEvents_thenShouldApplyWindowingOnTransformation() throws Exception {
    // given
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    SingleOutputStreamOperator<Tuple2<Integer, Long>> windowed = env.fromElements(new Tuple2<>(16, ZonedDateTime.now().plusMinutes(25).toInstant().getEpochSecond()), new Tuple2<>(15, ZonedDateTime.now().plusMinutes(2).toInstant().getEpochSecond()))
            .assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Tuple2<Integer, Long>>(Time.seconds(20)) {
                @Override
                public long extractTimestamp(Tuple2<Integer, Long> element) {
                    return element.f1 * 1000;
                }
            });

    SingleOutputStreamOperator<Tuple2<Integer, Long>> reduced = windowed.windowAll(TumblingEventTimeWindows.of(Time.seconds(5))).maxBy(0, true);

    reduced.print();

    // when
    env.execute();
}
Example 3
Source File: Main1.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.getConfig().setAutoWatermarkInterval(5000);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                });

        //Periodic Watermark
        data.assignTimestampsAndWatermarks(new WordPeriodicWatermark());

        data.print();
        env.execute("watermark demo");
    }
Example 4
Source File: Main.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                });

        //Punctuated Watermark
        data.assignTimestampsAndWatermarks(new WordPunctuatedWatermark());

        data.print();
        env.execute("watermark demo");
    }
Example 5
Source File: Main.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
    Properties props = KafkaConfigUtil.buildKafkaProps(parameterTool);

    SingleOutputStreamOperator<MetricEvent> metricData = env.addSource(new FlinkKafkaConsumer011<>(
            parameterTool.get(METRICS_TOPIC),
            new SimpleStringSchema(),
            props)).setParallelism(1)
            .map(string -> GsonUtil.fromJson(string, MetricEvent.class));

    metricData.print();

    CheckPointUtil.setCheckpointConfig(env, parameterTool)
            .execute("zhisheng --- checkpoint config example");
}
Example 6
Source File: Main.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    //参数检查
    if (args.length != 2) {
        System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>");
        return;
    }

    String hostname = args[0];
    Integer port = Integer.parseInt(args[1]);


    // set up the streaming execution environment
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //获取数据
    DataStreamSource<String> stream = env.socketTextStream(hostname, port);

    //计数
    SingleOutputStreamOperator<Tuple2<String, Integer>> sum = stream.flatMap(new LineSplitter())
            .keyBy(0)
            .sum(1);

    sum.print();

    env.execute("Java WordCount from SocketTextStream Example");
}
Example 7
Source File: Main1.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.getConfig().setAutoWatermarkInterval(5000);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                });

        //Periodic Watermark
        data.assignTimestampsAndWatermarks(new WordPeriodicWatermark());

        data.print();
        env.execute("watermark demo");
    }
Example 8
Source File: Main.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                });

        //Punctuated Watermark
        data.assignTimestampsAndWatermarks(new WordPunctuatedWatermark());

        data.print();
        env.execute("watermark demo");
    }
Example 9
Source File: Main.java    From flink-learning with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    //参数检查
    if (args.length != 2) {
        System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>");
        return;
    }

    String hostname = args[0];
    Integer port = Integer.parseInt(args[1]);


    // set up the streaming execution environment
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //获取数据
    DataStreamSource<String> stream = env.socketTextStream(hostname, port);

    //计数
    SingleOutputStreamOperator<Tuple2<String, Integer>> sum = stream.flatMap(new LineSplitter())
            .keyBy(0)
            .sum(1);

    sum.print();

    env.execute("Java WordCount from SocketTextStream Example");
}
Example 10
Source File: Main2.java    From flink-learning with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                });

        //BoundedOutOfOrdernessTimestampExtractor
        data.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Word>(Time.seconds(10)) {
            @Override
            public long extractTimestamp(Word element) {
                return element.getTimestamp();
            }
        });

        data.print();
        env.execute("watermark demo");
    }
Example 11
Source File: Main4.java    From flink-learning with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        OutputTag<Word> lateDataTag = new OutputTag<Word>("late") {
        };

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                }).assignTimestampsAndWatermarks(new WordPeriodicWatermark());

        SingleOutputStreamOperator<Word> sum = data.keyBy(0)
                .timeWindow(Time.seconds(10))
//                .allowedLateness(Time.milliseconds(2))
                .sideOutputLateData(lateDataTag)
                .sum(1);

        sum.print();

        sum.getSideOutput(lateDataTag)
                .print();

        env.execute("watermark demo");
    }
Example 12
Source File: Main4.java    From flink-learning with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        OutputTag<Word> lateDataTag = new OutputTag<Word>("late") {
        };

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                }).assignTimestampsAndWatermarks(new WordPeriodicWatermark());

        SingleOutputStreamOperator<Word> sum = data.keyBy(0)
                .timeWindow(Time.seconds(10))
//                .allowedLateness(Time.milliseconds(2))
                .sideOutputLateData(lateDataTag)
                .sum(1);

        sum.print();

        sum.getSideOutput(lateDataTag)
                .print();

        env.execute("watermark demo");
    }
Example 13
Source File: Main2.java    From flink-learning with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        //并行度设置为 1
        env.setParallelism(1);
//        env.setParallelism(4);

        SingleOutputStreamOperator<Word> data = env.socketTextStream("localhost", 9001)
                .map(new MapFunction<String, Word>() {
                    @Override
                    public Word map(String value) throws Exception {
                        String[] split = value.split(",");
                        return new Word(split[0], Integer.valueOf(split[1]), Long.valueOf(split[2]));
                    }
                });

        //BoundedOutOfOrdernessTimestampExtractor
        data.assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor<Word>(Time.seconds(10)) {
            @Override
            public long extractTimestamp(Word element) {
                return element.getTimestamp();
            }
        });

        data.print();
        env.execute("watermark demo");
    }
Example 14
Source File: WhyLate.java    From flink-training-exercises with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {

		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

		OutputTag<Event> lateDataTag = new OutputTag<Event>("late"){};

		// When PARTITIONS_PER_INSTANCE is greater than 1, there can be late events.
		DataStream<Event> events = env.addSource(new ParallelEventSource(PARTITIONS_PER_INSTANCE));

		// Count the number of events per user in one second event-time windows,
		// and capture late events on a side output.
		SingleOutputStreamOperator<Tuple2<String, Integer>> windowOperator = events
				.assignTimestampsAndWatermarks(new TimestampsAndWatermarks())
				.keyBy(e -> e.userId)
				.window(TumblingEventTimeWindows.of(Time.seconds(1)))
				.sideOutputLateData(lateDataTag)
				.process(new CountEventsPerUser());

		windowOperator.print();

		// Count the number of late events for every second of processing time.pri
		windowOperator.getSideOutput(lateDataTag)
				.windowAll(TumblingProcessingTimeWindows.of(Time.seconds(1)))
				.process(new CountLateEvents())
				.map(i -> new Tuple3<String, Integer, String>("LATE", i, Instant.now().toString()))
				.returns(Types.TUPLE(Types.STRING, Types.INT, Types.STRING))
				.print();

		env.execute();
	}
Example 15
Source File: Sort.java    From flink-training-exercises with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {

		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
		env.setParallelism(1);

		DataStream<Event> eventStream = env.addSource(new OutOfOrderEventSource())
				.assignTimestampsAndWatermarks(new TimestampsAndWatermarks());

		Pattern<Event, ?> matchEverything =
				Pattern.<Event>begin("any")
						.where(new SimpleCondition<Event>() {
							@Override
							public boolean filter(Event event) throws Exception {
								return true;
							}
						});

		PatternStream<Event> patternStream = CEP.pattern(eventStream, matchEverything);
		OutputTag<Event> lateDataOutputTag = new OutputTag<Event>("late-events"){};

		SingleOutputStreamOperator<Event> sorted = patternStream
				.sideOutputLateData(lateDataOutputTag)
				.select(new PatternSelectFunction<Event, Event>() {
					@Override
					public Event select(Map<String, List<Event>> map) throws Exception {
						return map.get("any").get(0);
					}
				});

		sorted.print();
		sorted
				.getSideOutput(lateDataOutputTag)
				.map(e -> new Tuple2<>(e, "LATE"))
				.returns(Types.TUPLE(TypeInformation.of(Event.class), Types.STRING))
				.print();

		env.execute();
	}
Example 16
Source File: WordCountIntegrationTest.java    From tutorials with MIT License 5 votes vote down vote up 
@Test
public void givenStreamOfEvents_whenProcessEvents_thenShouldPrintResultsOnSinkOperation() throws Exception {
    // given
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    DataStream<String> text = env.fromElements("This is a first sentence", "This is a second sentence with a one word");

    SingleOutputStreamOperator<String> upperCase = text.map(String::toUpperCase);

    upperCase.print();

    // when
    env.execute();
}
Example 17
Source File: ProcessAllWindowFunctionDemo.java    From blog_demos with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    // 使用事件时间
    env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

    // 并行度为1
    env.setParallelism(1);

    // 设置数据源，一共三个元素
    DataStream<Tuple2<String,Integer>> dataStream = env.addSource(new SourceFunction<Tuple2<String, Integer>>() {
        @Override
        public void run(SourceContext<Tuple2<String, Integer>> ctx) throws Exception {
            for(int i=1; i<Integer.MAX_VALUE; i++) {
                // 只有aaa和bbb两种name
                String name = 0==i%2 ? "aaa" : "bbb";

                // 使用当前时间作为时间戳
                long timeStamp = System.currentTimeMillis();

                // 将数据和时间戳打印出来，用来验证数据
                System.out.println(String.format("source，%s, %s\n",
                        name,
                        time(timeStamp)));

                // 发射一个元素，并且带上了时间戳
                ctx.collectWithTimestamp(new Tuple2<String, Integer>(name, 1), timeStamp);

                // 每发射一次就延时1秒
                Thread.sleep(1000);
            }
        }

        @Override
        public void cancel() {

        }
    });

    // 将数据用5秒的滚动窗口做划分，再用ProcessAllWindowFunction
    SingleOutputStreamOperator<String> mainDataStream = dataStream
            // 5秒一次的滚动窗口
            .timeWindowAll(Time.seconds(5))
            // 统计当前窗口内的元素数量，然后把数量、窗口起止时间整理成字符串发送给下游算子
            .process(new ProcessAllWindowFunction<Tuple2<String, Integer>, String, TimeWindow>() {
                @Override
                public void process(Context context, Iterable<Tuple2<String, Integer>> iterable, Collector<String> collector) throws Exception {
                    int count = 0;

                    // iterable可以访问当前窗口内的所有数据，
                    // 这里简单处理，只统计了元素数量
                    for (Tuple2<String, Integer> tuple2 : iterable) {
                        count++;
                    }

                    // 将当前窗口的起止时间和元素数量整理成字符串
                    String value = String.format("window, %s - %s, %d\n",
                            // 当前窗口的起始时间
                            time(context.window().getStart()),
                            // 当前窗口的结束时间
                            time(context.window().getEnd()),
                            // 当前key在当前窗口内元素总数
                            count);

                    // 发射到下游算子
                    collector.collect(value);
                }
            });

    // 打印结果，通过分析打印信息，检查ProcessWindowFunction中可以处理所有key的整个窗口的数据
    mainDataStream.print();

    env.execute("processfunction demo : processallwindowfunction");
}
Example 18
Source File: SideOutput.java    From blog_demos with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    // 并行度为1
    env.setParallelism(1);

    // 定义OutputTag
    final OutputTag<String> outputTag = new OutputTag<String>("side-output"){};

    // 创建一个List，里面有两个Tuple2元素
    List<Tuple2<String, Integer>> list = new ArrayList<>();
    list.add(new Tuple2("aaa", 1));
    list.add(new Tuple2("bbb", 2));
    list.add(new Tuple2("ccc", 3));

    //通过List创建DataStream
    DataStream<Tuple2<String, Integer>> fromCollectionDataStream = env.fromCollection(list);

    //所有元素都进入mainDataStream，f1字段为奇数的元素进入SideOutput
    SingleOutputStreamOperator<String> mainDataStream = fromCollectionDataStream
            .process(new ProcessFunction<Tuple2<String, Integer>, String>() {
                @Override
                public void processElement(Tuple2<String, Integer> value, Context ctx, Collector<String> out) throws Exception {

                    //进入主流程的下一个算子
                    out.collect("main, name : " + value.f0 + ", value : " + value.f1);

                    //f1字段为奇数的元素进入SideOutput
                    if(1 == value.f1 % 2) {
                        ctx.output(outputTag, "side, name : " + value.f0 + ", value : " + value.f1);
                    }
                }
            });

    // 禁止chanin，这样可以在页面上看清楚原始的DAG
    mainDataStream.disableChaining();

    // 取得旁路数据
    DataStream<String> sideDataStream = mainDataStream.getSideOutput(outputTag);


    mainDataStream.print();
    sideDataStream.print();

    env.execute("processfunction demo : sideoutput");
}