Java Code Examples for org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#socketTextStream()

public static void main(String[] args) throws Exception {
    //参数检查
    if (args.length != 2) {
        System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>");
        return;
    }

    String hostname = args[0];
    Integer port = Integer.parseInt(args[1]);


    // set up the streaming execution environment
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //获取数据
    DataStreamSource<String> stream = env.socketTextStream(hostname, port);

    //计数
    SingleOutputStreamOperator<Tuple2<String, Integer>> sum = stream.flatMap(new LineSplitter())
            .keyBy(0)
            .sum(1);

    sum.print();

    env.execute("Java WordCount from SocketTextStream Example");
}
 

public static void main(String[] args) throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<String> text = env.socketTextStream("127.0.0.1", 18081, "\n");

	DataStream<WordWithCount> windowCount = text.flatMap(new FlatMapFunction<String, WordWithCount>() {
		public void flatMap(String value, Collector<WordWithCount> out) throws Exception {
			String[] splits = value.split("\\s");
			for (String word:splits) {
				out.collect(new WordWithCount(word,1L));
			}
		}
	})
			.keyBy("word")
			.timeWindow(Time.seconds(5),Time.seconds(1))
			.sum("count");
	windowCount.print().setParallelism(1);
	env.execute("Flink Streaming Java API Skeleton");
}
 

public static void main(String[] args) throws Exception {
    //参数检查
    if (args.length != 2) {
        System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>");
        return;
    }

    String hostname = args[0];
    Integer port = Integer.parseInt(args[1]);


    // set up the streaming execution environment
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //获取数据
    DataStreamSource<String> stream = env.socketTextStream(hostname, port);

    //计数
    SingleOutputStreamOperator<Tuple2<String, Integer>> sum = stream.flatMap(new LineSplitter())
            .keyBy(0)
            .sum(1);

    sum.print();

    env.execute("Java WordCount from SocketTextStream Example");
}
 

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<String> dataStream = environment.socketTextStream("localhost", 9999);

    dataStream.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
        @Override
        public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
            String[] tokens = value.toLowerCase().split(",");
            for (String token : tokens) {
                if (token.length() > 0) {
                    out.collect(new Tuple2<>(token, 1));
                }
            }
        }
    }).keyBy(0)
            .timeWindow(Time.seconds(5))
            .sum(1)
            .print()
            .setParallelism(1);

    environment.execute("JavaWindowApp");
}
 

public static void main(String[] args) throws Exception{
        // 创建 execution environment
        final StreamExecutionEnvironment env =
                StreamExecutionEnvironment.getExecutionEnvironment();
        // 通过连接 socket 获取输入数据,这里连接到本地 9000 端口,如果 9000 端口已被占用,请换一个端口
        DataStream<String> text = env.socketTextStream("localhost", 9000, "\n");
        // 解析数据,按 word 分组,开窗,聚合
        DataStream<Tuple2<String, Integer>> windowCounts = text.
                flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
                    @Override
                    public void flatMap(String value, Collector<Tuple2<String,
                            Integer>> out) {
                        for (String word : value.split("\\s")) {
                            out.collect(Tuple2.of(word, 1));
                        }
                    }
                })
                .keyBy(0)
                .timeWindow(Time.seconds(5))
                .sum(1);
// 将结果打印到控制台,注意这里使用的是单线程打印,而非多线程
        windowCounts.print().setParallelism(1);
        env.execute("Socket Window WordCount");
    }
 

public static void main(String[] args) throws Exception {
        //参数检查
        if (args.length != 2) {
            System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>");
            return;
        }

        String hostname = args[0];
        Integer port = Integer.parseInt(args[1]);

        // set up the streaming execution environment
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //获取数据
        DataStreamSource<String> stream = env.socketTextStream(hostname, port);

        //计数
        stream.flatMap((s, collector) -> {
            for (String token : s.toLowerCase().split("\\W+")) {
                if (token.length() > 0) {
                    collector.collect(new Tuple2<String, Integer>(token, 1));
                }
            }
        })
//                .returns((TypeInformation) TupleTypeInfo.getBasicTupleTypeInfo(String.class, Integer.class))
                .keyBy(0)
                .sum(1)
                .print();

        env.execute("Java WordCount from SocketTextStream Example");
    }
 

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<String> dataStream = environment.socketTextStream("localhost", 9999);

    dataStream.flatMap(new FlatMapFunction<String, Tuple2<Integer, Integer>>() {
        @Override
        public void flatMap(String value, Collector<Tuple2<Integer, Integer>> out) throws Exception {
            String[] tokens = value.toLowerCase().split(",");
            for (String token : tokens) {
                if (token.length() > 0) {
                    out.collect(new Tuple2<>(1, Integer.parseInt(token)));
                }
            }
        }
    }).keyBy(0)
            .timeWindow(Time.seconds(5))
            .reduce(new ReduceFunction<Tuple2<Integer, Integer>>() {
                @Override
                public Tuple2<Integer, Integer> reduce(Tuple2<Integer, Integer> value1, Tuple2<Integer, Integer> value2) throws Exception {
                    System.out.println(value1 + "...." + value2);
                    return new Tuple2<>(value1.f0, value1.f1 + value2.f1);
                }
            })
            .print()
            .setParallelism(1);

    environment.execute("JavaWindowReduceApp");
}
 

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    DataStreamSource<String> source = env.socketTextStream("127.0.0.1", 9000);
    source.addSink(new MySink("6")).setParallelism(5);
    env.execute("xxxx");
}
 

public static void main(String[] args) throws Exception {
    //获取参数
    int port = 0;
    try {
        ParameterTool tool = ParameterTool.fromArgs(args);
        port = tool.getInt("port");
    } catch (Exception e) {
        System.err.println("Port undefined, use 9999.");
        port = 9999;
    }
    
    //获取执行环境
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //读取数据
    DataStreamSource<String> dataStreamSource = env.socketTextStream("localhost", 9999);

    //执行转换操作
    dataStreamSource.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
        @Override
        public void flatMap(String value, Collector<Tuple2<String, Integer>> collector) throws Exception {
            String[] tokens = value.toLowerCase().split(",");
            for (String token : tokens) {
                if (token.length() > 0) {
                    collector.collect(new Tuple2<String, Integer>(token, 1));
                }
            }
        }
    }).keyBy(0)
    .timeWindow(Time.seconds(5))
    .sum(1)
    .print()
    .setParallelism(1);

    env.execute("StreamingWCJavaApp");
}
 

public static void main(String[] args) throws Exception {
    final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(parameterTool);
    env.setParallelism(1);

    DataStreamSource<String> data = env.socketTextStream("127.0.0.1", 9200);

    Pattern<String, String> pattern = Pattern.<String>begin("start")
            .where(new SimpleCondition<String>() {
                @Override
                public boolean filter(String s) throws Exception {
                    return "a".equals(s);
                }
            })
            .times(5).optional();

    CEP.pattern(data, pattern)
            .select(new PatternSelectFunction<String, String>() {
                @Override
                public String select(Map<String, List<String>> map) throws Exception {
                    log.info(map.toString());
                    return map.get("start").get(0);
                }
            }).print();
    env.execute("flink learning cep Individual Pattern Quantifier");
}
 

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    env.setParallelism(2);
    DataStreamSource<String> data = env.socketTextStream("localhost", 9001);

    data.map(new MapFunction<String, Tuple2<String, Long>>() {
        @Override
        public Tuple2<String, Long> map(String s) throws Exception {
            String[] split = s.split(",");
            return new Tuple2<>(split[0], Long.valueOf(split[1]));
        }
    }).assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<Tuple2<String, Long>>() {
        private long currentTimestamp;

        @Nullable
        @Override
        public Watermark getCurrentWatermark() {
            return new Watermark(currentTimestamp);
        }

        @Override
        public long extractTimestamp(Tuple2<String, Long> tuple2, long l) {
            long timestamp = tuple2.f1;
            currentTimestamp = Math.max(timestamp, currentTimestamp);
            return timestamp;
        }
    }).keyBy(0)
            .window(EventTimeSessionWindows.withGap(Time.minutes(5)))
            .sum(1)
            .print("session ");
    System.out.println(env.getExecutionPlan());
    env.execute();
}
 

public static void main(String[] args) throws Exception {

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();


        // 添加数组作为数据输入源
        String[] elementInput = new String[]{"hello Flink", "Second Line"};
        DataStream<String> text = env.fromElements(elementInput);

        // 添加List集合作为数据输入源
        List<String> collectionInput = new ArrayList<>();
        collectionInput.add("hello Flink");
        DataStream<String> text2 = env.fromCollection(collectionInput);

        // 添加Socket作为数据输入源
        // 4个参数 -> (hostname:Ip地址, port:端口, delimiter:分隔符, maxRetry:最大重试次数)
        DataStream<String> text3 = env.socketTextStream("localhost", 9999, "\n", 4);


        // 添加文件源
        // 直接读取文本文件
        DataStream<String> text4 = env.readTextFile("/opt/history.log");
        // 指定 CsvInputFormat, 监控csv文件(两种模式), 时间间隔是10ms
        DataStream<String> text5 = env.readFile(new CsvInputFormat<String>(new Path("/opt/history.csv")) {
            @Override
            protected String fillRecord(String s, Object[] objects) {
                return null;
            }
        },"/opt/history.csv", FileProcessingMode.PROCESS_CONTINUOUSLY,10);

        text.print();

        env.execute("Inside DataSource Demo");
    }
 

public static void main(String[] args) throws Exception {
    final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(parameterTool);
    env.setParallelism(1);

    DataStreamSource<String> data = env.socketTextStream("127.0.0.1", 9200);

    Pattern<String, String> pattern = Pattern.<String>begin("start")
            .where(new SimpleCondition<String>() {
                @Override
                public boolean filter(String s) throws Exception {
                    return "a".equals(s);
                }
            })
            .times(5).optional();

    CEP.pattern(data, pattern)
            .select(new PatternSelectFunction<String, String>() {
                @Override
                public String select(Map<String, List<String>> map) throws Exception {
                    log.info(map.toString());
                    return map.get("start").get(0);
                }
            }).print();
    env.execute("flink learning cep Individual Pattern Quantifier");
}
 

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //如果不指定时间的话，默认是 ProcessingTime，但是如果指定为事件事件的话，需要事件中带有时间或者添加时间水印
//        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
        DataStreamSource<String> data = env.socketTextStream(parameterTool.get(HOST_NAME), parameterTool.getInt(PORT));

        data.flatMap(new LineSplitter())
                .keyBy(0)
                .timeWindowAll(Time.seconds(10));

        env.execute("zhisheng —— flink windowAll example");
    }
 

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    DataStreamSource<String> source = env.socketTextStream("127.0.0.1", 9000);
    source.addSink(new MySink("6")).setParallelism(5);
    env.execute("xxxx");
}
 

public static void main(String[] args) throws Exception {

		// the host and the port to connect to
		final String hostname;
		final int port;
		try {
			final ParameterTool params = ParameterTool.fromArgs(args);
			hostname = params.has("hostname") ? params.get("hostname") : "localhost";
			port = params.getInt("port");
		} catch (Exception e) {
			System.err.println("No port specified. Please run 'SocketWindowWordCount " +
				"--hostname <hostname> --port <port>', where hostname (localhost by default) " +
				"and port is the address of the text server");
			System.err.println("To start a simple text server, run 'netcat -l <port>' and " +
				"type the input text into the command line");
			return;
		}

		// get the execution environment
		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// get input data by connecting to the socket
		DataStream<String> text = env.socketTextStream(hostname, port, "\n");

		// parse the data, group it, window it, and aggregate the counts
		DataStream<WordWithCount> windowCounts = text

				.flatMap(new FlatMapFunction<String, WordWithCount>() {
					@Override
					public void flatMap(String value, Collector<WordWithCount> out) {
						for (String word : value.split("\\s")) {
							out.collect(new WordWithCount(word, 1L));
						}
					}
				})

				.keyBy("word")
				.timeWindow(Time.seconds(5))

				.reduce(new ReduceFunction<WordWithCount>() {
					@Override
					public WordWithCount reduce(WordWithCount a, WordWithCount b) {
						return new WordWithCount(a.word, a.count + b.count);
					}
				});

		// print the results with a single thread, rather than in parallel
		windowCounts.print().setParallelism(1);

		env.execute("Socket Window WordCount");
	}
 

public static void socketFunction(StreamExecutionEnvironment environment) {
    DataStreamSource<String> dataStreamSource = environment.socketTextStream("localhost", 9999);
    dataStreamSource.print().setParallelism(1);

}
 

public static void main(String[] args) throws Exception {

		// the host and the port to connect to
		final String hostname;
		final int port;
		try {
			final ParameterTool params = ParameterTool.fromArgs(args);
			hostname = params.has("hostname") ? params.get("hostname") : "localhost";
			port = params.getInt("port");
		} catch (Exception e) {
			System.err.println("No port specified. Please run 'SocketWindowWordCount " +
				"--hostname <hostname> --port <port>', where hostname (localhost by default) " +
				"and port is the address of the text server");
			System.err.println("To start a simple text server, run 'netcat -l <port>' and " +
				"type the input text into the command line");
			return;
		}

		// get the execution environment
		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// get input data by connecting to the socket
		DataStream<String> text = env.socketTextStream(hostname, port, "\n");

		// parse the data, group it, window it, and aggregate the counts
		DataStream<WordWithCount> windowCounts = text

				.flatMap(new FlatMapFunction<String, WordWithCount>() {
					@Override
					public void flatMap(String value, Collector<WordWithCount> out) {
						for (String word : value.split("\\s")) {
							out.collect(new WordWithCount(word, 1L));
						}
					}
				})

				.keyBy("word")
				.timeWindow(Time.seconds(5))

				.reduce(new ReduceFunction<WordWithCount>() {
					@Override
					public WordWithCount reduce(WordWithCount a, WordWithCount b) {
						return new WordWithCount(a.word, a.count + b.count);
					}
				});

		// print the results with a single thread, rather than in parallel
		windowCounts.print().setParallelism(1);

		env.execute("Socket Window WordCount");
	}
 

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //如果不指定时间的话，默认是 ProcessingTime，但是如果指定为事件事件的话，需要事件中带有时间或者添加时间水印
//        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
        DataStreamSource<String> data = env.socketTextStream(parameterTool.get(HOST_NAME), parameterTool.getInt(PORT));

        //基于时间窗口
/*        data.flatMap(new LineSplitter())
                .keyBy(1)
                .timeWindow(Time.seconds(30))
                .sum(0)
                .print();*/

        //基于滑动时间窗口
/*        data.flatMap(new LineSplitter())
                .keyBy(1)
                .timeWindow(Time.seconds(60), Time.seconds(30))
                .sum(0)
                .print();*/


        //基于事件数量窗口
/*        data.flatMap(new LineSplitter())
                .keyBy(1)
                .countWindow(3)
                .sum(0)
                .print();*/


        //基于事件数量滑动窗口
/*        data.flatMap(new LineSplitter())
                .keyBy(1)
                .countWindow(4, 3)
                .sum(0)
                .print();*/


        //基于会话时间窗口
        data.flatMap(new LineSplitter())
                .keyBy(1)
                .window(ProcessingTimeSessionWindows.withGap(Time.seconds(5))) //表示如果 5s 内没出现数据则认为超出会话时长，然后计算这个窗口的和
                .sum(0)
                .print();

        env.execute("zhisheng —— flink window example");
    }
 

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    // 并行度1
    env.setParallelism(1);

    // 处理时间
    env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

    // 监听本地9999端口，读取字符串
    DataStream<String> socketDataStream = env.socketTextStream("localhost", 9999);

    // 所有输入的单词，如果超过10秒没有再次出现，都可以通过CountWithTimeoutFunction得到
    DataStream<Tuple2<String, Long>> timeOutWord = socketDataStream
            // 对收到的字符串用空格做分割，得到多个单词
            .flatMap(new Splitter())
            // 设置时间戳分配器，用当前时间作为时间戳
            .assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<Tuple2<String, Integer>>() {

                @Override
                public long extractTimestamp(Tuple2<String, Integer> element, long previousElementTimestamp) {
                    // 使用当前系统时间作为时间戳
                    return System.currentTimeMillis();
                }

                @Override
                public Watermark getCurrentWatermark() {
                    // 本例不需要watermark，返回null
                    return null;
                }
            })
            // 将单词作为key分区
            .keyBy(0)
            // 按单词分区后的数据，交给自定义KeyedProcessFunction处理
            .process(new CountWithTimeoutFunction());

    // 所有输入的单词，如果超过10秒没有再次出现，就在此打印出来
    timeOutWord.print();

    env.execute("ProcessFunction demo : KeyedProcessFunction");
}