Java Code Examples for org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#socketTextStream()

The following examples show how to use org.apache.flink.streaming.api.environment.StreamExecutionEnvironment#socketTextStream() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: Main.java From flink-learning with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    //参数检查
    if (args.length != 2) {
        System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>");
        return;
    }

    String hostname = args[0];
    Integer port = Integer.parseInt(args[1]);


    // set up the streaming execution environment
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //获取数据
    DataStreamSource<String> stream = env.socketTextStream(hostname, port);

    //计数
    SingleOutputStreamOperator<Tuple2<String, Integer>> sum = stream.flatMap(new LineSplitter())
            .keyBy(0)
            .sum(1);

    sum.print();

    env.execute("Java WordCount from SocketTextStream Example");
}

Example 2

Source File: StreamingJob.java From blog_demos with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<String> text = env.socketTextStream("127.0.0.1", 18081, "\n");

	DataStream<WordWithCount> windowCount = text.flatMap(new FlatMapFunction<String, WordWithCount>() {
		public void flatMap(String value, Collector<WordWithCount> out) throws Exception {
			String[] splits = value.split("\\s");
			for (String word:splits) {
				out.collect(new WordWithCount(word,1L));
			}
		}
	})
			.keyBy("word")
			.timeWindow(Time.seconds(5),Time.seconds(1))
			.sum("count");
	windowCount.print().setParallelism(1);
	env.execute("Flink Streaming Java API Skeleton");
}

Example 3

Source File: Main.java From flink-learning with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception {
    //参数检查
    if (args.length != 2) {
        System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>");
        return;
    }

    String hostname = args[0];
    Integer port = Integer.parseInt(args[1]);


    // set up the streaming execution environment
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //获取数据
    DataStreamSource<String> stream = env.socketTextStream(hostname, port);

    //计数
    SingleOutputStreamOperator<Tuple2<String, Integer>> sum = stream.flatMap(new LineSplitter())
            .keyBy(0)
            .sum(1);

    sum.print();

    env.execute("Java WordCount from SocketTextStream Example");
}

Example 4

Source File: JavaWindowApp.java From 163-bigdate-note with GNU General Public License v3.0

6 votes

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<String> dataStream = environment.socketTextStream("localhost", 9999);

    dataStream.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
        @Override
        public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
            String[] tokens = value.toLowerCase().split(",");
            for (String token : tokens) {
                if (token.length() > 0) {
                    out.collect(new Tuple2<>(token, 1));
                }
            }
        }
    }).keyBy(0)
            .timeWindow(Time.seconds(5))
            .sum(1)
            .print()
            .setParallelism(1);

    environment.execute("JavaWindowApp");
}

Example 5

Source File: SocketWindowWordCount.java From 163-bigdate-note with GNU General Public License v3.0

6 votes

public static void main(String[] args) throws Exception{
        // 创建 execution environment
        final StreamExecutionEnvironment env =
                StreamExecutionEnvironment.getExecutionEnvironment();
        // 通过连接 socket 获取输入数据,这里连接到本地 9000 端口,如果 9000 端口已被占用,请换一个端口
        DataStream<String> text = env.socketTextStream("localhost", 9000, "\n");
        // 解析数据,按 word 分组,开窗,聚合
        DataStream<Tuple2<String, Integer>> windowCounts = text.
                flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
                    @Override
                    public void flatMap(String value, Collector<Tuple2<String,
                            Integer>> out) {
                        for (String word : value.split("\\s")) {
                            out.collect(Tuple2.of(word, 1));
                        }
                    }
                })
                .keyBy(0)
                .timeWindow(Time.seconds(5))
                .sum(1);
// 将结果打印到控制台,注意这里使用的是单线程打印,而非多线程
        windowCounts.print().setParallelism(1);
        env.execute("Socket Window WordCount");
    }

Example 6

Source File: LambdaMain.java From flink-learning with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
        //参数检查
        if (args.length != 2) {
            System.err.println("USAGE:\nSocketTextStreamWordCount <hostname> <port>");
            return;
        }

        String hostname = args[0];
        Integer port = Integer.parseInt(args[1]);

        // set up the streaming execution environment
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //获取数据
        DataStreamSource<String> stream = env.socketTextStream(hostname, port);

        //计数
        stream.flatMap((s, collector) -> {
            for (String token : s.toLowerCase().split("\\W+")) {
                if (token.length() > 0) {
                    collector.collect(new Tuple2<String, Integer>(token, 1));
                }
            }
        })
//                .returns((TypeInformation) TupleTypeInfo.getBasicTupleTypeInfo(String.class, Integer.class))
                .keyBy(0)
                .sum(1)
                .print();

        env.execute("Java WordCount from SocketTextStream Example");
    }

Example 7

Source File: JavaWindowReduceApp.java From 163-bigdate-note with GNU General Public License v3.0

5 votes

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<String> dataStream = environment.socketTextStream("localhost", 9999);

    dataStream.flatMap(new FlatMapFunction<String, Tuple2<Integer, Integer>>() {
        @Override
        public void flatMap(String value, Collector<Tuple2<Integer, Integer>> out) throws Exception {
            String[] tokens = value.toLowerCase().split(",");
            for (String token : tokens) {
                if (token.length() > 0) {
                    out.collect(new Tuple2<>(1, Integer.parseInt(token)));
                }
            }
        }
    }).keyBy(0)
            .timeWindow(Time.seconds(5))
            .reduce(new ReduceFunction<Tuple2<Integer, Integer>>() {
                @Override
                public Tuple2<Integer, Integer> reduce(Tuple2<Integer, Integer> value1, Tuple2<Integer, Integer> value2) throws Exception {
                    System.out.println(value1 + "...." + value2);
                    return new Tuple2<>(value1.f0, value1.f1 + value2.f1);
                }
            })
            .print()
            .setParallelism(1);

    environment.execute("JavaWindowReduceApp");
}

Example 8

Source File: Main2.java From flink-learning with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    DataStreamSource<String> source = env.socketTextStream("127.0.0.1", 9000);
    source.addSink(new MySink("6")).setParallelism(5);
    env.execute("xxxx");
}

Example 9

Source File: StreamingWCJava02App.java From 163-bigdate-note with GNU General Public License v3.0

5 votes

public static void main(String[] args) throws Exception {
    //获取参数
    int port = 0;
    try {
        ParameterTool tool = ParameterTool.fromArgs(args);
        port = tool.getInt("port");
    } catch (Exception e) {
        System.err.println("Port undefined, use 9999.");
        port = 9999;
    }
    
    //获取执行环境
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //读取数据
    DataStreamSource<String> dataStreamSource = env.socketTextStream("localhost", 9999);

    //执行转换操作
    dataStreamSource.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
        @Override
        public void flatMap(String value, Collector<Tuple2<String, Integer>> collector) throws Exception {
            String[] tokens = value.toLowerCase().split(",");
            for (String token : tokens) {
                if (token.length() > 0) {
                    collector.collect(new Tuple2<String, Integer>(token, 1));
                }
            }
        }
    }).keyBy(0)
    .timeWindow(Time.seconds(5))
    .sum(1)
    .print()
    .setParallelism(1);

    env.execute("StreamingWCJavaApp");
}

Example 10

Source File: IndividualPatternQuantifier.java From flink-learning with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
    final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(parameterTool);
    env.setParallelism(1);

    DataStreamSource<String> data = env.socketTextStream("127.0.0.1", 9200);

    Pattern<String, String> pattern = Pattern.<String>begin("start")
            .where(new SimpleCondition<String>() {
                @Override
                public boolean filter(String s) throws Exception {
                    return "a".equals(s);
                }
            })
            .times(5).optional();

    CEP.pattern(data, pattern)
            .select(new PatternSelectFunction<String, String>() {
                @Override
                public String select(Map<String, List<String>> map) throws Exception {
                    log.info(map.toString());
                    return map.get("start").get(0);
                }
            }).print();
    env.execute("flink learning cep Individual Pattern Quantifier");
}

Example 11

Source File: Main4.java From flink-learning with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    env.setParallelism(2);
    DataStreamSource<String> data = env.socketTextStream("localhost", 9001);

    data.map(new MapFunction<String, Tuple2<String, Long>>() {
        @Override
        public Tuple2<String, Long> map(String s) throws Exception {
            String[] split = s.split(",");
            return new Tuple2<>(split[0], Long.valueOf(split[1]));
        }
    }).assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<Tuple2<String, Long>>() {
        private long currentTimestamp;

        @Nullable
        @Override
        public Watermark getCurrentWatermark() {
            return new Watermark(currentTimestamp);
        }

        @Override
        public long extractTimestamp(Tuple2<String, Long> tuple2, long l) {
            long timestamp = tuple2.f1;
            currentTimestamp = Math.max(timestamp, currentTimestamp);
            return timestamp;
        }
    }).keyBy(0)
            .window(EventTimeSessionWindows.withGap(Time.minutes(5)))
            .sum(1)
            .print("session ");
    System.out.println(env.getExecutionPlan());
    env.execute();
}

Example 12

Source File: InsideDataSource.java From flink-simple-tutorial with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();


        // 添加数组作为数据输入源
        String[] elementInput = new String[]{"hello Flink", "Second Line"};
        DataStream<String> text = env.fromElements(elementInput);

        // 添加List集合作为数据输入源
        List<String> collectionInput = new ArrayList<>();
        collectionInput.add("hello Flink");
        DataStream<String> text2 = env.fromCollection(collectionInput);

        // 添加Socket作为数据输入源
        // 4个参数 -> (hostname:Ip地址, port:端口, delimiter:分隔符, maxRetry:最大重试次数)
        DataStream<String> text3 = env.socketTextStream("localhost", 9999, "\n", 4);


        // 添加文件源
        // 直接读取文本文件
        DataStream<String> text4 = env.readTextFile("/opt/history.log");
        // 指定 CsvInputFormat, 监控csv文件(两种模式), 时间间隔是10ms
        DataStream<String> text5 = env.readFile(new CsvInputFormat<String>(new Path("/opt/history.csv")) {
            @Override
            protected String fillRecord(String s, Object[] objects) {
                return null;
            }
        },"/opt/history.csv", FileProcessingMode.PROCESS_CONTINUOUSLY,10);

        text.print();

        env.execute("Inside DataSource Demo");
    }

Example 13

Source File: IndividualPatternQuantifier.java From flink-learning with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
    final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(parameterTool);
    env.setParallelism(1);

    DataStreamSource<String> data = env.socketTextStream("127.0.0.1", 9200);

    Pattern<String, String> pattern = Pattern.<String>begin("start")
            .where(new SimpleCondition<String>() {
                @Override
                public boolean filter(String s) throws Exception {
                    return "a".equals(s);
                }
            })
            .times(5).optional();

    CEP.pattern(data, pattern)
            .select(new PatternSelectFunction<String, String>() {
                @Override
                public String select(Map<String, List<String>> map) throws Exception {
                    log.info(map.toString());
                    return map.get("start").get(0);
                }
            }).print();
    env.execute("flink learning cep Individual Pattern Quantifier");
}

Example 14

Source File: WindowAll.java From flink-learning with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //如果不指定时间的话，默认是 ProcessingTime，但是如果指定为事件事件的话，需要事件中带有时间或者添加时间水印
//        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
        DataStreamSource<String> data = env.socketTextStream(parameterTool.get(HOST_NAME), parameterTool.getInt(PORT));

        data.flatMap(new LineSplitter())
                .keyBy(0)
                .timeWindowAll(Time.seconds(10));

        env.execute("zhisheng —— flink windowAll example");
    }

Example 15

Source File: Main2.java From flink-learning with Apache License 2.0

5 votes

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    DataStreamSource<String> source = env.socketTextStream("127.0.0.1", 9000);
    source.addSink(new MySink("6")).setParallelism(5);
    env.execute("xxxx");
}

Example 16

Source File: SocketWindowWordCount.java From flink with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {

		// the host and the port to connect to
		final String hostname;
		final int port;
		try {
			final ParameterTool params = ParameterTool.fromArgs(args);
			hostname = params.has("hostname") ? params.get("hostname") : "localhost";
			port = params.getInt("port");
		} catch (Exception e) {
			System.err.println("No port specified. Please run 'SocketWindowWordCount " +
				"--hostname <hostname> --port <port>', where hostname (localhost by default) " +
				"and port is the address of the text server");
			System.err.println("To start a simple text server, run 'netcat -l <port>' and " +
				"type the input text into the command line");
			return;
		}

		// get the execution environment
		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// get input data by connecting to the socket
		DataStream<String> text = env.socketTextStream(hostname, port, "\n");

		// parse the data, group it, window it, and aggregate the counts
		DataStream<WordWithCount> windowCounts = text

				.flatMap(new FlatMapFunction<String, WordWithCount>() {
					@Override
					public void flatMap(String value, Collector<WordWithCount> out) {
						for (String word : value.split("\\s")) {
							out.collect(new WordWithCount(word, 1L));
						}
					}
				})

				.keyBy("word")
				.timeWindow(Time.seconds(5))

				.reduce(new ReduceFunction<WordWithCount>() {
					@Override
					public WordWithCount reduce(WordWithCount a, WordWithCount b) {
						return new WordWithCount(a.word, a.count + b.count);
					}
				});

		// print the results with a single thread, rather than in parallel
		windowCounts.print().setParallelism(1);

		env.execute("Socket Window WordCount");
	}

Example 17

Source File: JavaDataStreamSourceApp.java From 163-bigdate-note with GNU General Public License v3.0

4 votes

public static void socketFunction(StreamExecutionEnvironment environment) {
    DataStreamSource<String> dataStreamSource = environment.socketTextStream("localhost", 9999);
    dataStreamSource.print().setParallelism(1);

}

Example 18

Source File: SocketWindowWordCount.java From flink with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {

		// the host and the port to connect to
		final String hostname;
		final int port;
		try {
			final ParameterTool params = ParameterTool.fromArgs(args);
			hostname = params.has("hostname") ? params.get("hostname") : "localhost";
			port = params.getInt("port");
		} catch (Exception e) {
			System.err.println("No port specified. Please run 'SocketWindowWordCount " +
				"--hostname <hostname> --port <port>', where hostname (localhost by default) " +
				"and port is the address of the text server");
			System.err.println("To start a simple text server, run 'netcat -l <port>' and " +
				"type the input text into the command line");
			return;
		}

		// get the execution environment
		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		// get input data by connecting to the socket
		DataStream<String> text = env.socketTextStream(hostname, port, "\n");

		// parse the data, group it, window it, and aggregate the counts
		DataStream<WordWithCount> windowCounts = text

				.flatMap(new FlatMapFunction<String, WordWithCount>() {
					@Override
					public void flatMap(String value, Collector<WordWithCount> out) {
						for (String word : value.split("\\s")) {
							out.collect(new WordWithCount(word, 1L));
						}
					}
				})

				.keyBy("word")
				.timeWindow(Time.seconds(5))

				.reduce(new ReduceFunction<WordWithCount>() {
					@Override
					public WordWithCount reduce(WordWithCount a, WordWithCount b) {
						return new WordWithCount(a.word, a.count + b.count);
					}
				});

		// print the results with a single thread, rather than in parallel
		windowCounts.print().setParallelism(1);

		env.execute("Socket Window WordCount");
	}

Example 19

Source File: Main.java From flink-learning with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        //如果不指定时间的话，默认是 ProcessingTime，但是如果指定为事件事件的话，需要事件中带有时间或者添加时间水印
//        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
        DataStreamSource<String> data = env.socketTextStream(parameterTool.get(HOST_NAME), parameterTool.getInt(PORT));

        //基于时间窗口
/*        data.flatMap(new LineSplitter())
                .keyBy(1)
                .timeWindow(Time.seconds(30))
                .sum(0)
                .print();*/

        //基于滑动时间窗口
/*        data.flatMap(new LineSplitter())
                .keyBy(1)
                .timeWindow(Time.seconds(60), Time.seconds(30))
                .sum(0)
                .print();*/


        //基于事件数量窗口
/*        data.flatMap(new LineSplitter())
                .keyBy(1)
                .countWindow(3)
                .sum(0)
                .print();*/


        //基于事件数量滑动窗口
/*        data.flatMap(new LineSplitter())
                .keyBy(1)
                .countWindow(4, 3)
                .sum(0)
                .print();*/


        //基于会话时间窗口
        data.flatMap(new LineSplitter())
                .keyBy(1)
                .window(ProcessingTimeSessionWindows.withGap(Time.seconds(5))) //表示如果 5s 内没出现数据则认为超出会话时长，然后计算这个窗口的和
                .sum(0)
                .print();

        env.execute("zhisheng —— flink window example");
    }

Example 20

Source File: ProcessTime.java From blog_demos with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    // 并行度1
    env.setParallelism(1);

    // 处理时间
    env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

    // 监听本地9999端口，读取字符串
    DataStream<String> socketDataStream = env.socketTextStream("localhost", 9999);

    // 所有输入的单词，如果超过10秒没有再次出现，都可以通过CountWithTimeoutFunction得到
    DataStream<Tuple2<String, Long>> timeOutWord = socketDataStream
            // 对收到的字符串用空格做分割，得到多个单词
            .flatMap(new Splitter())
            // 设置时间戳分配器，用当前时间作为时间戳
            .assignTimestampsAndWatermarks(new AssignerWithPeriodicWatermarks<Tuple2<String, Integer>>() {

                @Override
                public long extractTimestamp(Tuple2<String, Integer> element, long previousElementTimestamp) {
                    // 使用当前系统时间作为时间戳
                    return System.currentTimeMillis();
                }

                @Override
                public Watermark getCurrentWatermark() {
                    // 本例不需要watermark，返回null
                    return null;
                }
            })
            // 将单词作为key分区
            .keyBy(0)
            // 按单词分区后的数据，交给自定义KeyedProcessFunction处理
            .process(new CountWithTimeoutFunction());

    // 所有输入的单词，如果超过10秒没有再次出现，就在此打印出来
    timeOutWord.print();

    env.execute("ProcessFunction demo : KeyedProcessFunction");
}