org.apache.flink.streaming.api.functions.sink.PrintSinkFunction Java Examples

public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	setupEnvironment(env, pt);

	setBackendWithCustomTTLTimeProvider(env);

	TtlTestConfig config = TtlTestConfig.fromArgs(pt);
	StateTtlConfig ttlConfig = StateTtlConfig.newBuilder(config.ttl)
		.cleanupFullSnapshot()
		.cleanupInBackground()
		.build();

	env
		.addSource(new TtlStateUpdateSource(config.keySpace, config.sleepAfterElements, config.sleepTime))
		.name("TtlStateUpdateSource")
		.keyBy(TtlStateUpdate::getKey)
		.flatMap(new TtlVerifyUpdateFunction(ttlConfig, config.reportStatAfterUpdatesNum))
		.name("TtlVerifyUpdateFunction")
		.addSink(new PrintSinkFunction<>())
		.name("PrintFailedVerifications");

	env.execute("State TTL test job");
}
 

public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	setupEnvironment(env, pt);

	setBackendWithCustomTTLTimeProvider(env);

	TtlTestConfig config = TtlTestConfig.fromArgs(pt);
	StateTtlConfig ttlConfig = StateTtlConfig.newBuilder(config.ttl)
		.cleanupFullSnapshot()
		.cleanupInBackground()
		.build();

	env
		.addSource(new TtlStateUpdateSource(config.keySpace, config.sleepAfterElements, config.sleepTime))
		.name("TtlStateUpdateSource")
		.keyBy(TtlStateUpdate::getKey)
		.flatMap(new TtlVerifyUpdateFunction(ttlConfig, config.reportStatAfterUpdatesNum))
		.name("TtlVerifyUpdateFunction")
		.addSink(new PrintSinkFunction<>())
		.name("PrintFailedVerifications");

	env.execute("State TTL test job");
}
 

public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	setupEnvironment(env, pt);

	setBackendWithCustomTTLTimeProvider(env);

	TtlTestConfig config = TtlTestConfig.fromArgs(pt);
	StateTtlConfig ttlConfig = StateTtlConfig.newBuilder(config.ttl)
		.cleanupFullSnapshot()
		.build();

	env
		.addSource(new TtlStateUpdateSource(config.keySpace, config.sleepAfterElements, config.sleepTime))
		.name("TtlStateUpdateSource")
		.keyBy(TtlStateUpdate::getKey)
		.flatMap(new TtlVerifyUpdateFunction(ttlConfig, config.reportStatAfterUpdatesNum))
		.name("TtlVerifyUpdateFunction")
		.addSink(new PrintSinkFunction<>())
		.name("PrintFailedVerifications");

	env.execute("State TTL test job");
}
 

private static void executeUpgradedVariant(StreamExecutionEnvironment env, ParameterTool pt) throws Exception {
	KeyedStream<UpgradedEvent, Integer> source = env.addSource(createEventSource(pt))
		.name("EventSource")
		.uid("EventSource")
		.assignTimestampsAndWatermarks(createTimestampExtractor(pt))
		.map(new UpgradeEvent())
		.keyBy(UpgradedEvent::getKey);

	List<TypeSerializer<ComplexPayload>> stateSer =
		Collections.singletonList(new KryoSerializer<>(ComplexPayload.class, env.getConfig()));

	KeyedStream<UpgradedEvent, Integer> afterStatefulOperations =
		applyUpgradedStatefulOperations(source, stateSer, Collections.emptyList());

	afterStatefulOperations
		.map(new DowngradeEvent())
		.keyBy(Event::getKey)
		.flatMap(createSemanticsCheckMapper(pt))
		.name("SemanticsCheckMapper")
		.addSink(new PrintSinkFunction<>());

	env.execute("General purpose test job");
}
 

private static void executeOriginalVariant(StreamExecutionEnvironment env, ParameterTool pt) throws Exception {
	KeyedStream<Event, Integer> source = env.addSource(createEventSource(pt))
		.name("EventSource")
		.uid("EventSource")
		.assignTimestampsAndWatermarks(createTimestampExtractor(pt))
		.keyBy(Event::getKey);

	List<TypeSerializer<ComplexPayload>> stateSer =
		Collections.singletonList(new KryoSerializer<>(ComplexPayload.class, env.getConfig()));

	KeyedStream<Event, Integer> afterStatefulOperations =
		applyOriginalStatefulOperations(source, stateSer, Collections.emptyList());

	afterStatefulOperations
		.flatMap(createSemanticsCheckMapper(pt))
		.name("SemanticsCheckMapper")
		.addSink(new PrintSinkFunction<>());

	env.execute("General purpose test job");
}
 

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //并行度为1
    env.setParallelism(1);

    //创建一个List，里面有两个Tuple2元素
    List<Tuple2<String, Integer>> list = new ArrayList<>();
    list.add(new Tuple2("aaa", 1));

    //统计每个单词的数量
    env.fromCollection(list)
            .keyBy(0)
            .sum(1)
            .addSink(new PrintSinkFunction<>());

    env.execute("sink demo : print");
}
 

@Test
public void testPrintSinkStdOut() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>();
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));

	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.out", printSink.toString());
	assertEquals("hello world!" + line, arrayOutputStream.toString());
	printSink.close();
}
 

@Test
public void testPrintSinkStdErr() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>(true);
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.err", printSink.toString());
	assertEquals("hello world!" + line, arrayErrorStream.toString());
	printSink.close();
}
 

@Before
public void setUp() throws Exception {
	jobExecuteThread = new CheckedThread() {

		@Override
		public void go() throws Exception {
			StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
			env.addSource(new SourceFunction<String>() {

				@Override
				public void run(SourceContext<String> ctx) throws Exception {
					sync.block();
				}

				@Override
				public void cancel() {
					sync.releaseBlocker();
				}

			}).addSink(new PrintSinkFunction());

			env.execute();
		}

	};

	jobExecuteThread.start();
	sync.awaitBlocker();
}
 

@Test
public void testPrintSinkWithPrefix() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>();
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 2, 1));
	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.out", printSink.toString());
	assertEquals("2> hello world!" + line, arrayOutputStream.toString());
	printSink.close();
}
 

@Test
public void testPrintSinkWithIdentifierAndPrefix() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>("mySink", false);
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 2, 1));
	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.out", printSink.toString());
	assertEquals("mySink:2> hello world!" + line, arrayOutputStream.toString());
	printSink.close();
}
 

@Test
public void testPrintSinkWithIdentifierButNoPrefix() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>("mySink", false);
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.out", printSink.toString());
	assertEquals("mySink> hello world!" + line, arrayOutputStream.toString());
	printSink.close();
}
 

@Before
public void setUp() throws Exception {
	jobExecuteThread = new CheckedThread() {

		@Override
		public void go() throws Exception {
			StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
			env.addSource(new SourceFunction<String>() {

				@Override
				public void run(SourceContext<String> ctx) throws Exception {
					sync.block();
				}

				@Override
				public void cancel() {
					sync.releaseBlocker();
				}

			}).addSink(new PrintSinkFunction());

			env.execute();
		}

	};

	jobExecuteThread.start();
	sync.awaitBlocker();
}
 

@Test
public void testPrintSinkStdOut() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>();
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));

	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.out", printSink.toString());
	assertEquals("hello world!" + line, arrayOutputStream.toString());
	printSink.close();
}
 

@Test
public void testPrintSinkStdErr() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>(true);
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.err", printSink.toString());
	assertEquals("hello world!" + line, arrayErrorStream.toString());
	printSink.close();
}
 

@Test
public void testPrintSinkWithPrefix() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>();
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 2, 1));
	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.out", printSink.toString());
	assertEquals("2> hello world!" + line, arrayOutputStream.toString());
	printSink.close();
}
 

@Test
public void testPrintSinkWithIdentifierAndPrefix() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>("mySink", false);
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 2, 1));
	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.out", printSink.toString());
	assertEquals("mySink:2> hello world!" + line, arrayOutputStream.toString());
	printSink.close();
}
 

@Test
public void testPrintSinkWithIdentifierButNoPrefix() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>("mySink", false);
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.out", printSink.toString());
	assertEquals("mySink> hello world!" + line, arrayOutputStream.toString());
	printSink.close();
}
 

public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	setupEnvironment(env, pt);

	KeyedStream<Event, Integer> source = env.addSource(createEventSource(pt))
		.name("EventSource")
		.uid("EventSource")
		.assignTimestampsAndWatermarks(createTimestampExtractor(pt))
		.keyBy(Event::getKey);

	List<TypeSerializer<ComplexPayload>> stateSer =
		Collections.singletonList(new KryoSerializer<>(ComplexPayload.class, env.getConfig()));

	KeyedStream<Event, Integer> afterStatefulOperations = isOriginalJobVariant(pt) ?
		applyOriginalStatefulOperations(source, stateSer, Collections.emptyList()) :
		applyUpgradedStatefulOperations(source, stateSer, Collections.emptyList());

	afterStatefulOperations
		.flatMap(createSemanticsCheckMapper(pt))
		.name("SemanticsCheckMapper")
		.addSink(new PrintSinkFunction<>());

	env.execute("General purpose test job");
}
 

@Test
public void testPrintSinkStdOut() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>();
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));

	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.out", printSink.toString());
	assertEquals("hello world!" + line, arrayOutputStream.toString());
	printSink.close();
}
 

@Test
public void testPrintSinkStdErr() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>(true);
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.err", printSink.toString());
	assertEquals("hello world!" + line, arrayErrorStream.toString());
	printSink.close();
}
 

public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	setupEnvironment(env, pt);

	KeyedStream<Event, Integer> source = env.addSource(createEventSource(pt))
		.name("EventSource")
		.uid("EventSource")
		.assignTimestampsAndWatermarks(createTimestampExtractor(pt))
		.keyBy(Event::getKey);

	List<TypeSerializer<ComplexPayload>> stateSer =
		Collections.singletonList(new KryoSerializer<>(ComplexPayload.class, env.getConfig()));

	KeyedStream<Event, Integer> afterStatefulOperations = isOriginalJobVariant(pt) ?
		applyOriginalStatefulOperations(source, stateSer, Collections.emptyList()) :
		applyUpgradedStatefulOperations(source, stateSer, Collections.emptyList());

	afterStatefulOperations
		.flatMap(createSemanticsCheckMapper(pt))
		.name("SemanticsCheckMapper")
		.addSink(new PrintSinkFunction<>());

	env.execute("General purpose test job");
}
 

@Before
public void setUp() throws Exception {
	jobExecuteThread = new CheckedThread() {

		@Override
		public void go() throws Exception {
			StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
			env.addSource(new SourceFunction<String>() {

				@Override
				public void run(SourceContext<String> ctx) throws Exception {
					sync.block();
				}

				@Override
				public void cancel() {
					sync.releaseBlocker();
				}

			}).addSink(new PrintSinkFunction());

			env.execute();
		}

	};

	jobExecuteThread.start();
	sync.awaitBlocker();
}
 

@Test
public void testPrintSinkWithIdentifierButNoPrefix() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>("mySink", false);
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 1, 0));
	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.out", printSink.toString());
	assertEquals("mySink> hello world!" + line, arrayOutputStream.toString());
	printSink.close();
}
 

@Test
public void testPrintSinkWithPrefix() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>();
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 2, 1));
	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.out", printSink.toString());
	assertEquals("2> hello world!" + line, arrayOutputStream.toString());
	printSink.close();
}
 

@Test
public void testPrintSinkWithIdentifierAndPrefix() throws Exception {
	PrintSinkFunction<String> printSink = new PrintSinkFunction<>("mySink", false);
	printSink.setRuntimeContext(new MockStreamingRuntimeContext(false, 2, 1));
	printSink.open(new Configuration());

	printSink.invoke("hello world!", SinkContextUtil.forTimestamp(0));

	assertEquals("Print to System.out", printSink.toString());
	assertEquals("mySink:2> hello world!" + line, arrayOutputStream.toString());
	printSink.close();
}
 

public static void main(String[] args) throws Exception {

		final ParameterTool pt = ParameterTool.fromArgs(args);

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		env.setParallelism(pt.getInt("parallelism", 1));
		env.setMaxParallelism(pt.getInt("maxParallelism", pt.getInt("parallelism", 1)));
		env.enableCheckpointing(pt.getInt("checkpointInterval", 1000));
		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, pt.getInt("restartDelay", 0)));
		if (pt.getBoolean("externalizedCheckpoints", false)) {
			env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
		}

		String stateBackend = pt.get("stateBackend", "file");
		String checkpointDir = pt.getRequired("checkpointDir");

		boolean killJvmOnFail = pt.getBoolean("killJvmOnFail", false);

		if ("file".equals(stateBackend)) {
			boolean asyncCheckpoints = pt.getBoolean("asyncCheckpoints", true);
			env.setStateBackend(new FsStateBackend(checkpointDir, asyncCheckpoints));
		} else if ("rocks".equals(stateBackend)) {
			boolean incrementalCheckpoints = pt.getBoolean("incrementalCheckpoints", false);
			env.setStateBackend(new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
		} else {
			throw new IllegalArgumentException("Unknown backend: " + stateBackend);
		}

		// make parameters available in the web interface
		env.getConfig().setGlobalJobParameters(pt);

		// delay to throttle down the production of the source
		long delay = pt.getLong("delay", 0L);

		// the maximum number of attempts, before the job finishes with success
		int maxAttempts = pt.getInt("maxAttempts", 3);

		// size of one artificial value
		int valueSize = pt.getInt("valueSize", 10);

		env.addSource(new RandomLongSource(maxAttempts, delay))
			.keyBy((KeySelector<Long, Long>) aLong -> aLong)
			.flatMap(new StateCreatingFlatMap(valueSize, killJvmOnFail))
			.addSink(new PrintSinkFunction<>());

		env.execute("Sticky Allocation And Local Recovery Test");
	}
 

public static void main(String[] args) throws Exception {

		final ParameterTool pt = ParameterTool.fromArgs(args);

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		env.setParallelism(pt.getInt("parallelism", 1));
		env.setMaxParallelism(pt.getInt("maxParallelism", pt.getInt("parallelism", 1)));
		env.enableCheckpointing(pt.getInt("checkpointInterval", 1000));
		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, pt.getInt("restartDelay", 0)));
		if (pt.getBoolean("externalizedCheckpoints", false)) {
			env.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
		}

		String stateBackend = pt.get("stateBackend", "file");
		String checkpointDir = pt.getRequired("checkpointDir");

		boolean killJvmOnFail = pt.getBoolean("killJvmOnFail", false);

		if ("file".equals(stateBackend)) {
			boolean asyncCheckpoints = pt.getBoolean("asyncCheckpoints", true);
			env.setStateBackend(new FsStateBackend(checkpointDir, asyncCheckpoints));
		} else if ("rocks".equals(stateBackend)) {
			boolean incrementalCheckpoints = pt.getBoolean("incrementalCheckpoints", false);
			env.setStateBackend(new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
		} else {
			throw new IllegalArgumentException("Unknown backend: " + stateBackend);
		}

		// make parameters available in the web interface
		env.getConfig().setGlobalJobParameters(pt);

		// delay to throttle down the production of the source
		long delay = pt.getLong("delay", 0L);

		// the maximum number of attempts, before the job finishes with success
		int maxAttempts = pt.getInt("maxAttempts", 3);

		// size of one artificial value
		int valueSize = pt.getInt("valueSize", 10);

		env.addSource(new RandomLongSource(maxAttempts, delay))
			.keyBy((KeySelector<Long, Long>) aLong -> aLong)
			.flatMap(new StateCreatingFlatMap(valueSize, killJvmOnFail))
			.addSink(new PrintSinkFunction<>());

		env.execute("Sticky Allocation And Local Recovery Test");
	}
 

public static void main(String[] args) throws Exception {
	TestableStreamingJob job = new TestableStreamingJob(new RandomLongSource(), new PrintSinkFunction<>());
	job.execute();
}
 

public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    //设置并行度
    env.setParallelism(1);

    //连接kafka用到的属性对象
    Properties properties = new Properties();
    //broker地址
    properties.setProperty("bootstrap.servers", "192.168.50.43:9092");
    //zookeeper地址
    properties.setProperty("zookeeper.connect", "192.168.50.43:2181");
    //消费者的groupId
    properties.setProperty("group.id", "flink-connector");
    //实例化Consumer类
    FlinkKafkaConsumer<String> flinkKafkaConsumer = new FlinkKafkaConsumer<>(
            "test001",
            new SimpleStringSchema(),
            properties
    );

    //指定从最新位置开始消费，相当于放弃历史消息
    flinkKafkaConsumer.setStartFromLatest();

    //通过addSource方法得到DataSource
    DataStream<String> dataStream = env.addSource(flinkKafkaConsumer);

    DataStream<Tuple2<String, Long>> result = dataStream
            .flatMap(new FlatMapFunction<String, Tuple2<String, Long>>() {
                         @Override
                         public void flatMap(String value, Collector<Tuple2<String, Long>> out) {
                             String[] words = value.toLowerCase().split("\\s");

                             for (String word : words) {
                                 //cassandra的表中，每个word都是主键，因此不能为空
                                 if (!word.isEmpty()) {
                                     out.collect(new Tuple2<String, Long>(word, 1L));
                                 }
                             }
                         }
                     }
            )
            .keyBy(0)
            .timeWindow(Time.seconds(5))
            .sum(1);

    result.addSink(new PrintSinkFunction<>())
            .name("print Sink")
            .disableChaining();

    CassandraSink.addSink(result)
            .setQuery("INSERT INTO example.wordcount(word, count) values (?, ?);")
            .setHost("192.168.133.168")
            .build()
            .name("cassandra Sink")
            .disableChaining();

    env.execute("kafka-2.4 source, cassandra-3.11.6 sink, tuple2");
}