org.apache.flink.streaming.api.environment.StreamExecutionEnvironment Java Examples

The following examples show how to use org.apache.flink.streaming.api.environment.StreamExecutionEnvironment. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: Flink-CEPplus   Author: ljygz   File: ChainLengthIncreaseTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void createRestoredJob(StreamExecutionEnvironment env) {
	/**
	 * Original job: Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3)
	 * Modified job: Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3 -> StatefulMap4)
	 */
	DataStream<Integer> source = createSource(env, ExecutionMode.RESTORE);

	SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.RESTORE, source);
	first.startNewChain();

	SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.RESTORE, first);
	second.startNewChain();

	SingleOutputStreamOperator<Integer> stateless = createStatelessMap(second);

	SingleOutputStreamOperator<Integer> stateless2 = createStatelessMap(stateless);

	SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.RESTORE, stateless2);
}
 
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: CassandraTupleWriteAheadSinkExample.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(1000);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
	env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));

	CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
		.setQuery("INSERT INTO example.values (id, counter) values (?, ?);")
		.enableWriteAheadLog()
		.setClusterBuilder(new ClusterBuilder() {

			private static final long serialVersionUID = 2793938419775311824L;

			@Override
			public Cluster buildCluster(Cluster.Builder builder) {
				return builder.addContactPoint("127.0.0.1").build();
			}
		})
		.build();

	sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");

	env.execute();
}
 
Example #3
Source Project: gelly-streaming   Author: vasia   File: TestFilterEdges.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWithEmptyFilter() throws Exception {
	/*
	 * Test filterEdges() with a filter that constantly returns true
     */
       final String resultPath = getTempDirPath("result");
       final String expectedResult = "1,2,12\n" +
               "1,3,13\n" +
               "2,3,23\n" +
               "3,4,34\n" +
               "3,5,35\n" +
               "4,5,45\n" +
               "5,1,51\n";

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	GraphStream<Long, NullValue, Long> graph = new SimpleEdgeStream<>(GraphStreamTestUtils.getLongLongEdgeDataStream(env), env);
	graph.filterEdges(new EmptyFilter())
               .getEdges().writeAsCsv(resultPath, FileSystem.WriteMode.OVERWRITE);
	env.execute();

       compareResultsByLinesInMemory(expectedResult, resultPath);
   }
 
Example #4
Source Project: flink   Author: flink-tpc-ds   File: IntervalJoinITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = UnsupportedTimeCharacteristicException.class)
public void testExecutionFailsInProcessingTime() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
	env.setParallelism(1);

	DataStream<Tuple2<String, Integer>> streamOne = env.fromElements(Tuple2.of("1", 1));
	DataStream<Tuple2<String, Integer>> streamTwo = env.fromElements(Tuple2.of("1", 1));

	streamOne.keyBy(new Tuple2KeyExtractor())
		.intervalJoin(streamTwo.keyBy(new Tuple2KeyExtractor()))
		.between(Time.milliseconds(0), Time.milliseconds(0))
		.process(new ProcessJoinFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String>() {
			@Override
			public void processElement(Tuple2<String, Integer> left,
				Tuple2<String, Integer> right, Context ctx,
				Collector<String> out) throws Exception {
				out.collect(left + ":" + right);
			}
		});
}
 
Example #5
Source Project: Flink-CEPplus   Author: ljygz   File: StreamingJobGraphGeneratorNodeHashTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a changed operator name does not affect the hash.
 */
@Test
public void testChangedOperatorName() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.addSource(new NoOpSourceFunction(), "A").map(new NoOpMapFunction());
	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	JobVertexID expected = jobGraph.getVerticesAsArray()[0].getID();

	env = StreamExecutionEnvironment.createLocalEnvironment();
	env.addSource(new NoOpSourceFunction(), "B").map(new NoOpMapFunction());
	jobGraph = env.getStreamGraph().getJobGraph();

	JobVertexID actual = jobGraph.getVerticesAsArray()[0].getID();

	assertEquals(expected, actual);
}
 
Example #6
Source Project: flink   Author: flink-tpc-ds   File: WindowTranslationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testReduceEventTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(new TupleKeySelector())
			.window(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.reduce(new DummyReducer());

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ReducingStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #7
Source Project: flink-simple-tutorial   Author: CheckChe0803   File: AggregateFunctionDemo.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        // 构建输入数据
        List<Tuple2<String, Long>> data = new ArrayList<>();
        Tuple2<String, Long> a = new Tuple2<>("first event", 1L);
        Tuple2<String, Long> b = new Tuple2<>("second event", 2L);
        data.add(a);
        data.add(b);
        DataStreamSource<Tuple2<String, Long>> input = env.fromCollection(data);


        input.keyBy(x -> x.f1)
                .timeWindow(Time.seconds(10), Time.seconds(1))
                // 自定义一个AggregateFunciton, 将相同标号 f1 的数据的 f0字符串字段合并在一起
                // ("hello", 1L) + ("world", 1L) = ("hello world", 1L)
                .aggregate(new MyAggregateFunction());

        env.execute();
    }
 
Example #8
Source Project: solace-integration-guides   Author: SolaceLabs   File: BasicTopicStreamingSample.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(4);

        final Hashtable<String, String> jmsEnv = new Hashtable<>();
        jmsEnv.put(InitialContext.INITIAL_CONTEXT_FACTORY, "com.solacesystems.jndi.SolJNDIInitialContextFactory");
        jmsEnv.put(InitialContext.PROVIDER_URL, "smf://192.168.56.101");
        jmsEnv.put(Context.SECURITY_PRINCIPAL, "[email protected]_vpn");
        jmsEnv.put(Context.SECURITY_CREDENTIALS, "password");

        env.addSource(new JMSTopicSource<String>(jmsEnv,
                "flink_cf",
                "flink/topic",
                new JMSTextTranslator()))
                .print();

        env.execute();
    }
 
Example #9
Source Project: flink   Author: apache   File: SavepointTestBase.java    License: Apache License 2.0 6 votes vote down vote up
public <T> String takeSavepoint(Collection<T> data, Function<SourceFunction<T>, StreamExecutionEnvironment> jobGraphFactory) throws Exception {

		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().disableClosureCleaner();

		WaitingSource<T> waitingSource = createSource(data);

		JobGraph jobGraph = jobGraphFactory.apply(waitingSource).getStreamGraph().getJobGraph();
		JobID jobId = jobGraph.getJobID();

		ClusterClient<?> client = miniClusterResource.getClusterClient();

		try {
			JobSubmissionResult result = ClientUtils.submitJob(client, jobGraph);

			return CompletableFuture
				.runAsync(waitingSource::awaitSource)
				.thenCompose(ignore -> triggerSavepoint(client, result.getJobID()))
				.get(5, TimeUnit.MINUTES);
		} catch (Exception e) {
			throw new RuntimeException("Failed to take savepoint", e);
		} finally {
			client.cancel(jobId);
		}
	}
 
Example #10
Source Project: 163-bigdate-note   Author: jiaoqiyuan   File: SocketWindowWordCount.java    License: GNU General Public License v3.0 6 votes vote down vote up
public static void main(String[] args) throws Exception{
        // 创建 execution environment
        final StreamExecutionEnvironment env =
                StreamExecutionEnvironment.getExecutionEnvironment();
        // 通过连接 socket 获取输入数据,这里连接到本地 9000 端口,如果 9000 端口已被占用,请换一个端口
        DataStream<String> text = env.socketTextStream("localhost", 9000, "\n");
        // 解析数据,按 word 分组,开窗,聚合
        DataStream<Tuple2<String, Integer>> windowCounts = text.
                flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {
                    @Override
                    public void flatMap(String value, Collector<Tuple2<String,
                            Integer>> out) {
                        for (String word : value.split("\\s")) {
                            out.collect(Tuple2.of(word, 1));
                        }
                    }
                })
                .keyBy(0)
                .timeWindow(Time.seconds(5))
                .sum(1);
// 将结果打印到控制台,注意这里使用的是单线程打印,而非多线程
        windowCounts.print().setParallelism(1);
        env.execute("Socket Window WordCount");
    }
 
Example #11
Source Project: blog_demos   Author: zq2599   File: StreamingJob.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<String> text = env.socketTextStream("127.0.0.1", 18081, "\n");

	DataStream<WordWithCount> windowCount = text.flatMap(new FlatMapFunction<String, WordWithCount>() {
		public void flatMap(String value, Collector<WordWithCount> out) throws Exception {
			String[] splits = value.split("\\s");
			for (String word:splits) {
				out.collect(new WordWithCount(word,1L));
			}
		}
	})
			.keyBy("word")
			.timeWindow(Time.seconds(5),Time.seconds(1))
			.sum("count");
	windowCount.print().setParallelism(1);
	env.execute("Flink Streaming Java API Skeleton");
}
 
Example #12
Source Project: pulsar-flink   Author: streamnative   File: PulsarTableSource.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public DataStream<Row> getDataStream(StreamExecutionEnvironment execEnv) {
    FlinkPulsarRowSource source = new FlinkPulsarRowSource(serviceUrl, adminUrl, properties);
    switch (startupMode) {
        case EARLIEST:
            source.setStartFromEarliest();
            break;
        case LATEST:
            source.setStartFromLatest();
            break;
        case SPECIFIC_OFFSETS:
            source.setStartFromSpecificOffsets(specificStartupOffsets);
            break;
        case EXTERNAL_SUBSCRIPTION:
            source.setStartFromSubscription(externalSubscriptionName);
    }

    return execEnv.addSource(source).name(explainSource());
}
 
Example #13
Source Project: flink   Author: apache   File: AllWindowTranslationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSessionWithFoldFails() throws Exception {
	// verify that fold does not work with merging windows

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	AllWindowedStream<String, TimeWindow> windowedStream = env.fromElements("Hello", "Ciao")
			.windowAll(EventTimeSessionWindows.withGap(Time.seconds(5)));

	try {
		windowedStream.fold("", new FoldFunction<String, String>() {
			private static final long serialVersionUID = -4567902917104921706L;

			@Override
			public String fold(String accumulator, String value) throws Exception {
				return accumulator;
			}
		});
	} catch (UnsupportedOperationException e) {
		// expected
		// use a catch to ensure that the exception is thrown by the fold
		return;
	}

	fail("The fold call should fail.");
}
 
Example #14
Source Project: flink   Author: flink-tpc-ds   File: AbstractNonKeyedOperatorRestoreTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void createMigrationJob(StreamExecutionEnvironment env) {
	/**
	 * Source -> StatefulMap1 -> CHAIN(StatefulMap2 -> Map -> StatefulMap3)
	 */
	DataStream<Integer> source = createSource(env, ExecutionMode.MIGRATE);

	SingleOutputStreamOperator<Integer> first = createFirstStatefulMap(ExecutionMode.MIGRATE, source);
	first.startNewChain();

	SingleOutputStreamOperator<Integer> second = createSecondStatefulMap(ExecutionMode.MIGRATE, first);
	second.startNewChain();

	SingleOutputStreamOperator<Integer> stateless = createStatelessMap(second);

	SingleOutputStreamOperator<Integer> third = createThirdStatefulMap(ExecutionMode.MIGRATE, stateless);
}
 
Example #15
Source Project: flink   Author: flink-tpc-ds   File: StreamGraphGeneratorTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test slot sharing is enabled.
 */
@Test
public void testEnableSlotSharing() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
	DataStream<Integer> mapDataStream = sourceDataStream.map(x -> x + 1);

	final List<Transformation<?>> transformations = new ArrayList<>();
	transformations.add(sourceDataStream.getTransformation());
	transformations.add(mapDataStream.getTransformation());

	// all stream nodes share default group by default
	StreamGraph streamGraph = new StreamGraphGenerator(
			transformations, env.getConfig(), env.getCheckpointConfig())
		.generate();

	Collection<StreamNode> streamNodes = streamGraph.getStreamNodes();
	for (StreamNode streamNode : streamNodes) {
		assertEquals(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, streamNode.getSlotSharingGroup());
	}
}
 
Example #16
Source Project: flink-simple-tutorial   Author: CheckChe0803   File: TumblingWindow.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

    // 构建输入数据
    List<Tuple2<String, Long>> data = new ArrayList<>();
    Tuple2<String, Long> a = new Tuple2<>("first event", 1L);
    Tuple2<String, Long> b = new Tuple2<>("second event", 2L);
    data.add(a);
    data.add(b);
    DataStreamSource<Tuple2<String, Long>> input = env.fromCollection(data);

    // 使用 ProcessTime 滚动窗口, 10s 为一个窗口长度
    input.keyBy(x -> x.f1)
            .window(TumblingProcessingTimeWindows.of(Time.seconds(10)))
            .reduce(new MyWindowFunction());

    env.execute();
}
 
Example #17
Source Project: flink   Author: apache   File: CheckpointedStreamingProgram.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

			env.enableCheckpointing(CHECKPOINT_INTERVALL);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 100L));
	env.disableOperatorChaining();

	DataStream<String> text = env.addSource(new SimpleStringGenerator());
	text.map(new StatefulMapper()).addSink(new DiscardingSink<>());
	env.setParallelism(1);
	env.execute("Checkpointed Streaming Program");
}
 
Example #18
Source Project: flink   Author: flink-tpc-ds   File: AllWindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testApplyProcessingTimeTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.windowAll(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS)))
			.apply(new AllWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(
						TimeWindow window,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {
					for (Tuple2<String, Integer> in : values) {
						out.collect(in);
					}
				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));

}
 
Example #19
Source Project: flink   Author: apache   File: WindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAggregateProcessingTime() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple3<String, String, Integer>> source = env.fromElements(
		Tuple3.of("hello", "hallo", 1),
		Tuple3.of("hello", "hallo", 2));

	DataStream<Integer> window1 = source
			.keyBy(new Tuple3KeySelector())
			.window(SlidingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.aggregate(new DummyAggregationFunction());

	final OneInputTransformation<Tuple3<String, String, Integer>, Integer> transform =
		(OneInputTransformation<Tuple3<String, String, Integer>, Integer>) window1.getTransformation();

	final OneInputStreamOperator<Tuple3<String, String, Integer>, Integer> operator = transform.getOperator();

	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?> winOperator =
			(WindowOperator<String, Tuple3<String, String, Integer>, ?, ?, ?>) operator;

	Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingProcessingTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof AggregatingStateDescriptor);

	processElementAndEnsureOutput(
			winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple3<>("hello", "hallo", 1));
}
 
Example #20
Source Project: flink   Author: apache   File: KafkaShuffleITCase.java    License: Apache License 2.0 5 votes vote down vote up
private StreamExecutionEnvironment createEnvironment(
		int producerParallelism,
		TimeCharacteristic timeCharacteristic) {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(producerParallelism);
	env.setStreamTimeCharacteristic(timeCharacteristic);
	env.setRestartStrategy(RestartStrategies.noRestart());

	return env;
}
 
Example #21
Source Project: flink-tutorials   Author: cloudera   File: SqlScriptExecutor.java    License: Apache License 2.0 5 votes vote down vote up
public static StreamTableEnvironment createTableEnv() {
	EnvironmentSettings settings = EnvironmentSettings
			.newInstance()
			.useBlinkPlanner()
			.inStreamingMode()
			.build();

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	StreamTableEnvironment tableEnv = StreamTableEnvironment
			.create(env, settings);

	return tableEnv;
}
 
Example #22
Source Project: Flink-CEPplus   Author: ljygz   File: SideOutputITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSideOutputNameClash() throws Exception {
	final OutputTag<String> sideOutputTag1 = new OutputTag<String>("side"){};
	final OutputTag<Integer> sideOutputTag2 = new OutputTag<Integer>("side"){};

	TestListResultSink<String> sideOutputResultSink1 = new TestListResultSink<>();
	TestListResultSink<Integer> sideOutputResultSink2 = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	SingleOutputStreamOperator<Integer> passThroughtStream = dataStream
			.process(new ProcessFunction<Integer, Integer>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void processElement(
						Integer value, Context ctx, Collector<Integer> out) throws Exception {
					out.collect(value);
					ctx.output(sideOutputTag1, "sideout-" + String.valueOf(value));
					ctx.output(sideOutputTag2, 13);
				}
			});

	passThroughtStream.getSideOutput(sideOutputTag1).addSink(sideOutputResultSink1);

	expectedException.expect(UnsupportedOperationException.class);
	passThroughtStream.getSideOutput(sideOutputTag2).addSink(sideOutputResultSink2);
}
 
Example #23
Source Project: aws-doc-sdk-examples   Author: awsdocs   File: kda-java-firehose.java    License: Apache License 2.0 5 votes vote down vote up
private static DataStream<String> createSourceFromStaticConfig(StreamExecutionEnvironment env) {
	Properties inputProperties = new Properties();
	inputProperties.setProperty(ConsumerConfigConstants.AWS_REGION, region);
	inputProperties.setProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, "LATEST");

	return env.addSource(new FlinkKinesisConsumer<>(inputStreamName, new SimpleStringSchema(), inputProperties));
}
 
Example #24
Source Project: flink   Author: apache   File: StreamExecutionEnvironmentTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSources() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	SourceFunction<Integer> srcFun = new SourceFunction<Integer>() {
		private static final long serialVersionUID = 1L;

		@Override
		public void run(SourceContext<Integer> ctx) throws Exception {
		}

		@Override
		public void cancel() {
		}
	};
	DataStreamSource<Integer> src1 = env.addSource(srcFun);
	src1.addSink(new DiscardingSink<Integer>());
	assertEquals(srcFun, getFunctionFromDataSource(src1));

	List<Long> list = Arrays.asList(0L, 1L, 2L);

	DataStreamSource<Long> src2 = env.generateSequence(0, 2);
	assertTrue(getFunctionFromDataSource(src2) instanceof StatefulSequenceSource);

	DataStreamSource<Long> src3 = env.fromElements(0L, 1L, 2L);
	assertTrue(getFunctionFromDataSource(src3) instanceof FromElementsFunction);

	DataStreamSource<Long> src4 = env.fromCollection(list);
	assertTrue(getFunctionFromDataSource(src4) instanceof FromElementsFunction);
}
 
Example #25
Source Project: flink   Author: flink-tpc-ds   File: KafkaConsumerTestBase.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test ensures that the consumer correctly uses user-supplied specific offsets when explicitly configured to
 * start from specific offsets. For partitions which a specific offset can not be found for, the starting position
 * for them should fallback to the group offsets behaviour.
 *
 * <p>4 partitions will have 50 records with offsets 0 to 49. The supplied specific offsets map is:
 * 	partition 0 --> start from offset 19
 * 	partition 1 --> not set
 * 	partition 2 --> start from offset 22
 * 	partition 3 --> not set
 * 	partition 4 --> start from offset 26 (this should be ignored because the partition does not exist)
 *
 * <p>The partitions and their committed group offsets are setup as:
 * 	partition 0 --> committed offset 23
 * 	partition 1 --> committed offset 31
 * 	partition 2 --> committed offset 43
 * 	partition 3 --> no commit offset
 *
 * <p>When configured to start from these specific offsets, each partition should read:
 * 	partition 0 --> start from offset 19, read to offset 49 (31 records)
 * 	partition 1 --> fallback to group offsets, so start from offset 31, read to offset 49 (19 records)
 * 	partition 2 --> start from offset 22, read to offset 49 (28 records)
 * 	partition 3 --> fallback to group offsets, but since there is no group offset for this partition,
 * 	                will default to "auto.offset.reset" (set to "earliest"),
 * 	                so start from offset 0, read to offset 49 (50 records)
 */
public void runStartFromSpecificOffsets() throws Exception {
	// 4 partitions with 50 records each (offsets 0-49)
	final int parallelism = 4;
	final int recordsInEachPartition = 50;

	final String topicName = writeSequence("testStartFromSpecificOffsetsTopic", recordsInEachPartition, parallelism, 1);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().disableSysoutLogging();
	env.setParallelism(parallelism);

	Properties readProps = new Properties();
	readProps.putAll(standardProps);
	readProps.setProperty("auto.offset.reset", "earliest"); // partition 3 should default back to this behaviour

	Map<KafkaTopicPartition, Long> specificStartupOffsets = new HashMap<>();
	specificStartupOffsets.put(new KafkaTopicPartition(topicName, 0), 19L);
	specificStartupOffsets.put(new KafkaTopicPartition(topicName, 2), 22L);
	specificStartupOffsets.put(new KafkaTopicPartition(topicName, 4), 26L); // non-existing partition, should be ignored

	// only the committed offset for partition 1 should be used, because partition 1 has no entry in specific offset map
	KafkaTestEnvironment.KafkaOffsetHandler kafkaOffsetHandler = kafkaServer.createOffsetHandler();
	kafkaOffsetHandler.setCommittedOffset(topicName, 0, 23);
	kafkaOffsetHandler.setCommittedOffset(topicName, 1, 31);
	kafkaOffsetHandler.setCommittedOffset(topicName, 2, 43);

	Map<Integer, Tuple2<Integer, Integer>> partitionsToValueCountAndStartOffsets = new HashMap<>();
	partitionsToValueCountAndStartOffsets.put(0, new Tuple2<>(31, 19)); // partition 0 should read offset 19-49
	partitionsToValueCountAndStartOffsets.put(1, new Tuple2<>(19, 31)); // partition 1 should read offset 31-49
	partitionsToValueCountAndStartOffsets.put(2, new Tuple2<>(28, 22));	// partition 2 should read offset 22-49
	partitionsToValueCountAndStartOffsets.put(3, new Tuple2<>(50, 0));	// partition 3 should read offset 0-49

	readSequence(env, StartupMode.SPECIFIC_OFFSETS, specificStartupOffsets, null, readProps, topicName, partitionsToValueCountAndStartOffsets);

	kafkaOffsetHandler.close();
	deleteTestTopic(topicName);
}
 
Example #26
Source Project: flink-training-exercises   Author: ververica   File: TaxiQuerySolution.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

		ParameterTool params = ParameterTool.fromArgs(args);
		final String input = params.get("input", ExerciseBase.pathToRideData);

		final int maxEventDelay = 60;       	// events are out of order by at most 60 seconds
		final int servingSpeedFactor = 1800; 	// 30 minutes worth of events are served every second

		// set up streaming execution environment
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
		env.setParallelism(ExerciseBase.parallelism);

		// setup a stream of taxi rides
		DataStream<TaxiRide> rides = env.addSource(rideSourceOrTest(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor)));

		// add a socket source for the query stream
		BroadcastStream<String> queryStream = env
				.addSource(stringSourceOrTest(new SocketTextStreamFunction("localhost", 9999, "\n", -1)))
				.assignTimestampsAndWatermarks(new QueryStreamAssigner())
				.broadcast(queryDescriptor);

		// connect the two streams and process queries
		DataStream<Tuple2<String, String>> results = rides
				.keyBy((TaxiRide ride) -> ride.taxiId)
				.connect(queryStream)
				.process(new QueryProcessor());

		printOrTest(results);

		env.execute("Taxi Query");
	}
 
Example #27
Source Project: flink   Author: apache   File: BroadcastStream.java    License: Apache License 2.0 5 votes vote down vote up
protected BroadcastStream(
		final StreamExecutionEnvironment env,
		final DataStream<T> input,
		final MapStateDescriptor<?, ?>... broadcastStateDescriptors) {

	this.environment = requireNonNull(env);
	this.inputStream = requireNonNull(input);
	this.broadcastStateDescriptors = Arrays.asList(requireNonNull(broadcastStateDescriptors));
}
 
Example #28
Source Project: flink   Author: flink-tpc-ds   File: WindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testMergingWindowsWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Integer> source = env.fromElements(1, 2);

	DataStream<String> window1 = source
			.keyBy(new KeySelector<Integer, String>() {
				@Override
				public String getKey(Integer value) throws Exception {
					return value.toString();
				}
			})
			.window(EventTimeSessionWindows.withGap(Time.seconds(5)))
			.evictor(CountEvictor.of(5))
			.process(new TestProcessWindowFunction());

	final OneInputTransformation<Integer, String> transform = (OneInputTransformation<Integer, String>) window1.getTransformation();
	final OneInputStreamOperator<Integer, String> operator = transform.getOperator();
	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Integer, ?, ?, ?> winOperator = (WindowOperator<String, Integer, ?, ?, ?>) operator;
	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof EventTimeSessionWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO,  1);
}
 
Example #29
Source Project: Flink-CEPplus   Author: ljygz   File: AllWindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAggregateWithEvictor() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
			.windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS)))
			.evictor(CountEvictor.of(100))
			.aggregate(new DummyAggregationFunction());

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform =
			(OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();

	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();

	Assert.assertTrue(operator instanceof WindowOperator);
	WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator =
			(WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;

	Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
	Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);

	processElementAndEnsureOutput(
			winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}
 
Example #30
Source Project: Flink-CEPplus   Author: ljygz   File: TestJob.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	final DataStreamSource<Integer> source = env.fromElements(1, 2, 3, 4);
	final SingleOutputStreamOperator<Integer> mapper = source.map(element -> 2 * element);
	mapper.addSink(new DiscardingSink<>());

	ParameterTool parameterTool = ParameterTool.fromArgs(args);
	env.execute(TestJob.class.getCanonicalName() + "-" + parameterTool.getRequired("arg"));
}