Java Code Examples for org.apache.flink.streaming.api.functions.windowing.WindowFunction

The following examples show how to use org.apache.flink.streaming.api.functions.windowing.WindowFunction. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: WindowedStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Applies the given window function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the window function is
 * interpreted as a regular non-windowed stream.
 *
 * <p>Arriving data is incrementally aggregated using the given aggregate function. This means
 * that the window function typically has only a single value to process when called.
 *
 * @param aggFunction The aggregate function that is used for incremental aggregation.
 * @param windowFunction The window function.
 *
 * @return The data stream that is the result of applying the window function to the window.
 *
 * @param <ACC> The type of the AggregateFunction's accumulator
 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input
 * @param <R> The type of the elements in the resulting stream, equal to the
 *            WindowFunction's result type
 */
@PublicEvolving
public <ACC, V, R> SingleOutputStreamOperator<R> aggregate(
		AggregateFunction<T, ACC, V> aggFunction,
		WindowFunction<V, R, K, W> windowFunction) {

	checkNotNull(aggFunction, "aggFunction");
	checkNotNull(windowFunction, "windowFunction");

	TypeInformation<ACC> accumulatorType = TypeExtractor.getAggregateFunctionAccumulatorType(
			aggFunction, input.getType(), null, false);

	TypeInformation<V> aggResultType = TypeExtractor.getAggregateFunctionReturnType(
			aggFunction, input.getType(), null, false);

	TypeInformation<R> resultType = getWindowFunctionReturnType(windowFunction, aggResultType);

	return aggregate(aggFunction, windowFunction, accumulatorType, resultType);
}
 
Example 2
Source Project: Flink-CEPplus   Source File: StateDescriptorPassingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testApplyWindowState() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
	env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);

	DataStream<File> src = env.fromElements(new File("/"));

	SingleOutputStreamOperator<?> result = src
			.keyBy(new KeySelector<File, String>() {
				@Override
				public String getKey(File value) {
					return null;
				}
			})
			.timeWindow(Time.milliseconds(1000))
			.apply(new WindowFunction<File, String, String, TimeWindow>() {
				@Override
				public void apply(String s, TimeWindow window,
									Iterable<File> input, Collector<String> out) {}
			});

	validateListStateDescriptorConfigured(result);
}
 
Example 3
Source Project: flink   Source File: WindowedStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Applies the given window function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the window function is
 * interpreted as a regular non-windowed stream.
 *
 * <p>Arriving data is incrementally aggregated using the given aggregate function. This means
 * that the window function typically has only a single value to process when called.
 *
 * @param aggFunction The aggregate function that is used for incremental aggregation.
 * @param windowFunction The window function.
 *
 * @return The data stream that is the result of applying the window function to the window.
 *
 * @param <ACC> The type of the AggregateFunction's accumulator
 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input
 * @param <R> The type of the elements in the resulting stream, equal to the
 *            WindowFunction's result type
 */
@PublicEvolving
public <ACC, V, R> SingleOutputStreamOperator<R> aggregate(
		AggregateFunction<T, ACC, V> aggFunction,
		WindowFunction<V, R, K, W> windowFunction) {

	checkNotNull(aggFunction, "aggFunction");
	checkNotNull(windowFunction, "windowFunction");

	TypeInformation<ACC> accumulatorType = TypeExtractor.getAggregateFunctionAccumulatorType(
			aggFunction, input.getType(), null, false);

	TypeInformation<V> aggResultType = TypeExtractor.getAggregateFunctionReturnType(
			aggFunction, input.getType(), null, false);

	TypeInformation<R> resultType = getWindowFunctionReturnType(windowFunction, aggResultType);

	return aggregate(aggFunction, windowFunction, accumulatorType, resultType);
}
 
Example 4
Source Project: flink   Source File: StateDescriptorPassingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testApplyWindowState() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
	env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);

	DataStream<File> src = env.fromElements(new File("/"));

	SingleOutputStreamOperator<?> result = src
			.keyBy(new KeySelector<File, String>() {
				@Override
				public String getKey(File value) {
					return null;
				}
			})
			.timeWindow(Time.milliseconds(1000))
			.apply(new WindowFunction<File, String, String, TimeWindow>() {
				@Override
				public void apply(String s, TimeWindow window,
									Iterable<File> input, Collector<String> out) {}
			});

	validateListStateDescriptorConfigured(result);
}
 
Example 5
Source Project: flink   Source File: WindowedStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Applies the given window function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the window function is
 * interpreted as a regular non-windowed stream.
 *
 * <p>Arriving data is incrementally aggregated using the given aggregate function. This means
 * that the window function typically has only a single value to process when called.
 *
 * @param aggFunction The aggregate function that is used for incremental aggregation.
 * @param windowFunction The window function.
 *
 * @return The data stream that is the result of applying the window function to the window.
 *
 * @param <ACC> The type of the AggregateFunction's accumulator
 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input
 * @param <R> The type of the elements in the resulting stream, equal to the
 *            WindowFunction's result type
 */
@PublicEvolving
public <ACC, V, R> SingleOutputStreamOperator<R> aggregate(
		AggregateFunction<T, ACC, V> aggFunction,
		WindowFunction<V, R, K, W> windowFunction) {

	checkNotNull(aggFunction, "aggFunction");
	checkNotNull(windowFunction, "windowFunction");

	TypeInformation<ACC> accumulatorType = TypeExtractor.getAggregateFunctionAccumulatorType(
			aggFunction, input.getType(), null, false);

	TypeInformation<V> aggResultType = TypeExtractor.getAggregateFunctionReturnType(
			aggFunction, input.getType(), null, false);

	TypeInformation<R> resultType = getWindowFunctionReturnType(windowFunction, aggResultType);

	return aggregate(aggFunction, windowFunction, accumulatorType, resultType);
}
 
Example 6
Source Project: flink   Source File: StateDescriptorPassingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testApplyWindowState() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
	env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);

	DataStream<File> src = env.fromElements(new File("/"));

	SingleOutputStreamOperator<?> result = src
			.keyBy(new KeySelector<File, String>() {
				@Override
				public String getKey(File value) {
					return null;
				}
			})
			.timeWindow(Time.milliseconds(1000))
			.apply(new WindowFunction<File, String, String, TimeWindow>() {
				@Override
				public void apply(String s, TimeWindow window,
									Iterable<File> input, Collector<String> out) {}
			});

	validateListStateDescriptorConfigured(result);
}
 
Example 7
Source Project: Flink-CEPplus   Source File: SessionWindowITCase.java    License: Apache License 2.0 5 votes vote down vote up
private void runTest(
		SourceFunction<SessionEvent<Integer, TestEventPayload>> dataSource,
		WindowFunction<SessionEvent<Integer, TestEventPayload>,
				String, Tuple, TimeWindow> windowFunction) throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	WindowedStream<SessionEvent<Integer, TestEventPayload>, Tuple, TimeWindow> windowedStream =
			env.addSource(dataSource).keyBy("sessionKey")
			.window(EventTimeSessionWindows.withGap(Time.milliseconds(MAX_SESSION_EVENT_GAP_MS)));

	if (ALLOWED_LATENESS_MS != Long.MAX_VALUE) {
		windowedStream = windowedStream.allowedLateness(Time.milliseconds(ALLOWED_LATENESS_MS));
	}

	if (PURGE_WINDOW_ON_FIRE) {
		windowedStream = windowedStream.trigger(PurgingTrigger.of(EventTimeTrigger.create()));
	}

	windowedStream.apply(windowFunction).print();
	JobExecutionResult result = env.execute();

	// check that overall event counts match with our expectations. remember that late events within lateness will
	// each trigger a window!
	Assert.assertEquals(
		(LATE_EVENTS_PER_SESSION + 1) * NUMBER_OF_SESSIONS * EVENTS_PER_SESSION,
		(long) result.getAccumulatorResult(SESSION_COUNTER_ON_TIME_KEY));
	Assert.assertEquals(
		NUMBER_OF_SESSIONS * (LATE_EVENTS_PER_SESSION * (LATE_EVENTS_PER_SESSION + 1) / 2),
		(long) result.getAccumulatorResult(SESSION_COUNTER_LATE_KEY));
}
 
Example 8
Source Project: Flink-CEPplus   Source File: WindowedStream.java    License: Apache License 2.0 5 votes vote down vote up
private static <IN, OUT, KEY> TypeInformation<OUT> getWindowFunctionReturnType(
	WindowFunction<IN, OUT, KEY, ?> function,
	TypeInformation<IN> inType) {
	return TypeExtractor.getUnaryOperatorReturnType(
		function,
		WindowFunction.class,
		0,
		1,
		new int[]{3, 0},
		inType,
		null,
		false);
}
 
Example 9
Source Project: Flink-CEPplus   Source File: TimeWindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that calls to timeWindow() instantiate a regular
 * windowOperator instead of an aligned one.
 */
@Test
public void testAlignedWindowDeprecation() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DummyReducer reducer = new DummyReducer();

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(0)
			.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS), Time.of(100, TimeUnit.MILLISECONDS))
			.reduce(reducer);

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator();
	Assert.assertTrue(operator1 instanceof WindowOperator);

	DataStream<Tuple2<String, Integer>> window2 = source
			.keyBy(0)
			.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS))
			.apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Tuple tuple,
						TimeWindow window,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {

				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform2 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window2.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator2 = transform2.getOperator();
	Assert.assertTrue(operator2 instanceof WindowOperator);
}
 
Example 10
Source Project: Flink-CEPplus   Source File: TimeWindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testApplyEventTimeWindows() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(
			Tuple2.of("hello", 1),
			Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
		.keyBy(0)
		.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS))
		.apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
			private static final long serialVersionUID = 1L;

			@Override
			public void apply(Tuple tuple,
				TimeWindow window,
				Iterable<Tuple2<String, Integer>> values,
				Collector<Tuple2<String, Integer>> out) throws Exception {

			}
		});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator();
	Assert.assertTrue(operator1 instanceof WindowOperator);
	WindowOperator winOperator1 = (WindowOperator) operator1;
	Assert.assertTrue(winOperator1.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator1.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator1.getStateDescriptor() instanceof ListStateDescriptor);
}
 
Example 11
Source Project: flink   Source File: SessionWindowITCase.java    License: Apache License 2.0 5 votes vote down vote up
private void runTest(
		SourceFunction<SessionEvent<Integer, TestEventPayload>> dataSource,
		WindowFunction<SessionEvent<Integer, TestEventPayload>,
				String, Tuple, TimeWindow> windowFunction) throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	WindowedStream<SessionEvent<Integer, TestEventPayload>, Tuple, TimeWindow> windowedStream =
			env.addSource(dataSource).keyBy("sessionKey")
			.window(EventTimeSessionWindows.withGap(Time.milliseconds(MAX_SESSION_EVENT_GAP_MS)));

	if (ALLOWED_LATENESS_MS != Long.MAX_VALUE) {
		windowedStream = windowedStream.allowedLateness(Time.milliseconds(ALLOWED_LATENESS_MS));
	}

	if (PURGE_WINDOW_ON_FIRE) {
		windowedStream = windowedStream.trigger(PurgingTrigger.of(EventTimeTrigger.create()));
	}

	windowedStream.apply(windowFunction).print();
	JobExecutionResult result = env.execute();

	// check that overall event counts match with our expectations. remember that late events within lateness will
	// each trigger a window!
	Assert.assertEquals(
		(LATE_EVENTS_PER_SESSION + 1) * NUMBER_OF_SESSIONS * EVENTS_PER_SESSION,
		(long) result.getAccumulatorResult(SESSION_COUNTER_ON_TIME_KEY));
	Assert.assertEquals(
		NUMBER_OF_SESSIONS * (LATE_EVENTS_PER_SESSION * (LATE_EVENTS_PER_SESSION + 1) / 2),
		(long) result.getAccumulatorResult(SESSION_COUNTER_LATE_KEY));
}
 
Example 12
Source Project: flink   Source File: WindowedStream.java    License: Apache License 2.0 5 votes vote down vote up
private static <IN, OUT, KEY> TypeInformation<OUT> getWindowFunctionReturnType(
	WindowFunction<IN, OUT, KEY, ?> function,
	TypeInformation<IN> inType) {
	return TypeExtractor.getUnaryOperatorReturnType(
		function,
		WindowFunction.class,
		0,
		1,
		new int[]{3, 0},
		inType,
		null,
		false);
}
 
Example 13
Source Project: flink   Source File: TimeWindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that calls to timeWindow() instantiate a regular
 * windowOperator instead of an aligned one.
 */
@Test
public void testAlignedWindowDeprecation() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DummyReducer reducer = new DummyReducer();

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(0)
			.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS), Time.of(100, TimeUnit.MILLISECONDS))
			.reduce(reducer);

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator();
	Assert.assertTrue(operator1 instanceof WindowOperator);

	DataStream<Tuple2<String, Integer>> window2 = source
			.keyBy(0)
			.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS))
			.apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Tuple tuple,
						TimeWindow window,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {

				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform2 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window2.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator2 = transform2.getOperator();
	Assert.assertTrue(operator2 instanceof WindowOperator);
}
 
Example 14
Source Project: flink   Source File: TimeWindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testApplyEventTimeWindows() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(
			Tuple2.of("hello", 1),
			Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
		.keyBy(0)
		.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS))
		.apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
			private static final long serialVersionUID = 1L;

			@Override
			public void apply(Tuple tuple,
				TimeWindow window,
				Iterable<Tuple2<String, Integer>> values,
				Collector<Tuple2<String, Integer>> out) throws Exception {

			}
		});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator();
	Assert.assertTrue(operator1 instanceof WindowOperator);
	WindowOperator winOperator1 = (WindowOperator) operator1;
	Assert.assertTrue(winOperator1.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator1.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator1.getStateDescriptor() instanceof ListStateDescriptor);
}
 
Example 15
/**
 * Sum - Window function, summing already happened in reduce function
 */
private static WindowFunction<Tuple3<String, String, Long>, Tuple3<String, String, Long>, Tuple, TimeWindow> sumWindowFunction() {
  return new WindowFunction<Tuple3<String, String, Long>, Tuple3<String, String, Long>, Tuple, TimeWindow>() {
    @Override
    public void apply(Tuple keyTuple, TimeWindow window, Iterable<Tuple3<String, String, Long>> values, Collector<Tuple3<String, String, Long>> out) throws Exception {
      Iterator<Tuple3<String, String, Long>> valIter = values.iterator();
      Tuple3<String, String, Long> tuple = valIter.next();
      if (valIter.hasNext()) {
        throw new IllegalStateException("Unexpected");
      }
      tuple.f1 = Long.toString(window.getEnd());
      out.collect(tuple); // collect end time here
    }
  };
}
 
Example 16
Source Project: flink   Source File: SessionWindowITCase.java    License: Apache License 2.0 5 votes vote down vote up
private void runTest(
		SourceFunction<SessionEvent<Integer, TestEventPayload>> dataSource,
		WindowFunction<SessionEvent<Integer, TestEventPayload>,
				String, Tuple, TimeWindow> windowFunction) throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	WindowedStream<SessionEvent<Integer, TestEventPayload>, Tuple, TimeWindow> windowedStream =
			env.addSource(dataSource).keyBy("sessionKey")
			.window(EventTimeSessionWindows.withGap(Time.milliseconds(MAX_SESSION_EVENT_GAP_MS)));

	if (ALLOWED_LATENESS_MS != Long.MAX_VALUE) {
		windowedStream = windowedStream.allowedLateness(Time.milliseconds(ALLOWED_LATENESS_MS));
	}

	if (PURGE_WINDOW_ON_FIRE) {
		windowedStream = windowedStream.trigger(PurgingTrigger.of(EventTimeTrigger.create()));
	}

	windowedStream.apply(windowFunction).print();
	JobExecutionResult result = env.execute();

	// check that overall event counts match with our expectations. remember that late events within lateness will
	// each trigger a window!
	Assert.assertEquals(
		(LATE_EVENTS_PER_SESSION + 1) * NUMBER_OF_SESSIONS * EVENTS_PER_SESSION,
		(long) result.getAccumulatorResult(SESSION_COUNTER_ON_TIME_KEY));
	Assert.assertEquals(
		NUMBER_OF_SESSIONS * (LATE_EVENTS_PER_SESSION * (LATE_EVENTS_PER_SESSION + 1) / 2),
		(long) result.getAccumulatorResult(SESSION_COUNTER_LATE_KEY));
}
 
Example 17
public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);
	final boolean useValueState = pt.getBoolean("useValueState", false);
	final boolean useListState = pt.getBoolean("useListState", false);
	final boolean useMapState = pt.getBoolean("useMapState", false);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	setupEnvironment(env, pt);
	KeyedStream<Event, Integer> keyedStream = env
		.addSource(DataStreamAllroundTestJobFactory.createEventSource(pt))
		.name(EVENT_SOURCE.getName())
		.uid(EVENT_SOURCE.getUid())
		.assignTimestampsAndWatermarks(createTimestampExtractor(pt))
		.keyBy(Event::getKey);

	keyedStream.map(new ValueStateMapper(useValueState)).name("ValueStateMapper").uid("ValueStateMapper");
	keyedStream.map(new ListStateMapper(useListState)).name("ListStateMapper").uid("ListStateMapper");
	keyedStream.map(new MapStateMapper(useMapState)).name("MapStateMapper").uid("MapStateMapper");

	boolean useWindow = pt.getBoolean("useWindow", false);
	if (useWindow) {
		applyTumblingWindows(keyedStream, pt)
			.apply(new WindowFunction<Event, Event, Integer, TimeWindow>() {
				@Override
				public void apply(Integer integer, TimeWindow window, Iterable<Event> input, Collector<Event> out) {
					for (Event e : input) {
						out.collect(e);
					}
				}
			})
			.name(TIME_WINDOW_OPER.getName())
			.uid(TIME_WINDOW_OPER.getUid());
	}

	env.execute("RocksDB test job");
}
 
Example 18
Source Project: flink   Source File: WindowedStream.java    License: Apache License 2.0 5 votes vote down vote up
private static <IN, OUT, KEY> TypeInformation<OUT> getWindowFunctionReturnType(
	WindowFunction<IN, OUT, KEY, ?> function,
	TypeInformation<IN> inType) {
	return TypeExtractor.getUnaryOperatorReturnType(
		function,
		WindowFunction.class,
		0,
		1,
		new int[]{3, 0},
		inType,
		null,
		false);
}
 
Example 19
Source Project: flink   Source File: TimeWindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that calls to timeWindow() instantiate a regular
 * windowOperator instead of an aligned one.
 */
@Test
public void testAlignedWindowDeprecation() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));

	DummyReducer reducer = new DummyReducer();

	DataStream<Tuple2<String, Integer>> window1 = source
			.keyBy(0)
			.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS), Time.of(100, TimeUnit.MILLISECONDS))
			.reduce(reducer);

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator();
	Assert.assertTrue(operator1 instanceof WindowOperator);

	DataStream<Tuple2<String, Integer>> window2 = source
			.keyBy(0)
			.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS))
			.apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Tuple tuple,
						TimeWindow window,
						Iterable<Tuple2<String, Integer>> values,
						Collector<Tuple2<String, Integer>> out) throws Exception {

				}
			});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform2 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window2.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator2 = transform2.getOperator();
	Assert.assertTrue(operator2 instanceof WindowOperator);
}
 
Example 20
Source Project: flink   Source File: TimeWindowTranslationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@SuppressWarnings("rawtypes")
public void testApplyEventTimeWindows() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);

	DataStream<Tuple2<String, Integer>> source = env.fromElements(
			Tuple2.of("hello", 1),
			Tuple2.of("hello", 2));

	DataStream<Tuple2<String, Integer>> window1 = source
		.keyBy(0)
		.timeWindow(Time.of(1000, TimeUnit.MILLISECONDS))
		.apply(new WindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple, TimeWindow>() {
			private static final long serialVersionUID = 1L;

			@Override
			public void apply(Tuple tuple,
				TimeWindow window,
				Iterable<Tuple2<String, Integer>> values,
				Collector<Tuple2<String, Integer>> out) throws Exception {

			}
		});

	OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform1 = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
	OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator1 = transform1.getOperator();
	Assert.assertTrue(operator1 instanceof WindowOperator);
	WindowOperator winOperator1 = (WindowOperator) operator1;
	Assert.assertTrue(winOperator1.getTrigger() instanceof EventTimeTrigger);
	Assert.assertTrue(winOperator1.getWindowAssigner() instanceof TumblingEventTimeWindows);
	Assert.assertTrue(winOperator1.getStateDescriptor() instanceof ListStateDescriptor);
}
 
Example 21
Source Project: Flink-CEPplus   Source File: SideOutputITCase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testKeyedWindowLateArrivingEvents() throws Exception {
	TestListResultSink<String> resultSink = new TestListResultSink<>();
	TestListResultSink<Integer> lateResultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);
	see.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	OutputTag<Integer> lateDataTag = new OutputTag<Integer>("late"){};

	SingleOutputStreamOperator<String> windowOperator = dataStream
			.assignTimestampsAndWatermarks(new TestWatermarkAssigner())
			.keyBy(new TestKeySelector())
			.timeWindow(Time.milliseconds(1), Time.milliseconds(1))
			.allowedLateness(Time.milliseconds(2))
			.sideOutputLateData(lateDataTag)
			.apply(new WindowFunction<Integer, String, Integer, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Integer key, TimeWindow window, Iterable<Integer> input, Collector<String> out) throws Exception {
					for (Integer val : input) {
						out.collect(String.valueOf(key) + "-" + String.valueOf(val));
					}
				}
			});

	windowOperator
			.addSink(resultSink);

	windowOperator
			.getSideOutput(lateDataTag)
			.addSink(lateResultSink);

	see.execute();
	assertEquals(Arrays.asList("1-1", "2-2", "4-4", "5-5"), resultSink.getSortedResult());
	assertEquals(Collections.singletonList(3), lateResultSink.getSortedResult());
}
 
Example 22
Source Project: Flink-CEPplus   Source File: PythonApplyFunction.java    License: Apache License 2.0 4 votes vote down vote up
public PythonApplyFunction(WindowFunction<PyObject, Object, Object, W> fun) throws IOException {
	super(fun);
}
 
Example 23
public InternalIterableWindowFunction(WindowFunction<IN, OUT, KEY, W> wrappedFunction) {
	super(wrappedFunction);
}
 
Example 24
public InternalSingleValueWindowFunction(WindowFunction<IN, OUT, KEY, W> wrappedFunction) {
	super(wrappedFunction);
}
 
Example 25
Source Project: flink   Source File: SideOutputITCase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testKeyedWindowLateArrivingEvents() throws Exception {
	TestListResultSink<String> resultSink = new TestListResultSink<>();
	TestListResultSink<Integer> lateResultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);
	see.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	OutputTag<Integer> lateDataTag = new OutputTag<Integer>("late"){};

	SingleOutputStreamOperator<String> windowOperator = dataStream
			.assignTimestampsAndWatermarks(new TestWatermarkAssigner())
			.keyBy(new TestKeySelector())
			.timeWindow(Time.milliseconds(1), Time.milliseconds(1))
			.allowedLateness(Time.milliseconds(2))
			.sideOutputLateData(lateDataTag)
			.apply(new WindowFunction<Integer, String, Integer, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Integer key, TimeWindow window, Iterable<Integer> input, Collector<String> out) throws Exception {
					for (Integer val : input) {
						out.collect(String.valueOf(key) + "-" + String.valueOf(val));
					}
				}
			});

	windowOperator
			.addSink(resultSink);

	windowOperator
			.getSideOutput(lateDataTag)
			.addSink(lateResultSink);

	see.execute();
	assertEquals(Arrays.asList("1-1", "2-2", "4-4", "5-5"), resultSink.getSortedResult());
	assertEquals(Collections.singletonList(3), lateResultSink.getSortedResult());
}
 
Example 26
Source Project: flink   Source File: InternalIterableWindowFunction.java    License: Apache License 2.0 4 votes vote down vote up
public InternalIterableWindowFunction(WindowFunction<IN, OUT, KEY, W> wrappedFunction) {
	super(wrappedFunction);
}
 
Example 27
Source Project: flink   Source File: InternalSingleValueWindowFunction.java    License: Apache License 2.0 4 votes vote down vote up
public InternalSingleValueWindowFunction(WindowFunction<IN, OUT, KEY, W> wrappedFunction) {
	super(wrappedFunction);
}
 
Example 28
Source Project: Mastering-Apache-Flink   Source File: StreamingJob.java    License: MIT License 4 votes vote down vote up
public static void main(String[] args) throws Exception {
	// set up the streaming execution environment
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	// env.enableCheckpointing(5000);
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

	Properties properties = new Properties();
	properties.setProperty("bootstrap.servers", "localhost:9092");

	properties.setProperty("zookeeper.connect", "localhost:2181");
	properties.setProperty("group.id", "test");

	FlinkKafkaConsumer09<String> myConsumer = new FlinkKafkaConsumer09<>("temp", new SimpleStringSchema(),
			properties);
	myConsumer.assignTimestampsAndWatermarks(new CustomWatermarkEmitter());


	DataStream<Tuple2<String, Double>> keyedStream = env.addSource(myConsumer).flatMap(new Splitter()).keyBy(0)
			.timeWindow(Time.seconds(300))
			.apply(new WindowFunction<Tuple2<String, Double>, Tuple2<String, Double>, Tuple, TimeWindow>() {

				@Override
				public void apply(Tuple key, TimeWindow window, Iterable<Tuple2<String, Double>> input,
						Collector<Tuple2<String, Double>> out) throws Exception {
					double sum = 0L;
					int count = 0;
					for (Tuple2<String, Double> record : input) {
						sum += record.f1;
						count++;
					}

					Tuple2<String, Double> result = input.iterator().next();
					result.f1 = (sum/count);
					out.collect(result);

				}
			});

	keyedStream.print();

	// execute program
	env.execute("Flink Streaming Java API Skeleton");
}
 
Example 29
Source Project: blog_demos   Source File: StreamingJob.java    License: Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.enableCheckpointing(5000); // 要设置启动检查点
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

		Properties props = new Properties();
		props.setProperty("bootstrap.servers", "kafka1:9092");
		props.setProperty("group.id", "flink-group");

		//数据源配置,是一个kafka消息的消费者
		FlinkKafkaConsumer011<String> consumer =
				new FlinkKafkaConsumer011<>("topic001", new SimpleStringSchema(), props);

		//增加时间水位设置类
		consumer.assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<String> (){
			@Override
			public long extractTimestamp(String element, long previousElementTimestamp) {
				return JSONHelper.getTimeLongFromRawMessage(element);
			}

			@Nullable
			@Override
			public Watermark checkAndGetNextWatermark(String lastElement, long extractedTimestamp) {
				if (lastElement != null) {
					return new Watermark(JSONHelper.getTimeLongFromRawMessage(lastElement));
				}
				return null;
			}
		});

		env.addSource(consumer)
				//将原始消息转成Tuple2对象,保留用户名称和访问次数(每个消息访问次数为1)
				.flatMap((FlatMapFunction<String, Tuple2<String, Long>>) (s, collector) -> {
					SingleMessage singleMessage = JSONHelper.parse(s);

					if (null != singleMessage) {
						collector.collect(new Tuple2<>(singleMessage.getName(), 1L));
					}
				})
				//以用户名为key
				.keyBy(0)
				//时间窗口为2秒
				.timeWindow(Time.seconds(2))
				//将每个用户访问次数累加起来
				.apply((WindowFunction<Tuple2<String, Long>, Tuple2<String, Long>, Tuple, TimeWindow>) (tuple, window, input, out) -> {
					long sum = 0L;
					for (Tuple2<String, Long> record: input) {
						sum += record.f1;
					}

					Tuple2<String, Long> result = input.iterator().next();
					result.f1 = sum;
					out.collect(result);
				})
				//输出方式是STDOUT
				.print();

		env.execute("Flink-Kafka demo");
	}
 
Example 30
Source Project: flink   Source File: SideOutputITCase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testKeyedWindowLateArrivingEvents() throws Exception {
	TestListResultSink<String> resultSink = new TestListResultSink<>();
	TestListResultSink<Integer> lateResultSink = new TestListResultSink<>();

	StreamExecutionEnvironment see = StreamExecutionEnvironment.getExecutionEnvironment();
	see.setParallelism(3);
	see.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

	DataStream<Integer> dataStream = see.fromCollection(elements);

	OutputTag<Integer> lateDataTag = new OutputTag<Integer>("late"){};

	SingleOutputStreamOperator<String> windowOperator = dataStream
			.assignTimestampsAndWatermarks(new TestWatermarkAssigner())
			.keyBy(new TestKeySelector())
			.timeWindow(Time.milliseconds(1), Time.milliseconds(1))
			.allowedLateness(Time.milliseconds(2))
			.sideOutputLateData(lateDataTag)
			.apply(new WindowFunction<Integer, String, Integer, TimeWindow>() {
				private static final long serialVersionUID = 1L;

				@Override
				public void apply(Integer key, TimeWindow window, Iterable<Integer> input, Collector<String> out) throws Exception {
					for (Integer val : input) {
						out.collect(String.valueOf(key) + "-" + String.valueOf(val));
					}
				}
			});

	windowOperator
			.addSink(resultSink);

	windowOperator
			.getSideOutput(lateDataTag)
			.addSink(lateResultSink);

	see.execute();
	assertEquals(Arrays.asList("1-1", "2-2", "4-4", "5-5"), resultSink.getSortedResult());
	assertEquals(Collections.singletonList(3), lateResultSink.getSortedResult());
}