org.apache.flink.api.common.functions.FoldFunction Java Examples

The following examples show how to use org.apache.flink.api.common.functions.FoldFunction. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StateDescriptorPassingTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testFoldWindowState() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
	env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);

	DataStream<String> src = env.fromElements("abc");

	SingleOutputStreamOperator<?> result = src
			.keyBy(new KeySelector<String, String>() {
				@Override
				public String getKey(String value) {
					return null;
				}
			})
			.timeWindow(Time.milliseconds(1000))
			.fold(new File("/"), new FoldFunction<String, File>() {

				@Override
				public File fold(File a, String e) {
					return null;
				}
			});

	validateStateDescriptorConfigured(result);
}
 
Example #2
Source File: AllWindowTranslationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSessionWithFoldFails() throws Exception {
	// verify that fold does not work with merging windows

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	AllWindowedStream<String, TimeWindow> windowedStream = env.fromElements("Hello", "Ciao")
			.windowAll(EventTimeSessionWindows.withGap(Time.seconds(5)));

	try {
		windowedStream.fold("", new FoldFunction<String, String>() {
			private static final long serialVersionUID = -4567902917104921706L;

			@Override
			public String fold(String accumulator, String value) throws Exception {
				return accumulator;
			}
		});
	} catch (UnsupportedOperationException e) {
		// expected
		// use a catch to ensure that the exception is thrown by the fold
		return;
	}

	fail("The fold call should fail.");
}
 
Example #3
Source File: AllWindowTranslationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSessionWithFoldFails() throws Exception {
	// verify that fold does not work with merging windows

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	AllWindowedStream<String, TimeWindow> windowedStream = env.fromElements("Hello", "Ciao")
			.windowAll(EventTimeSessionWindows.withGap(Time.seconds(5)));

	try {
		windowedStream.fold("", new FoldFunction<String, String>() {
			private static final long serialVersionUID = -4567902917104921706L;

			@Override
			public String fold(String accumulator, String value) throws Exception {
				return accumulator;
			}
		});
	} catch (UnsupportedOperationException e) {
		// expected
		// use a catch to ensure that the exception is thrown by the fold
		return;
	}

	fail("The fold call should fail.");
}
 
Example #4
Source File: StateDescriptorPassingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testFoldWindowState() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
	env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);

	DataStream<String> src = env.fromElements("abc");

	SingleOutputStreamOperator<?> result = src
			.keyBy(new KeySelector<String, String>() {
				@Override
				public String getKey(String value) {
					return null;
				}
			})
			.timeWindow(Time.milliseconds(1000))
			.fold(new File("/"), new FoldFunction<String, File>() {

				@Override
				public File fold(File a, String e) {
					return null;
				}
			});

	validateStateDescriptorConfigured(result);
}
 
Example #5
Source File: StateDescriptorPassingTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testFoldWindowAllState() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
	env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);

	DataStream<String> src = env.fromElements("abc");

	SingleOutputStreamOperator<?> result = src
			.timeWindowAll(Time.milliseconds(1000))
			.fold(new File("/"), new FoldFunction<String, File>() {

				@Override
				public File fold(File a, String e) {
					return null;
				}
			});

	validateStateDescriptorConfigured(result);
}
 
Example #6
Source File: AllWindowTranslationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testSessionWithFoldFails() throws Exception {
	// verify that fold does not work with merging windows

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	AllWindowedStream<String, TimeWindow> windowedStream = env.fromElements("Hello", "Ciao")
			.windowAll(EventTimeSessionWindows.withGap(Time.seconds(5)));

	try {
		windowedStream.fold("", new FoldFunction<String, String>() {
			private static final long serialVersionUID = -4567902917104921706L;

			@Override
			public String fold(String accumulator, String value) throws Exception {
				return accumulator;
			}
		});
	} catch (UnsupportedOperationException e) {
		// expected
		// use a catch to ensure that the exception is thrown by the fold
		return;
	}

	fail("The fold call should fail.");
}
 
Example #7
Source File: RocksDBFoldingState.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@code RocksDBFoldingState}.
 *
 * @param columnFamily The RocksDB column family that this state is associated to.
 * @param namespaceSerializer The serializer for the namespace.
 * @param valueSerializer The serializer for the state.
 * @param defaultValue The default value for the state.
 * @param foldFunction The fold function used for folding state.
 * @param backend The backend for which this state is bind to.
 */
private RocksDBFoldingState(
	ColumnFamilyHandle columnFamily,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<ACC> valueSerializer,
	ACC defaultValue,
	FoldFunction<T, ACC> foldFunction,
	RocksDBKeyedStateBackend<K> backend) {

	super(columnFamily, namespaceSerializer, valueSerializer, defaultValue, backend);

	this.foldFunction = foldFunction;
}
 
Example #8
Source File: HeapFoldingState.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new key/value state for the given hash map of key/value pairs.
 *
 * @param stateTable The state table for which this state is associated to.
 * @param keySerializer The serializer for the keys.
 * @param valueSerializer The serializer for the state.
 * @param namespaceSerializer The serializer for the namespace.
 * @param defaultValue The default value for the state.
 * @param foldFunction The fold function used for folding state.
 */
private HeapFoldingState(
	StateTable<K, N, ACC> stateTable,
	TypeSerializer<K> keySerializer,
	TypeSerializer<ACC> valueSerializer,
	TypeSerializer<N> namespaceSerializer,
	ACC defaultValue,
	FoldFunction<T, ACC> foldFunction) {
	super(stateTable, keySerializer, valueSerializer, namespaceSerializer, defaultValue);
	this.foldTransformation = new FoldTransformation(foldFunction);
}
 
Example #9
Source File: FoldingStateDescriptor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@code ValueStateDescriptor} with the given name and default value.
 *
 * @param name The (unique) name for the state.
 * @param initialValue The initial value of the fold.
 * @param foldFunction The {@code FoldFunction} used to aggregate the state.
 * @param typeSerializer The type serializer of the values in the state.
 */
public FoldingStateDescriptor(String name, ACC initialValue, FoldFunction<T, ACC> foldFunction, TypeSerializer<ACC> typeSerializer) {
	super(name, typeSerializer, initialValue);
	this.foldFunction = requireNonNull(foldFunction);

	if (foldFunction instanceof RichFunction) {
		throw new UnsupportedOperationException("FoldFunction of FoldingState can not be a RichFunction.");
	}
}
 
Example #10
Source File: HeapFoldingState.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new key/value state for the given hash map of key/value pairs.
 *
 * @param stateTable The state table for which this state is associated to.
 * @param keySerializer The serializer for the keys.
 * @param valueSerializer The serializer for the state.
 * @param namespaceSerializer The serializer for the namespace.
 * @param defaultValue The default value for the state.
 * @param foldFunction The fold function used for folding state.
 */
private HeapFoldingState(
	StateTable<K, N, ACC> stateTable,
	TypeSerializer<K> keySerializer,
	TypeSerializer<ACC> valueSerializer,
	TypeSerializer<N> namespaceSerializer,
	ACC defaultValue,
	FoldFunction<T, ACC> foldFunction) {
	super(stateTable, keySerializer, valueSerializer, namespaceSerializer, defaultValue);
	this.foldTransformation = new FoldTransformation(foldFunction);
}
 
Example #11
Source File: FoldingStateDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@code FoldingStateDescriptor} with the given name and default value.
 *
 * @param name The (unique) name for the state.
 * @param initialValue The initial value of the fold.
 * @param foldFunction The {@code FoldFunction} used to aggregate the state.
 * @param typeInfo The type of the values in the state.
 */
public FoldingStateDescriptor(String name, ACC initialValue, FoldFunction<T, ACC> foldFunction, TypeInformation<ACC> typeInfo) {
	super(name, typeInfo, initialValue);
	this.foldFunction = requireNonNull(foldFunction);

	if (foldFunction instanceof RichFunction) {
		throw new UnsupportedOperationException("FoldFunction of FoldingState can not be a RichFunction.");
	}
}
 
Example #12
Source File: FoldingStateDescriptor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@code FoldingStateDescriptor} with the given name and default value.
 *
 * @param name The (unique) name for the state.
 * @param initialValue The initial value of the fold.
 * @param foldFunction The {@code FoldFunction} used to aggregate the state.
 * @param typeInfo The type of the values in the state.
 */
public FoldingStateDescriptor(String name, ACC initialValue, FoldFunction<T, ACC> foldFunction, TypeInformation<ACC> typeInfo) {
	super(name, typeInfo, initialValue);
	this.foldFunction = requireNonNull(foldFunction);

	if (foldFunction instanceof RichFunction) {
		throw new UnsupportedOperationException("FoldFunction of FoldingState can not be a RichFunction.");
	}
}
 
Example #13
Source File: TypeExtractor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * @deprecated will be removed in a future version
 */
@PublicEvolving
@Deprecated
public static <IN, OUT> TypeInformation<OUT> getFoldReturnTypes(FoldFunction<IN, OUT> foldInterface, TypeInformation<IN> inType, String functionName, boolean allowMissing)
{
	return getUnaryOperatorReturnType(
		(Function) foldInterface,
		FoldFunction.class,
		0,
		1,
		NO_INDEX,
		inType,
		functionName,
		allowMissing);
}
 
Example #14
Source File: TypeExtractor.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * @deprecated will be removed in a future version
 */
@PublicEvolving
@Deprecated
public static <IN, OUT> TypeInformation<OUT> getFoldReturnTypes(FoldFunction<IN, OUT> foldInterface, TypeInformation<IN> inType)
{
	return getFoldReturnTypes(foldInterface, inType, null, false);
}
 
Example #15
Source File: UdfStreamOperatorCheckpointingITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Assembles a stream of a grouping field and some long data. Applies reduce functions
 * on this stream.
 */
@Override
public void testProgram(StreamExecutionEnvironment env) {

	// base stream
	KeyedStream<Tuple2<Integer, Long>, Tuple> stream = env.addSource(new StatefulMultipleSequence())
			.keyBy(0);

	stream
			// testing built-in aggregate
			.min(1)
			// failure generation
			.map(new OnceFailingIdentityMapFunction(NUM_INPUT))
			.keyBy(0)
			.addSink(new MinEvictingQueueSink());

	stream
			// testing UDF reducer
			.reduce(new ReduceFunction<Tuple2<Integer, Long>>() {
				@Override
				public Tuple2<Integer, Long> reduce(
						Tuple2<Integer, Long> value1, Tuple2<Integer, Long> value2) throws Exception {
					return Tuple2.of(value1.f0, value1.f1 + value2.f1);
				}
			})
			.keyBy(0)
			.addSink(new SumEvictingQueueSink());

	stream
			// testing UDF folder
			.fold(Tuple2.of(0, 0L), new FoldFunction<Tuple2<Integer, Long>, Tuple2<Integer, Long>>() {
				@Override
				public Tuple2<Integer, Long> fold(
						Tuple2<Integer, Long> accumulator, Tuple2<Integer, Long> value) throws Exception {
					return Tuple2.of(value.f0, accumulator.f1 + value.f1);
				}
			})
			.keyBy(0)
			.addSink(new FoldEvictingQueueSink());
}
 
Example #16
Source File: AllWindowedStream.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Applies the given fold function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the reduce function is
 * interpreted as a regular non-windowed stream.
 *
 * @param function The fold function.
 * @return The data stream that is the result of applying the fold function to the window.
 *
 * @deprecated use {@link #aggregate(AggregateFunction)} instead
 */
@Deprecated
public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R> function) {
	if (function instanceof RichFunction) {
		throw new UnsupportedOperationException("FoldFunction of fold can not be a RichFunction. " +
				"Please use fold(FoldFunction, WindowFunction) instead.");
	}

	TypeInformation<R> resultType = TypeExtractor.getFoldReturnTypes(function, input.getType(),
			Utils.getCallLocationName(), true);

	return fold(initialValue, function, resultType);
}
 
Example #17
Source File: WindowedStream.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Applies the given window function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the window function is
 * interpreted as a regular non-windowed stream.
 *
 * <p>Arriving data is incrementally aggregated using the given fold function.
 *
 * @param initialValue The initial value of the fold.
 * @param foldFunction The fold function that is used for incremental aggregation.
 * @param windowFunction The window function.
 * @return The data stream that is the result of applying the window function to the window.
 *
 * @deprecated use {@link #aggregate(AggregateFunction, WindowFunction)} instead
 */
@PublicEvolving
@Deprecated
public <R, ACC> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunction<T, ACC> foldFunction, ProcessWindowFunction<ACC, R, K, W> windowFunction) {
	if (foldFunction instanceof RichFunction) {
		throw new UnsupportedOperationException("FoldFunction can not be a RichFunction.");
	}

	TypeInformation<ACC> foldResultType = TypeExtractor.getFoldReturnTypes(foldFunction, input.getType(),
			Utils.getCallLocationName(), true);

	TypeInformation<R> windowResultType = getProcessWindowFunctionReturnType(windowFunction, foldResultType, Utils.getCallLocationName());

	return fold(initialValue, foldFunction, windowFunction, foldResultType, windowResultType);
}
 
Example #18
Source File: TypeExtractor.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * @deprecated will be removed in a future version
 */
@PublicEvolving
@Deprecated
public static <IN, OUT> TypeInformation<OUT> getFoldReturnTypes(FoldFunction<IN, OUT> foldInterface, TypeInformation<IN> inType, String functionName, boolean allowMissing)
{
	return getUnaryOperatorReturnType(
		(Function) foldInterface,
		FoldFunction.class,
		0,
		1,
		NO_INDEX,
		inType,
		functionName,
		allowMissing);
}
 
Example #19
Source File: FoldingStateDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@code ValueStateDescriptor} with the given name and default value.
 *
 * @param name The (unique) name for the state.
 * @param initialValue The initial value of the fold.
 * @param foldFunction The {@code FoldFunction} used to aggregate the state.
 * @param typeSerializer The type serializer of the values in the state.
 */
public FoldingStateDescriptor(String name, ACC initialValue, FoldFunction<T, ACC> foldFunction, TypeSerializer<ACC> typeSerializer) {
	super(name, typeSerializer, initialValue);
	this.foldFunction = requireNonNull(foldFunction);

	if (foldFunction instanceof RichFunction) {
		throw new UnsupportedOperationException("FoldFunction of FoldingState can not be a RichFunction.");
	}
}
 
Example #20
Source File: UdfStreamOperatorCheckpointingITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Assembles a stream of a grouping field and some long data. Applies reduce functions
 * on this stream.
 */
@Override
public void testProgram(StreamExecutionEnvironment env) {

	// base stream
	KeyedStream<Tuple2<Integer, Long>, Tuple> stream = env.addSource(new StatefulMultipleSequence())
			.keyBy(0);

	stream
			// testing built-in aggregate
			.min(1)
			// failure generation
			.map(new OnceFailingIdentityMapFunction(NUM_INPUT))
			.keyBy(0)
			.addSink(new MinEvictingQueueSink());

	stream
			// testing UDF reducer
			.reduce(new ReduceFunction<Tuple2<Integer, Long>>() {
				@Override
				public Tuple2<Integer, Long> reduce(
						Tuple2<Integer, Long> value1, Tuple2<Integer, Long> value2) throws Exception {
					return Tuple2.of(value1.f0, value1.f1 + value2.f1);
				}
			})
			.keyBy(0)
			.addSink(new SumEvictingQueueSink());

	stream
			// testing UDF folder
			.fold(Tuple2.of(0, 0L), new FoldFunction<Tuple2<Integer, Long>, Tuple2<Integer, Long>>() {
				@Override
				public Tuple2<Integer, Long> fold(
						Tuple2<Integer, Long> accumulator, Tuple2<Integer, Long> value) throws Exception {
					return Tuple2.of(value.f0, accumulator.f1 + value.f1);
				}
			})
			.keyBy(0)
			.addSink(new FoldEvictingQueueSink());
}
 
Example #21
Source File: StreamingRuntimeContextTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFoldingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	@SuppressWarnings("unchecked")
	FoldFunction<String, TaskInfo> folder = (FoldFunction<String, TaskInfo>) mock(FoldFunction.class);

	FoldingStateDescriptor<String, TaskInfo> descr =
			new FoldingStateDescriptor<>("name", null, folder, TaskInfo.class);

	context.getFoldingState(descr);

	FoldingStateDescriptor<?, ?> descrIntercepted = (FoldingStateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #22
Source File: WindowedStream.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Applies the given fold function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the reduce function is
 * interpreted as a regular non-windowed stream.
 *
 * @param function The fold function.
 * @return The data stream that is the result of applying the fold function to the window.
 *
 * @deprecated use {@link #aggregate(AggregationFunction)} instead
 */
@Deprecated
public <R> SingleOutputStreamOperator<R> fold(R initialValue, FoldFunction<T, R> function) {
	if (function instanceof RichFunction) {
		throw new UnsupportedOperationException("FoldFunction can not be a RichFunction. " +
			"Please use fold(FoldFunction, WindowFunction) instead.");
	}

	TypeInformation<R> resultType = TypeExtractor.getFoldReturnTypes(function, input.getType(),
			Utils.getCallLocationName(), true);

	return fold(initialValue, function, resultType);
}
 
Example #23
Source File: FoldApplyAllWindowFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
public FoldApplyAllWindowFunction(ACC initialValue,
		FoldFunction<T, ACC> foldFunction,
		AllWindowFunction<ACC, R, W> windowFunction,
		TypeInformation<ACC> accTypeInformation) {
	super(windowFunction);
	this.accTypeInformation = accTypeInformation;
	this.foldFunction = foldFunction;
	this.initialValue = initialValue;
}
 
Example #24
Source File: TypeExtractor.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * @deprecated will be removed in a future version
 */
@PublicEvolving
@Deprecated
public static <IN, OUT> TypeInformation<OUT> getFoldReturnTypes(FoldFunction<IN, OUT> foldInterface, TypeInformation<IN> inType, String functionName, boolean allowMissing)
{
	return getUnaryOperatorReturnType(
		(Function) foldInterface,
		FoldFunction.class,
		0,
		1,
		NO_INDEX,
		inType,
		functionName,
		allowMissing);
}
 
Example #25
Source File: StreamingRuntimeContextTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testFoldingStateInstantiation() throws Exception {

	final ExecutionConfig config = new ExecutionConfig();
	config.registerKryoType(Path.class);

	final AtomicReference<Object> descriptorCapture = new AtomicReference<>();

	StreamingRuntimeContext context = new StreamingRuntimeContext(
			createDescriptorCapturingMockOp(descriptorCapture, config),
			createMockEnvironment(),
			Collections.<String, Accumulator<?, ?>>emptyMap());

	@SuppressWarnings("unchecked")
	FoldFunction<String, TaskInfo> folder = (FoldFunction<String, TaskInfo>) mock(FoldFunction.class);

	FoldingStateDescriptor<String, TaskInfo> descr =
			new FoldingStateDescriptor<>("name", null, folder, TaskInfo.class);

	context.getFoldingState(descr);

	FoldingStateDescriptor<?, ?> descrIntercepted = (FoldingStateDescriptor<?, ?>) descriptorCapture.get();
	TypeSerializer<?> serializer = descrIntercepted.getSerializer();

	// check that the Path class is really registered, i.e., the execution config was applied
	assertTrue(serializer instanceof KryoSerializer);
	assertTrue(((KryoSerializer<?>) serializer).getKryo().getRegistration(Path.class).getId() > 0);
}
 
Example #26
Source File: TypeExtractor.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * @deprecated will be removed in a future version
 */
@PublicEvolving
@Deprecated
public static <IN, OUT> TypeInformation<OUT> getFoldReturnTypes(FoldFunction<IN, OUT> foldInterface, TypeInformation<IN> inType)
{
	return getFoldReturnTypes(foldInterface, inType, null, false);
}
 
Example #27
Source File: WindowedStream.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Applies the given window function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the window function is
 * interpreted as a regular non-windowed stream.
 *
 * <p>Arriving data is incrementally aggregated using the given fold function.
 *
 * @param initialValue The initial value of the fold.
 * @param foldFunction The fold function that is used for incremental aggregation.
 * @param windowFunction The window function.
 * @return The data stream that is the result of applying the window function to the window.
 *
 * @deprecated use {@link #aggregate(AggregateFunction, WindowFunction)} instead
 */
@PublicEvolving
@Deprecated
public <R, ACC> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunction<T, ACC> foldFunction, ProcessWindowFunction<ACC, R, K, W> windowFunction) {
	if (foldFunction instanceof RichFunction) {
		throw new UnsupportedOperationException("FoldFunction can not be a RichFunction.");
	}

	TypeInformation<ACC> foldResultType = TypeExtractor.getFoldReturnTypes(foldFunction, input.getType(),
			Utils.getCallLocationName(), true);

	TypeInformation<R> windowResultType = getProcessWindowFunctionReturnType(windowFunction, foldResultType, Utils.getCallLocationName());

	return fold(initialValue, foldFunction, windowFunction, foldResultType, windowResultType);
}
 
Example #28
Source File: FoldApplyAllWindowFunction.java    From flink with Apache License 2.0 5 votes vote down vote up
public FoldApplyAllWindowFunction(ACC initialValue,
		FoldFunction<T, ACC> foldFunction,
		AllWindowFunction<ACC, R, W> windowFunction,
		TypeInformation<ACC> accTypeInformation) {
	super(windowFunction);
	this.accTypeInformation = accTypeInformation;
	this.foldFunction = foldFunction;
	this.initialValue = initialValue;
}
 
Example #29
Source File: UdfStreamOperatorCheckpointingITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Assembles a stream of a grouping field and some long data. Applies reduce functions
 * on this stream.
 */
@Override
public void testProgram(StreamExecutionEnvironment env) {

	// base stream
	KeyedStream<Tuple2<Integer, Long>, Tuple> stream = env.addSource(new StatefulMultipleSequence())
			.keyBy(0);

	stream
			// testing built-in aggregate
			.min(1)
			// failure generation
			.map(new OnceFailingIdentityMapFunction(NUM_INPUT))
			.keyBy(0)
			.addSink(new MinEvictingQueueSink());

	stream
			// testing UDF reducer
			.reduce(new ReduceFunction<Tuple2<Integer, Long>>() {
				@Override
				public Tuple2<Integer, Long> reduce(
						Tuple2<Integer, Long> value1, Tuple2<Integer, Long> value2) throws Exception {
					return Tuple2.of(value1.f0, value1.f1 + value2.f1);
				}
			})
			.keyBy(0)
			.addSink(new SumEvictingQueueSink());

	stream
			// testing UDF folder
			.fold(Tuple2.of(0, 0L), new FoldFunction<Tuple2<Integer, Long>, Tuple2<Integer, Long>>() {
				@Override
				public Tuple2<Integer, Long> fold(
						Tuple2<Integer, Long> accumulator, Tuple2<Integer, Long> value) throws Exception {
					return Tuple2.of(value.f0, accumulator.f1 + value.f1);
				}
			})
			.keyBy(0)
			.addSink(new FoldEvictingQueueSink());
}
 
Example #30
Source File: WindowedStream.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Applies the given window function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the window function is
 * interpreted as a regular non-windowed stream.
 *
 * <p>Arriving data is incrementally aggregated using the given fold function.
 *
 * @param initialValue The initial value of the fold.
 * @param foldFunction The fold function that is used for incremental aggregation.
 * @param windowFunction The window function.
 * @return The data stream that is the result of applying the window function to the window.
 *
 * @deprecated use {@link #aggregate(AggregateFunction, WindowFunction)} instead
 */
@PublicEvolving
@Deprecated
public <R, ACC> SingleOutputStreamOperator<R> fold(ACC initialValue, FoldFunction<T, ACC> foldFunction, ProcessWindowFunction<ACC, R, K, W> windowFunction) {
	if (foldFunction instanceof RichFunction) {
		throw new UnsupportedOperationException("FoldFunction can not be a RichFunction.");
	}

	TypeInformation<ACC> foldResultType = TypeExtractor.getFoldReturnTypes(foldFunction, input.getType(),
			Utils.getCallLocationName(), true);

	TypeInformation<R> windowResultType = getProcessWindowFunctionReturnType(windowFunction, foldResultType, Utils.getCallLocationName());

	return fold(initialValue, foldFunction, windowFunction, foldResultType, windowResultType);
}