org.apache.flink.api.common.functions.util.FunctionUtils Java Examples

The following examples show how to use org.apache.flink.api.common.functions.util.FunctionUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BatchTask.java    From flink with Apache License 2.0 6 votes vote down vote up
protected S initStub(Class<? super S> stubSuperClass) throws Exception {
	try {
		ClassLoader userCodeClassLoader = getUserCodeClassLoader();
		S stub = config.<S>getStubWrapper(userCodeClassLoader).getUserCodeObject(stubSuperClass, userCodeClassLoader);
		// check if the class is a subclass, if the check is required
		if (stubSuperClass != null && !stubSuperClass.isAssignableFrom(stub.getClass())) {
			throw new RuntimeException("The class '" + stub.getClass().getName() + "' is not a subclass of '" + 
					stubSuperClass.getName() + "' as is required.");
		}
		FunctionUtils.setFunctionRuntimeContext(stub, this.runtimeUdfContext);
		return stub;
	}
	catch (ClassCastException ccex) {
		throw new Exception("The stub class is not a proper subclass of " + stubSuperClass.getName(), ccex);
	}
}
 
Example #2
Source File: FlatMapOperatorBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected List<OUT> executeOnCollections(List<IN> input, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	FlatMapFunction<IN, OUT> function = userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, parameters);

	ArrayList<OUT> result = new ArrayList<OUT>(input.size());

	TypeSerializer<IN> inSerializer = getOperatorInfo().getInputType().createSerializer(executionConfig);
	TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

	CopyingListCollector<OUT> resultCollector = new CopyingListCollector<OUT>(result, outSerializer);

	for (IN element : input) {
		IN inCopy = inSerializer.copy(element);
		function.flatMap(inCopy, resultCollector);
	}

	FunctionUtils.closeFunction(function);

	return result;
}
 
Example #3
Source File: MapPartitionOperatorBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
protected List<OUT> executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	MapPartitionFunction<IN, OUT> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);
	
	ArrayList<OUT> result = new ArrayList<OUT>(inputData.size() / 4);

	TypeSerializer<IN> inSerializer = getOperatorInfo().getInputType().createSerializer(executionConfig);
	TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

	CopyingIterator<IN> source = new CopyingIterator<IN>(inputData.iterator(), inSerializer);
	CopyingListCollector<OUT> resultCollector = new CopyingListCollector<OUT>(result, outSerializer);

	function.mapPartition(source, resultCollector);

	result.trimToSize();
	FunctionUtils.closeFunction(function);
	return result;
}
 
Example #4
Source File: FilterOperatorBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
protected List<T> executeOnCollections(List<T> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	FlatMapFunction<T, T> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);
	
	ArrayList<T> result = new ArrayList<T>(inputData.size());
	ListCollector<T> collector = new ListCollector<T>(result);

	for (T element : inputData) {
		function.flatMap(element, collector);
	}
	
	FunctionUtils.closeFunction(function);
	
	return result;
}
 
Example #5
Source File: FilterOperatorBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected List<T> executeOnCollections(List<T> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	FlatMapFunction<T, T> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);
	
	ArrayList<T> result = new ArrayList<T>(inputData.size());
	ListCollector<T> collector = new ListCollector<T>(result);

	for (T element : inputData) {
		function.flatMap(element, collector);
	}
	
	FunctionUtils.closeFunction(function);
	
	return result;
}
 
Example #6
Source File: CrossOperatorBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
protected List<OUT> executeOnCollections(List<IN1> inputData1, List<IN2> inputData2, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	CrossFunction<IN1, IN2, OUT> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);

	ArrayList<OUT> result = new ArrayList<OUT>(inputData1.size() * inputData2.size());
	
	TypeSerializer<IN1> inSerializer1 = getOperatorInfo().getFirstInputType().createSerializer(executionConfig);
	TypeSerializer<IN2> inSerializer2 = getOperatorInfo().getSecondInputType().createSerializer(executionConfig);
	TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

	for (IN1 element1 : inputData1) {
		for (IN2 element2 : inputData2) {
			IN1 copy1 = inSerializer1.copy(element1);
			IN2 copy2 = inSerializer2.copy(element2);
			OUT o = function.cross(copy1, copy2);
			result.add(outSerializer.copy(o));
		}
	}

	FunctionUtils.closeFunction(function);
	return result;
}
 
Example #7
Source File: AbstractMapBundleOperator.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void close() throws Exception {
	try {
		finishBundle();
	} finally {
		Exception exception = null;

		try {
			super.close();
			if (function != null) {
				FunctionUtils.closeFunction(function);
			}
		} catch (InterruptedException interrupted) {
			exception = interrupted;

			Thread.currentThread().interrupt();
		} catch (Exception e) {
			exception = e;
		}

		if (exception != null) {
			LOG.warn("Errors occurred while closing the BundleOperator.", exception);
		}
	}
}
 
Example #8
Source File: MapOperatorBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected List<OUT> executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	MapFunction<IN, OUT> function = this.userFunction.getUserCodeObject();
	
	FunctionUtils.setFunctionRuntimeContext(function, ctx);
	FunctionUtils.openFunction(function, this.parameters);
	
	ArrayList<OUT> result = new ArrayList<OUT>(inputData.size());

	TypeSerializer<IN> inSerializer = getOperatorInfo().getInputType().createSerializer(executionConfig);
	TypeSerializer<OUT> outSerializer = getOperatorInfo().getOutputType().createSerializer(executionConfig);

	for (IN element : inputData) {
		IN inCopy = inSerializer.copy(element);
		OUT out = function.map(inCopy);
		result.add(outSerializer.copy(out));
	}

	FunctionUtils.closeFunction(function);
	
	return result;
}
 
Example #9
Source File: BatchTask.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
protected S initStub(Class<? super S> stubSuperClass) throws Exception {
	try {
		ClassLoader userCodeClassLoader = getUserCodeClassLoader();
		S stub = config.<S>getStubWrapper(userCodeClassLoader).getUserCodeObject(stubSuperClass, userCodeClassLoader);
		// check if the class is a subclass, if the check is required
		if (stubSuperClass != null && !stubSuperClass.isAssignableFrom(stub.getClass())) {
			throw new RuntimeException("The class '" + stub.getClass().getName() + "' is not a subclass of '" + 
					stubSuperClass.getName() + "' as is required.");
		}
		FunctionUtils.setFunctionRuntimeContext(stub, this.runtimeUdfContext);
		return stub;
	}
	catch (ClassCastException ccex) {
		throw new Exception("The stub class is not a proper subclass of " + stubSuperClass.getName(), ccex);
	}
}
 
Example #10
Source File: FoldApplyProcessAllWindowFunction.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void open(Configuration configuration) throws Exception {
	FunctionUtils.openFunction(this.windowFunction, configuration);

	if (serializedInitialValue == null) {
		throw new RuntimeException("No initial value was serialized for the fold " +
			"window function. Probably the setOutputType method was not called.");
	}

	ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
	DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
	initialValue = accSerializer.deserialize(in);

	ctx = new InternalProcessApplyAllWindowContext<>(windowFunction);

}
 
Example #11
Source File: LookupJoinRunner.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void close() throws Exception {
	super.close();
	if (fetcher != null) {
		FunctionUtils.closeFunction(fetcher);
	}
	if (collector != null) {
		FunctionUtils.closeFunction(collector);
	}
}
 
Example #12
Source File: AsyncLookupJoinRunner.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.fetcher = generatedFetcher.newInstance(getRuntimeContext().getUserCodeClassLoader());
	FunctionUtils.setFunctionRuntimeContext(fetcher, getRuntimeContext());
	FunctionUtils.openFunction(fetcher, parameters);

	// try to compile the generated ResultFuture, fail fast if the code is corrupt.
	generatedResultFuture.compile(getRuntimeContext().getUserCodeClassLoader());

	// row converter is stateless which is thread-safe
	RowConverter rowConverter;
	if (fetcherReturnType instanceof RowTypeInfo) {
		rowConverter = (RowConverter) DataFormatConverters.getConverterForDataType(
				fromLegacyInfoToDataType(fetcherReturnType));
	} else if (fetcherReturnType instanceof BaseRowTypeInfo) {
		rowConverter = null;
	} else {
		throw new IllegalStateException("This should never happen, " +
			"currently fetcherReturnType can only be BaseRowTypeInfo or RowTypeInfo");
	}

	// asyncBufferCapacity + 1 as the queue size in order to avoid
	// blocking on the queue when taking a collector.
	this.resultFutureBuffer = new ArrayBlockingQueue<>(asyncBufferCapacity + 1);
	this.allResultFutures = new ArrayList<>();
	for (int i = 0; i < asyncBufferCapacity + 1; i++) {
		JoinedRowResultFuture rf = new JoinedRowResultFuture(
			resultFutureBuffer,
			createFetcherResultFuture(parameters),
			rowConverter,
			isLeftOuterJoin,
			rightRowTypeInfo.getArity());
		// add will throw exception immediately if the queue is full which should never happen
		resultFutureBuffer.add(rf);
		allResultFutures.add(rf);
	}
}
 
Example #13
Source File: KeyedStateInputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void open(KeyGroupRangeInputSplit split) throws IOException {
	registry = new CloseableRegistry();

	final Environment environment = new SavepointEnvironment
		.Builder(getRuntimeContext(), split.getNumKeyGroups())
		.setSubtaskIndex(split.getSplitNumber())
		.setPrioritizedOperatorSubtaskState(split.getPrioritizedOperatorSubtaskState())
		.build();

	final StreamOperatorStateContext context = getStreamOperatorStateContext(environment);

	keyedStateBackend = (AbstractKeyedStateBackend<K>) context.keyedStateBackend();

	final DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, getRuntimeContext().getExecutionConfig());
	SavepointRuntimeContext ctx = new SavepointRuntimeContext(getRuntimeContext(), keyedStateStore);
	FunctionUtils.setFunctionRuntimeContext(userFunction, ctx);

	keys = getKeyIterator(ctx);

	final InternalTimerService<VoidNamespace> timerService = restoreTimerService(context);
	try {
		this.ctx = new Context(keyedStateBackend, timerService);
	} catch (Exception e) {
		throw new IOException("Failed to restore timer state", e);
	}
}
 
Example #14
Source File: GroupCombineChainedDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	this.parent = parent;

	@SuppressWarnings("unchecked")
	final GroupReduceFunction<IN, OUT> combiner =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, GroupReduceFunction.class);
	this.reducer = combiner;
	FunctionUtils.setFunctionRuntimeContext(combiner, getUdfRuntimeContext());
}
 
Example #15
Source File: AsyncLookupJoinRunner.java    From flink with Apache License 2.0 5 votes vote down vote up
public TableFunctionResultFuture<BaseRow> createFetcherResultFuture(Configuration parameters) throws Exception {
	TableFunctionResultFuture<BaseRow> resultFuture = generatedResultFuture.newInstance(
		getRuntimeContext().getUserCodeClassLoader());
	FunctionUtils.setFunctionRuntimeContext(resultFuture, getRuntimeContext());
	FunctionUtils.openFunction(resultFuture, parameters);
	return resultFuture;
}
 
Example #16
Source File: LookupJoinRunner.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.fetcher = generatedFetcher.newInstance(getRuntimeContext().getUserCodeClassLoader());
	this.collector = generatedCollector.newInstance(getRuntimeContext().getUserCodeClassLoader());

	FunctionUtils.setFunctionRuntimeContext(fetcher, getRuntimeContext());
	FunctionUtils.setFunctionRuntimeContext(collector, getRuntimeContext());
	FunctionUtils.openFunction(fetcher, parameters);
	FunctionUtils.openFunction(collector, parameters);

	this.nullRow = new GenericRow(tableFieldsCount);
	this.outRow = new JoinedRow();
}
 
Example #17
Source File: LookupJoinWithCalcRunner.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.calc = generatedCalc.newInstance(getRuntimeContext().getUserCodeClassLoader());
	FunctionUtils.setFunctionRuntimeContext(calc, getRuntimeContext());
	FunctionUtils.openFunction(calc, parameters);
	this.calcCollector = new CalcCollector(collector);
}
 
Example #18
Source File: RichCompositeIterativeCondition.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	for (IterativeCondition<T> nestedCondition : nestedConditions) {
		FunctionUtils.openFunction(nestedCondition, parameters);
	}
}
 
Example #19
Source File: AsyncLookupJoinWithCalcRunner.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public TableFunctionResultFuture<BaseRow> createFetcherResultFuture(Configuration parameters) throws Exception {
	TableFunctionResultFuture<BaseRow> joinConditionCollector = super.createFetcherResultFuture(parameters);
	FlatMapFunction<BaseRow, BaseRow> calc = generatedCalc.newInstance(getRuntimeContext().getUserCodeClassLoader());
	FunctionUtils.setFunctionRuntimeContext(calc, getRuntimeContext());
	FunctionUtils.openFunction(calc, parameters);
	return new TemporalTableCalcResultFuture(calc, joinConditionCollector);
}
 
Example #20
Source File: ChainedFlatMapDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void cancelTask() {
	try {
		FunctionUtils.closeFunction(this.mapper);
	}
	catch (Throwable t) {
	}
}
 
Example #21
Source File: RichCompositeIterativeCondition.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void close() throws Exception {
	super.close();
	for (IterativeCondition<T> nestedCondition : nestedConditions) {
		FunctionUtils.closeFunction(nestedCondition);
	}
}
 
Example #22
Source File: ChainedMapDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	@SuppressWarnings("unchecked")
	final MapFunction<IT, OT> mapper =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, MapFunction.class);
	this.mapper = mapper;
	FunctionUtils.setFunctionRuntimeContext(mapper, getUdfRuntimeContext());
}
 
Example #23
Source File: AbstractUdfStreamOperator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void dispose() throws Exception {
	super.dispose();
	if (!functionsClosed) {
		functionsClosed = true;
		FunctionUtils.closeFunction(userFunction);
	}
}
 
Example #24
Source File: FoldApplyProcessWindowFunction.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration configuration) throws Exception {
	FunctionUtils.openFunction(this.windowFunction, configuration);

	if (serializedInitialValue == null) {
		throw new RuntimeException("No initial value was serialized for the fold " +
			"window function. Probably the setOutputType method was not called.");
	}

	ByteArrayInputStream bais = new ByteArrayInputStream(serializedInitialValue);
	DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais);
	initialValue = accSerializer.deserialize(in);

	ctx = new InternalProcessApplyWindowContext<>(windowFunction);
}
 
Example #25
Source File: ChainedMapDriver.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void cancelTask() {
	try {
		FunctionUtils.closeFunction(this.mapper);
	} catch (Throwable t) {
	}
}
 
Example #26
Source File: ChainedReduceCombineDriver.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	this.parent = parent;
	running = true;

	strategy = config.getDriverStrategy();

	reducer = BatchTask.instantiateUserCode(config, userCodeClassLoader, ReduceFunction.class);
	FunctionUtils.setFunctionRuntimeContext(reducer, getUdfRuntimeContext());
}
 
Example #27
Source File: ChainedMapDriver.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void cancelTask() {
	try {
		FunctionUtils.closeFunction(this.mapper);
	} catch (Throwable t) {
	}
}
 
Example #28
Source File: ChainedMapDriver.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	@SuppressWarnings("unchecked")
	final MapFunction<IT, OT> mapper =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, MapFunction.class);
	this.mapper = mapper;
	FunctionUtils.setFunctionRuntimeContext(mapper, getUdfRuntimeContext());
}
 
Example #29
Source File: GroupCombineChainedDriver.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void setup(AbstractInvokable parent) {
	this.parent = parent;

	@SuppressWarnings("unchecked")
	final GroupReduceFunction<IN, OUT> combiner =
		BatchTask.instantiateUserCode(this.config, userCodeClassLoader, GroupReduceFunction.class);
	this.reducer = combiner;
	FunctionUtils.setFunctionRuntimeContext(combiner, getUdfRuntimeContext());
}
 
Example #30
Source File: ChainedAllReduceDriver.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void cancelTask() {
	try {
		FunctionUtils.closeFunction(this.reducer);
	} catch (Throwable t) {
	}
}