org.apache.flink.api.common.operators.util.UserCodeWrapper Java Examples

The following examples show how to use org.apache.flink.api.common.operators.util.UserCodeWrapper. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: flink   Author: apache   File: JobGraphGenerator.java    License: Apache License 2.0 6 votes vote down vote up
private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException {
	final InputOutputFormatVertex vertex = new InputOutputFormatVertex(node.getNodeName());
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());

	final OperatorID operatorID = new OperatorID();

	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass(DataSinkTask.class);
	vertex.setFormatDescription(operatorID, getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));

	// set user code
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, (UserCodeWrapper<? extends OutputFormat<?>>) node.getProgramOperator().getUserCodeWrapper())
		.addParameters(operatorID, node.getProgramOperator().getParameters())
		.write(config);

	return vertex;
}
 
Example #2
Source Project: flink   Author: flink-tpc-ds   File: InputOutputFormatContainerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOnlyInputFormat() {
	InputOutputFormatContainer formatContainer = new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader());

	OperatorID operatorID = new OperatorID();
	formatContainer.addInputFormat(operatorID, new TestInputFormat("test input format"));
	formatContainer.addParameters(operatorID, "parameter1", "abc123");

	TaskConfig taskConfig = new TaskConfig(new Configuration());
	formatContainer.write(taskConfig);

	InputOutputFormatContainer loadedFormatContainer = new InputOutputFormatContainer(taskConfig, getClass().getClassLoader());

	Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = loadedFormatContainer.getInputFormats();
	assertEquals(1, inputFormats.size());
	assertEquals(0, loadedFormatContainer.getOutputFormats().size());

	TestInputFormat inputFormat = (TestInputFormat) inputFormats.get(operatorID).getUserCodeObject();
	assertEquals("test input format", inputFormat.getName());

	Configuration parameters = loadedFormatContainer.getParameters(operatorID);
	assertEquals(1, parameters.keySet().size());
	assertEquals("abc123", parameters.getString("parameter1", null));
}
 
Example #3
Source Project: flink   Author: flink-tpc-ds   File: JobGraphGenerator.java    License: Apache License 2.0 6 votes vote down vote up
private JobVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException {
	final InputOutputFormatVertex vertex = new InputOutputFormatVertex(node.getNodeName());
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());

	final OperatorID operatorID = new OperatorID();

	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass(DataSourceTask.class);
	vertex.setFormatDescription(operatorID, getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));

	// set user code
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addInputFormat(operatorID, (UserCodeWrapper<? extends InputFormat<?, ?>>) node.getProgramOperator().getUserCodeWrapper())
		.addParameters(operatorID, node.getProgramOperator().getParameters())
		.write(config);

	config.setOutputSerializer(node.getSerializer());
	return vertex;
}
 
Example #4
Source Project: flink   Author: flink-tpc-ds   File: JobGraphGenerator.java    License: Apache License 2.0 6 votes vote down vote up
private JobVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException {
	final InputOutputFormatVertex vertex = new InputOutputFormatVertex(node.getNodeName());
	final TaskConfig config = new TaskConfig(vertex.getConfiguration());

	final OperatorID operatorID = new OperatorID();

	vertex.setResources(node.getMinResources(), node.getPreferredResources());
	vertex.setInvokableClass(DataSinkTask.class);
	vertex.setFormatDescription(operatorID, getDescriptionForUserCode(node.getProgramOperator().getUserCodeWrapper()));

	// set user code
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, (UserCodeWrapper<? extends OutputFormat<?>>) node.getProgramOperator().getUserCodeWrapper())
		.addParameters(operatorID, node.getProgramOperator().getParameters())
		.write(config);

	return vertex;
}
 
Example #5
Source Project: Flink-CEPplus   Author: ljygz   File: CrossOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
public CrossOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, String name) {
	super(udf, operatorInfo, name);
	
	if (this instanceof CrossWithSmall) {
		setCrossHint(CrossHint.SECOND_IS_SMALL);
	}
	else if (this instanceof CrossWithLarge) {
		setCrossHint(CrossHint.FIRST_IS_SMALL);
	}
}
 
Example #6
Source Project: flink   Author: apache   File: CrossOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
public CrossOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, String name) {
	super(udf, operatorInfo, name);
	
	if (this instanceof CrossWithSmall) {
		setCrossHint(CrossHint.SECOND_IS_SMALL);
	}
	else if (this instanceof CrossWithLarge) {
		setCrossHint(CrossHint.FIRST_IS_SMALL);
	}
}
 
Example #7
Source Project: flink   Author: apache   File: InputOutputFormatVertex.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void finalizeOnMaster(ClassLoader loader) throws Exception {
	final InputOutputFormatContainer formatContainer = initInputOutputformatContainer(loader);

	final ClassLoader original = Thread.currentThread().getContextClassLoader();
	try {
		// set user classloader before calling user code
		Thread.currentThread().setContextClassLoader(loader);

		// configure input formats and invoke finalizeGlobal()
		Map<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> outputFormats = formatContainer.getOutputFormats();
		for (Map.Entry<OperatorID, UserCodeWrapper<? extends OutputFormat<?>>> entry : outputFormats.entrySet()) {
			final OutputFormat<?> outputFormat;

			try {
				outputFormat = entry.getValue().getUserCodeObject();
				outputFormat.configure(formatContainer.getParameters(entry.getKey()));
			} catch (Throwable t) {
				throw new Exception("Configuring the output format (" + getFormatDescription(entry.getKey()) + ") failed: "
					+ t.getMessage(), t);
			}

			if (outputFormat instanceof FinalizeOnMaster) {
				((FinalizeOnMaster) outputFormat).finalizeGlobal(getParallelism());
			}
		}

	} finally {
		// restore original classloader
		Thread.currentThread().setContextClassLoader(original);
	}
}
 
Example #8
Source Project: Flink-CEPplus   Author: ljygz   File: TaskConfig.java    License: Apache License 2.0 5 votes vote down vote up
public void setStubWrapper(UserCodeWrapper<?> wrapper) {
	try {
		InstantiationUtil.writeObjectToConfig(wrapper, this.config, STUB_OBJECT);
	} catch (IOException e) {
		throw new CorruptConfigurationException("Could not write the user code wrapper " + wrapper.getClass() + " : " + e.toString(), e);
	}
}
 
Example #9
Source Project: Flink-CEPplus   Author: ljygz   File: TaskConfig.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public <T> UserCodeWrapper<T> getStubWrapper(ClassLoader cl) {
	try {
		return (UserCodeWrapper<T>) InstantiationUtil.readObjectFromConfig(this.config, STUB_OBJECT, cl);
	} catch (ClassNotFoundException | IOException e) {
		throw new CorruptConfigurationException("Could not read the user code wrapper: " + e.getMessage(), e);
	}
}
 
Example #10
Source Project: flink   Author: apache   File: TaskConfig.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public <T> UserCodeWrapper<T> getStubWrapper(ClassLoader cl) {
	try {
		return (UserCodeWrapper<T>) InstantiationUtil.readObjectFromConfig(this.config, STUB_OBJECT, cl);
	} catch (ClassNotFoundException | IOException e) {
		throw new CorruptConfigurationException("Could not read the user code wrapper: " + e.getMessage(), e);
	}
}
 
Example #11
Source Project: flink   Author: flink-tpc-ds   File: CrossOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
public CrossOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, String name) {
	super(udf, operatorInfo, name);
	
	if (this instanceof CrossWithSmall) {
		setCrossHint(CrossHint.SECOND_IS_SMALL);
	}
	else if (this instanceof CrossWithLarge) {
		setCrossHint(CrossHint.FIRST_IS_SMALL);
	}
}
 
Example #12
Source Project: flink   Author: flink-tpc-ds   File: InputOutputFormatContainer.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public <OT, T extends InputSplit> Pair<OperatorID, InputFormat<OT, T>> getUniqueInputFormat() {
	Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = formats.getInputFormats();
	Preconditions.checkState(inputFormats.size() == 1);

	Map.Entry<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> entry = inputFormats.entrySet().iterator().next();

	return new ImmutablePair<>(entry.getKey(),
		(InputFormat<OT, T>) entry.getValue().getUserCodeObject(InputFormat.class, userCodeClassLoader));
}
 
Example #13
Source Project: flink   Author: apache   File: InputOutputFormatContainer.java    License: Apache License 2.0 5 votes vote down vote up
public void addOutputFormat(OperatorID operatorId, UserCodeWrapper<? extends OutputFormat<?>> wrapper) {
	if (outputFormats.containsKey(checkNotNull(operatorId))) {
		throw new IllegalStateException("The output format has been set for the operator: " + operatorId);
	}

	outputFormats.put(operatorId, checkNotNull(wrapper));
}
 
Example #14
Source Project: flink   Author: flink-tpc-ds   File: InputOutputFormatContainer.java    License: Apache License 2.0 5 votes vote down vote up
public void addInputFormat(OperatorID operatorId, UserCodeWrapper<? extends InputFormat<?, ?>> wrapper) {
	if (inputFormats.containsKey(checkNotNull(operatorId))) {
		throw new IllegalStateException("The input format has been set for the operator: " + operatorId);
	}

	inputFormats.put(operatorId, checkNotNull(wrapper));
}
 
Example #15
Source Project: flink   Author: flink-tpc-ds   File: InputOutputFormatContainer.java    License: Apache License 2.0 5 votes vote down vote up
public void addOutputFormat(OperatorID operatorId, UserCodeWrapper<? extends OutputFormat<?>> wrapper) {
	if (outputFormats.containsKey(checkNotNull(operatorId))) {
		throw new IllegalStateException("The output format has been set for the operator: " + operatorId);
	}

	outputFormats.put(operatorId, checkNotNull(wrapper));
}
 
Example #16
Source Project: flink   Author: flink-tpc-ds   File: TaskConfig.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public <T> UserCodeWrapper<T> getStubWrapper(ClassLoader cl) {
	try {
		return (UserCodeWrapper<T>) InstantiationUtil.readObjectFromConfig(this.config, STUB_OBJECT, cl);
	} catch (ClassNotFoundException | IOException e) {
		throw new CorruptConfigurationException("Could not read the user code wrapper: " + e.getMessage(), e);
	}
}
 
Example #17
Source Project: flink   Author: apache   File: InputOutputFormatContainer.java    License: Apache License 2.0 5 votes vote down vote up
public void addInputFormat(OperatorID operatorId, UserCodeWrapper<? extends InputFormat<?, ?>> wrapper) {
	if (inputFormats.containsKey(checkNotNull(operatorId))) {
		throw new IllegalStateException("The input format has been set for the operator: " + operatorId);
	}

	inputFormats.put(operatorId, checkNotNull(wrapper));
}
 
Example #18
Source Project: flink   Author: apache   File: InputOutputFormatContainer.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public <OT, T extends InputSplit> Pair<OperatorID, InputFormat<OT, T>> getUniqueInputFormat() {
	Map<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> inputFormats = formats.getInputFormats();
	Preconditions.checkState(inputFormats.size() == 1);

	Map.Entry<OperatorID, UserCodeWrapper<? extends InputFormat<?, ?>>> entry = inputFormats.entrySet().iterator().next();

	return new ImmutablePair<>(entry.getKey(),
		(InputFormat<OT, T>) entry.getValue().getUserCodeObject(InputFormat.class, userCodeClassLoader));
}
 
Example #19
Source Project: Flink-CEPplus   Author: ljygz   File: InnerJoinOperatorBase.java    License: Apache License 2.0 4 votes vote down vote up
public InnerJoinOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo,
		int[] keyPositions1, int[] keyPositions2, String name) {
	super(udf, operatorInfo, keyPositions1, keyPositions2, name);
}
 
Example #20
Source Project: Flink-CEPplus   Author: ljygz   File: DeltaIterationBase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public UserCodeWrapper<?> getUserCodeWrapper() {
	return null;
}
 
Example #21
Source Project: Flink-CEPplus   Author: ljygz   File: DeltaIterationBase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public UserCodeWrapper<?> getUserCodeWrapper() {
	return null;
}
 
Example #22
Source Project: Flink-CEPplus   Author: ljygz   File: CoGroupRawOperatorBase.java    License: Apache License 2.0 4 votes vote down vote up
public CoGroupRawOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, int[] keyPositions1, int[] keyPositions2, String name) {
	super(udf, operatorInfo, keyPositions1, keyPositions2, name);
	this.combinableFirst = false;
	this.combinableSecond = false;
}
 
Example #23
Source Project: flink   Author: apache   File: BulkIterationBase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public UserCodeWrapper<?> getUserCodeWrapper() {
	return null;
}
 
Example #24
Source Project: Flink-CEPplus   Author: ljygz   File: MapPartitionOperatorBase.java    License: Apache License 2.0 4 votes vote down vote up
public MapPartitionOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, String name) {
	super(udf, operatorInfo, name);
}
 
Example #25
Source Project: Flink-CEPplus   Author: ljygz   File: BulkIterationBase.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public UserCodeWrapper<?> getUserCodeWrapper() {
	return null;
}
 
Example #26
Source Project: Flink-CEPplus   Author: ljygz   File: FilterOperatorBase.java    License: Apache License 2.0 4 votes vote down vote up
public FilterOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<T, T> operatorInfo, String name) {
	super(udf, operatorInfo, name);
}
 
Example #27
Source Project: Flink-CEPplus   Author: ljygz   File: GroupReduceOperatorBase.java    License: Apache License 2.0 4 votes vote down vote up
public GroupReduceOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, int[] keyPositions, String name) {
	super(udf, operatorInfo, keyPositions, name);
}
 
Example #28
Source Project: flink   Author: apache   File: MapOperatorBase.java    License: Apache License 2.0 4 votes vote down vote up
public MapOperatorBase(UserCodeWrapper<FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, String name) {
	super(udf, operatorInfo, name);
}
 
Example #29
Source Project: Flink-CEPplus   Author: ljygz   File: OuterJoinOperatorBase.java    License: Apache License 2.0 4 votes vote down vote up
public OuterJoinOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo,
		int[] keyPositions1, int[] keyPositions2, String name, OuterJoinType outerJoinType) {
	super(udf, operatorInfo, keyPositions1, keyPositions2, name);
	this.outerJoinType = outerJoinType;
}
 
Example #30
Source Project: flink   Author: apache   File: CoGroupOperatorBase.java    License: Apache License 2.0 4 votes vote down vote up
public CoGroupOperatorBase(UserCodeWrapper<FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, int[] keyPositions1, int[] keyPositions2, String name) {
	super(udf, operatorInfo, keyPositions1, keyPositions2, name);
	this.combinableFirst = false;
	this.combinableSecond = false;
}