org.apache.flink.api.common.operators.util.UserCodeClassWrapper Java Examples

private void addChainedOperator() {
	final TaskConfig chainedConfig = new TaskConfig(new Configuration());

	// input
	chainedConfig.addInputToGroup(0);
	chainedConfig.setInputSerializer(serFact, 0);

	// output
	chainedConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
	chainedConfig.setOutputSerializer(serFact);

	// driver
	chainedConfig.setDriverStrategy(DriverStrategy.FLAT_MAP);

	// udf
	chainedConfig.setStubWrapper(new UserCodeClassWrapper<>(DuplicatingFlatMapFunction.class));

	getTaskConfig().addChainedTask(ChainedFlatMapDriver.class, chainedConfig, CHAINED_OPERATOR_NAME);
}
 

private void addChainedOperator() {
	final TaskConfig chainedConfig = new TaskConfig(new Configuration());

	// input
	chainedConfig.addInputToGroup(0);
	chainedConfig.setInputSerializer(serFact, 0);

	// output
	chainedConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
	chainedConfig.setOutputSerializer(serFact);

	// driver
	chainedConfig.setDriverStrategy(DriverStrategy.FLAT_MAP);

	// udf
	chainedConfig.setStubWrapper(new UserCodeClassWrapper<>(DuplicatingFlatMapFunction.class));

	getTaskConfig().addChainedTask(ChainedFlatMapDriver.class, chainedConfig, CHAINED_OPERATOR_NAME);
}
 

private void addChainedOperator() {
	final TaskConfig chainedConfig = new TaskConfig(new Configuration());

	// input
	chainedConfig.addInputToGroup(0);
	chainedConfig.setInputSerializer(serFact, 0);

	// output
	chainedConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
	chainedConfig.setOutputSerializer(serFact);

	// driver
	chainedConfig.setDriverStrategy(DriverStrategy.FLAT_MAP);

	// udf
	chainedConfig.setStubWrapper(new UserCodeClassWrapper<>(DuplicatingFlatMapFunction.class));

	getTaskConfig().addChainedTask(ChainedFlatMapDriver.class, chainedConfig, CHAINED_OPERATOR_NAME);
}
 

public void registerTask(
		@SuppressWarnings("rawtypes") Class<? extends Driver> driver,
		Class<? extends RichFunction> stubClass) {

	final TaskConfig config = new TaskConfig(this.mockEnv.getTaskConfiguration());
	config.setDriver(driver);
	config.setStubWrapper(new UserCodeClassWrapper<>(stubClass));
}
 

public void registerTask(
		@SuppressWarnings("rawtypes") Class<? extends Driver> driver,
		Class<? extends RichFunction> stubClass) {

	final TaskConfig config = new TaskConfig(this.mockEnv.getTaskConfiguration());
	config.setDriver(driver);
	config.setStubWrapper(new UserCodeClassWrapper<>(stubClass));
}
 

/**
 * Creates a new instance for the given file using the given input format.
 *
 * @param format The {@link org.apache.flink.api.common.io.InputFormat} implementation used to read the data.
 * @param operatorInfo The type information for the operator.
 * @param name The given name for the Pact, used in plans, logs and progress messages.
 */
public GenericDataSourceBase(Class<? extends T> format, OperatorInformation<OUT> operatorInfo, String name) {
	super(operatorInfo, name);

	if (format == null) {
		throw new IllegalArgumentException("Input format may not be null.");
	}

	this.formatWrapper = new UserCodeClassWrapper<T>(format);
}
 

@Test
public void testDataSourceTaskOutputInCloseMethod() throws IOException {
	final int numChainedTasks = 10;
	final int keyCnt = 100;
	final int valCnt = 10;
	final File tempTestFile = new File(tempFolder.getRoot(), UUID.randomUUID().toString());
	DataSourceTaskTest.InputFilePreparator.prepareInputFile(
		new UniformRecordGenerator(keyCnt, valCnt, false), tempTestFile, true);
	initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
	addOutput(outList);
	final DataSourceTask<Record> testTask = new DataSourceTask<>(mockEnv);
	registerFileInputTask(
		testTask, DataSourceTaskTest.MockInputFormat.class, tempTestFile.toURI().toString(), "\n");
	for (int i = 0; i < numChainedTasks; i++) {
		final TaskConfig taskConfig = new TaskConfig(new Configuration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serFact);
		taskConfig.setStubWrapper(
			new UserCodeClassWrapper<>(ChainTaskTest.MockDuplicateLastValueMapFunction.class));
		getTaskConfig().addChainedTask(
			ChainedFlatMapDriver.class, taskConfig, "chained-" + i);
	}
	try {
		testTask.invoke();
		Assert.assertEquals(keyCnt * valCnt + numChainedTasks, outList.size());
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Invoke method caused exception.");
	}
}
 

@Test
public void testBatchTaskOutputInCloseMethod() {
	final int numChainedTasks = 10;
	final int keyCnt = 100;
	final int valCnt = 10;
	try {
		initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
		addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
		addOutput(outList);
		registerTask(FlatMapDriver.class, MockMapStub.class);
		for (int i = 0; i < numChainedTasks; i++) {
			final TaskConfig taskConfig = new TaskConfig(new Configuration());
			taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
			taskConfig.setOutputSerializer(serFact);
			taskConfig.setStubWrapper(
				new UserCodeClassWrapper<>(MockDuplicateLastValueMapFunction.class));
			getTaskConfig().addChainedTask(
				ChainedFlatMapDriver.class, taskConfig, "chained-" + i);
		}
		final BatchTask<FlatMapFunction<Record, Record>, Record> testTask =
			new BatchTask<>(mockEnv);
		testTask.invoke();
		Assert.assertEquals(keyCnt * valCnt + numChainedTasks, outList.size());
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

/**
 * Creates a new instance for the given file using the given input format, using the default name.
 *
 * @param format The {@link org.apache.flink.api.common.io.InputFormat} implementation used to read the data.
 * @param operatorInfo The type information for the operator.
 */
public GenericDataSourceBase(Class<? extends T> format, OperatorInformation<OUT> operatorInfo) {
	super(operatorInfo, DEFAULT_NAME);
	
	if (format == null) {
		throw new IllegalArgumentException("Input format may not be null.");
	}
	
	this.formatWrapper = new UserCodeClassWrapper<T>(format);
}
 

/**
 * Creates a new instance for the given file using the given input format.
 *
 * @param format The {@link org.apache.flink.api.common.io.InputFormat} implementation used to read the data.
 * @param operatorInfo The type information for the operator.
 * @param name The given name for the Pact, used in plans, logs and progress messages.
 */
public GenericDataSourceBase(Class<? extends T> format, OperatorInformation<OUT> operatorInfo, String name) {
	super(operatorInfo, name);

	if (format == null) {
		throw new IllegalArgumentException("Input format may not be null.");
	}

	this.formatWrapper = new UserCodeClassWrapper<T>(format);
}
 

/**
 * Creates a new instance for the given file using the given input format, using the default name.
 *
 * @param format The {@link org.apache.flink.api.common.io.InputFormat} implementation used to read the data.
 * @param operatorInfo The type information for the operator.
 */
public GenericDataSourceBase(Class<? extends T> format, OperatorInformation<OUT> operatorInfo) {
	super(operatorInfo, DEFAULT_NAME);
	
	if (format == null) {
		throw new IllegalArgumentException("Input format may not be null.");
	}
	
	this.formatWrapper = new UserCodeClassWrapper<T>(format);
}
 

public void registerTask(
		@SuppressWarnings("rawtypes") Class<? extends Driver> driver,
		Class<? extends RichFunction> stubClass) {

	final TaskConfig config = new TaskConfig(this.mockEnv.getTaskConfiguration());
	config.setDriver(driver);
	config.setStubWrapper(new UserCodeClassWrapper<>(stubClass));
}
 

/**
 * Creates a new instance for the given file using the given input format, using the default name.
 *
 * @param format The {@link org.apache.flink.api.common.io.InputFormat} implementation used to read the data.
 * @param operatorInfo The type information for the operator.
 */
public GenericDataSourceBase(Class<? extends T> format, OperatorInformation<OUT> operatorInfo) {
	super(operatorInfo, DEFAULT_NAME);
	
	if (format == null) {
		throw new IllegalArgumentException("Input format may not be null.");
	}
	
	this.formatWrapper = new UserCodeClassWrapper<T>(format);
}
 

/**
 * Creates a new instance for the given file using the given input format.
 *
 * @param format The {@link org.apache.flink.api.common.io.InputFormat} implementation used to read the data.
 * @param operatorInfo The type information for the operator.
 * @param name The given name for the Pact, used in plans, logs and progress messages.
 */
public GenericDataSourceBase(Class<? extends T> format, OperatorInformation<OUT> operatorInfo, String name) {
	super(operatorInfo, name);

	if (format == null) {
		throw new IllegalArgumentException("Input format may not be null.");
	}

	this.formatWrapper = new UserCodeClassWrapper<T>(format);
}
 

/** 
 * Creates a new Union operator.
 */
public Union(BinaryOperatorInformation<T, T, T> operatorInfo, String unionLocationName) {
	// we pass it an AbstractFunction, because currently all operators expect some form of UDF
	super(new UserCodeClassWrapper<AbstractRichFunction>(AbstractRichFunction.class), operatorInfo, "Union at "+unionLocationName);
}
 

public MapOperatorBase(Class<? extends FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, String name) {
	super(new UserCodeClassWrapper<FT>(udf), operatorInfo, name);
}
 

@Test
public void testMapTask() throws Exception {
	final int keyCnt = 100;
	final int valCnt = 20;

	final double memoryFraction = 1.0;

	// environment
	initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
	mockEnv.getExecutionConfig().enableObjectReuse();
	addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
	addOutput(this.outList);

	// chained reduce config
	{
		final TaskConfig reduceConfig = new TaskConfig(new Configuration());

		// input
		reduceConfig.addInputToGroup(0);
		reduceConfig.setInputSerializer(serFact, 0);

		// output
		reduceConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		reduceConfig.setOutputSerializer(serFact);

		// driver
		reduceConfig.setDriverStrategy(DriverStrategy.ALL_REDUCE);
		reduceConfig.setDriverComparator(compFact, 0);
		reduceConfig.setDriverComparator(compFact, 1);
		reduceConfig.setRelativeMemoryDriver(memoryFraction);

		// udf
		reduceConfig.setStubWrapper(new UserCodeClassWrapper<>(MockReduceStub.class));

		getTaskConfig().addChainedTask(ChainedAllReduceDriver.class, reduceConfig, "reduce");
	}

	// chained map+reduce
	{
		registerTask(FlatMapDriver.class, MockMapStub.class);
		BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(mockEnv);

		testTask.invoke();
	}

	int sumTotal = valCnt * keyCnt * (keyCnt - 1) / 2;

	Assert.assertEquals(1, this.outList.size());
	Assert.assertEquals(sumTotal, this.outList.get(0).getField(0, IntValue.class).getValue());
}
 

public CrossOperatorBase(Class<? extends FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, String name) {
	this(new UserCodeClassWrapper<FT>(udf), operatorInfo, name);
}
 

public FlatMapOperatorBase(Class<? extends FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, String name) {
	super(new UserCodeClassWrapper<FT>(udf), operatorInfo, name);
}
 

public NoOpUnaryUdfOp() {
	// pass null here because we override getOutputType to return type
	// of input operator
	super(new UserCodeClassWrapper<NoOpFunction>(NoOpFunction.class), null, "");
}
 

public JoinOperatorBase(Class<? extends FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, int[] keyPositions1, int[] keyPositions2, String name) {
	super(new UserCodeClassWrapper<>(udf), operatorInfo, keyPositions1, keyPositions2, name);
}
 

public NoOpBinaryUdfOp(TypeInformation<OUT> type) {
	super(new UserCodeClassWrapper<NoOpFunction>(NoOpFunction.class), new BinaryOperatorInformation<OUT, OUT, OUT>(type, type, type), "NoContract");
}
 

public CoGroupRawOperatorBase(Class<? extends FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo, int[] keyPositions1, int[] keyPositions2, String name) {
	this(new UserCodeClassWrapper<FT>(udf), operatorInfo, keyPositions1, keyPositions2, name);
}
 

public DeltaIterationBase(BinaryOperatorInformation<ST, WT, ST> operatorInfo, int[] keyPositions, String name) {
	super(new UserCodeClassWrapper<AbstractRichFunction>(AbstractRichFunction.class), operatorInfo, name);
	this.solutionSetKeyFields = keyPositions;
	solutionSetPlaceholder = new SolutionSetPlaceHolder<ST>(this, new OperatorInformation<ST>(operatorInfo.getFirstInputType()));
	worksetPlaceholder = new WorksetPlaceHolder<WT>(this, new OperatorInformation<WT>(operatorInfo.getSecondInputType()));
}
 

public InnerJoinOperatorBase(Class<? extends FT> udf, BinaryOperatorInformation<IN1, IN2, OUT> operatorInfo,
		int[] keyPositions1, int[] keyPositions2, String name) {
	super(new UserCodeClassWrapper<FT>(udf), operatorInfo, keyPositions1, keyPositions2, name);
}
 

public FilterOperatorBase(Class<? extends FT> udf, UnaryOperatorInformation<T, T> operatorInfo, String name) {
	super(new UserCodeClassWrapper<FT>(udf), operatorInfo, name);
}
 

public NoOpBinaryUdfOp(TypeInformation<OUT> type) {
	super(new UserCodeClassWrapper<NoOpFunction>(NoOpFunction.class), new BinaryOperatorInformation<OUT, OUT, OUT>(type, type, type), "NoContract");
}
 

@Test
public void testMapTask() throws Exception {
	final int keyCnt = 100;
	final int valCnt = 20;

	final double memoryFraction = 1.0;

	// environment
	initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
	mockEnv.getExecutionConfig().enableObjectReuse();
	addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
	addOutput(this.outList);

	// chained reduce config
	{
		final TaskConfig reduceConfig = new TaskConfig(new Configuration());

		// input
		reduceConfig.addInputToGroup(0);
		reduceConfig.setInputSerializer(serFact, 0);

		// output
		reduceConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		reduceConfig.setOutputSerializer(serFact);

		// driver
		reduceConfig.setDriverStrategy(DriverStrategy.ALL_REDUCE);
		reduceConfig.setDriverComparator(compFact, 0);
		reduceConfig.setDriverComparator(compFact, 1);
		reduceConfig.setRelativeMemoryDriver(memoryFraction);

		// udf
		reduceConfig.setStubWrapper(new UserCodeClassWrapper<>(MockReduceStub.class));

		getTaskConfig().addChainedTask(ChainedAllReduceDriver.class, reduceConfig, "reduce");
	}

	// chained map+reduce
	{
		registerTask(FlatMapDriver.class, MockMapStub.class);
		BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(mockEnv);

		testTask.invoke();
	}

	int sumTotal = valCnt * keyCnt * (keyCnt - 1) / 2;

	Assert.assertEquals(1, this.outList.size());
	Assert.assertEquals(sumTotal, this.outList.get(0).getField(0, IntValue.class).getValue());
}
 

/** 
 * Creates a new Union operator.
 */
public Union(BinaryOperatorInformation<T, T, T> operatorInfo, String unionLocationName) {
	// we pass it an AbstractFunction, because currently all operators expect some form of UDF
	super(new UserCodeClassWrapper<AbstractRichFunction>(AbstractRichFunction.class), operatorInfo, "Union at "+unionLocationName);
}
 

public FlatMapOperatorBase(Class<? extends FT> udf, UnaryOperatorInformation<IN, OUT> operatorInfo, String name) {
	super(new UserCodeClassWrapper<FT>(udf), operatorInfo, name);
}