Java Code Examples for org.apache.flink.runtime.operators.util.TaskConfig#addOutputShipStrategy()

The following examples show how to use org.apache.flink.runtime.operators.util.TaskConfig#addOutputShipStrategy() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ChainedOperatorsMetricTest.java From Flink-CEPplus with Apache License 2.0

6 votes

private void addChainedOperator() {
	final TaskConfig chainedConfig = new TaskConfig(new Configuration());

	// input
	chainedConfig.addInputToGroup(0);
	chainedConfig.setInputSerializer(serFact, 0);

	// output
	chainedConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
	chainedConfig.setOutputSerializer(serFact);

	// driver
	chainedConfig.setDriverStrategy(DriverStrategy.FLAT_MAP);

	// udf
	chainedConfig.setStubWrapper(new UserCodeClassWrapper<>(DuplicatingFlatMapFunction.class));

	getTaskConfig().addChainedTask(ChainedFlatMapDriver.class, chainedConfig, CHAINED_OPERATOR_NAME);
}

Example 2

Source File: ChainedOperatorsMetricTest.java From flink with Apache License 2.0

6 votes

private void addChainedOperator() {
	final TaskConfig chainedConfig = new TaskConfig(new Configuration());

	// input
	chainedConfig.addInputToGroup(0);
	chainedConfig.setInputSerializer(serFact, 0);

	// output
	chainedConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
	chainedConfig.setOutputSerializer(serFact);

	// driver
	chainedConfig.setDriverStrategy(DriverStrategy.FLAT_MAP);

	// udf
	chainedConfig.setStubWrapper(new UserCodeClassWrapper<>(DuplicatingFlatMapFunction.class));

	getTaskConfig().addChainedTask(ChainedFlatMapDriver.class, chainedConfig, CHAINED_OPERATOR_NAME);
}

Example 3

Source File: ChainedOperatorsMetricTest.java From flink with Apache License 2.0

6 votes

private void addChainedOperator() {
	final TaskConfig chainedConfig = new TaskConfig(new Configuration());

	// input
	chainedConfig.addInputToGroup(0);
	chainedConfig.setInputSerializer(serFact, 0);

	// output
	chainedConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
	chainedConfig.setOutputSerializer(serFact);

	// driver
	chainedConfig.setDriverStrategy(DriverStrategy.FLAT_MAP);

	// udf
	chainedConfig.setStubWrapper(new UserCodeClassWrapper<>(DuplicatingFlatMapFunction.class));

	getTaskConfig().addChainedTask(ChainedFlatMapDriver.class, chainedConfig, CHAINED_OPERATOR_NAME);
}

Example 4

Source File: ChainTaskTest.java From flink with Apache License 2.0

5 votes

@Test
public void testBatchTaskOutputInCloseMethod() {
	final int numChainedTasks = 10;
	final int keyCnt = 100;
	final int valCnt = 10;
	try {
		initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
		addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
		addOutput(outList);
		registerTask(FlatMapDriver.class, MockMapStub.class);
		for (int i = 0; i < numChainedTasks; i++) {
			final TaskConfig taskConfig = new TaskConfig(new Configuration());
			taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
			taskConfig.setOutputSerializer(serFact);
			taskConfig.setStubWrapper(
				new UserCodeClassWrapper<>(MockDuplicateLastValueMapFunction.class));
			getTaskConfig().addChainedTask(
				ChainedFlatMapDriver.class, taskConfig, "chained-" + i);
		}
		final BatchTask<FlatMapFunction<Record, Record>, Record> testTask =
			new BatchTask<>(mockEnv);
		testTask.invoke();
		Assert.assertEquals(keyCnt * valCnt + numChainedTasks, outList.size());
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example 5

Source File: ChainTaskTest.java From flink with Apache License 2.0

5 votes

@Test
public void testDataSourceTaskOutputInCloseMethod() throws IOException {
	final int numChainedTasks = 10;
	final int keyCnt = 100;
	final int valCnt = 10;
	final File tempTestFile = new File(tempFolder.getRoot(), UUID.randomUUID().toString());
	DataSourceTaskTest.InputFilePreparator.prepareInputFile(
		new UniformRecordGenerator(keyCnt, valCnt, false), tempTestFile, true);
	initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
	addOutput(outList);
	final DataSourceTask<Record> testTask = new DataSourceTask<>(mockEnv);
	registerFileInputTask(
		testTask, DataSourceTaskTest.MockInputFormat.class, tempTestFile.toURI().toString(), "\n");
	for (int i = 0; i < numChainedTasks; i++) {
		final TaskConfig taskConfig = new TaskConfig(new Configuration());
		taskConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		taskConfig.setOutputSerializer(serFact);
		taskConfig.setStubWrapper(
			new UserCodeClassWrapper<>(ChainTaskTest.MockDuplicateLastValueMapFunction.class));
		getTaskConfig().addChainedTask(
			ChainedFlatMapDriver.class, taskConfig, "chained-" + i);
	}
	try {
		testTask.invoke();
		Assert.assertEquals(keyCnt * valCnt + numChainedTasks, outList.size());
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail("Invoke method caused exception.");
	}
}

Example 6

Source File: ChainedAllReduceDriverTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testMapTask() throws Exception {
	final int keyCnt = 100;
	final int valCnt = 20;

	final double memoryFraction = 1.0;

	// environment
	initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
	mockEnv.getExecutionConfig().enableObjectReuse();
	addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
	addOutput(this.outList);

	// chained reduce config
	{
		final TaskConfig reduceConfig = new TaskConfig(new Configuration());

		// input
		reduceConfig.addInputToGroup(0);
		reduceConfig.setInputSerializer(serFact, 0);

		// output
		reduceConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		reduceConfig.setOutputSerializer(serFact);

		// driver
		reduceConfig.setDriverStrategy(DriverStrategy.ALL_REDUCE);
		reduceConfig.setDriverComparator(compFact, 0);
		reduceConfig.setDriverComparator(compFact, 1);
		reduceConfig.setRelativeMemoryDriver(memoryFraction);

		// udf
		reduceConfig.setStubWrapper(new UserCodeClassWrapper<>(MockReduceStub.class));

		getTaskConfig().addChainedTask(ChainedAllReduceDriver.class, reduceConfig, "reduce");
	}

	// chained map+reduce
	{
		registerTask(FlatMapDriver.class, MockMapStub.class);
		BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(mockEnv);

		testTask.invoke();
	}

	int sumTotal = valCnt * keyCnt * (keyCnt - 1) / 2;

	Assert.assertEquals(1, this.outList.size());
	Assert.assertEquals(sumTotal, this.outList.get(0).getField(0, IntValue.class).getValue());
}

Example 7

Source File: TaskTestBase.java From Flink-CEPplus with Apache License 2.0

4 votes

public void addOutput(List<Record> output) {
	this.mockEnv.addOutput(output);
	TaskConfig conf = new TaskConfig(this.mockEnv.getTaskConfiguration());
	conf.addOutputShipStrategy(ShipStrategyType.FORWARD);
	conf.setOutputSerializer(RecordSerializerFactory.get());
}

Example 8

Source File: ChainedAllReduceDriverTest.java From flink with Apache License 2.0

4 votes

@Test
public void testMapTask() throws Exception {
	final int keyCnt = 100;
	final int valCnt = 20;

	final double memoryFraction = 1.0;

	// environment
	initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
	mockEnv.getExecutionConfig().enableObjectReuse();
	addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
	addOutput(this.outList);

	// chained reduce config
	{
		final TaskConfig reduceConfig = new TaskConfig(new Configuration());

		// input
		reduceConfig.addInputToGroup(0);
		reduceConfig.setInputSerializer(serFact, 0);

		// output
		reduceConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		reduceConfig.setOutputSerializer(serFact);

		// driver
		reduceConfig.setDriverStrategy(DriverStrategy.ALL_REDUCE);
		reduceConfig.setDriverComparator(compFact, 0);
		reduceConfig.setDriverComparator(compFact, 1);
		reduceConfig.setRelativeMemoryDriver(memoryFraction);

		// udf
		reduceConfig.setStubWrapper(new UserCodeClassWrapper<>(MockReduceStub.class));

		getTaskConfig().addChainedTask(ChainedAllReduceDriver.class, reduceConfig, "reduce");
	}

	// chained map+reduce
	{
		registerTask(FlatMapDriver.class, MockMapStub.class);
		BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(mockEnv);

		testTask.invoke();
	}

	int sumTotal = valCnt * keyCnt * (keyCnt - 1) / 2;

	Assert.assertEquals(1, this.outList.size());
	Assert.assertEquals(sumTotal, this.outList.get(0).getField(0, IntValue.class).getValue());
}

Example 9

Source File: TaskTestBase.java From flink with Apache License 2.0

4 votes

public void addOutput(List<Record> output) {
	this.mockEnv.addOutput(output);
	TaskConfig conf = new TaskConfig(this.mockEnv.getTaskConfiguration());
	conf.addOutputShipStrategy(ShipStrategyType.FORWARD);
	conf.setOutputSerializer(RecordSerializerFactory.get());
}

Example 10

Source File: ChainedAllReduceDriverTest.java From flink with Apache License 2.0

4 votes

@Test
public void testMapTask() throws Exception {
	final int keyCnt = 100;
	final int valCnt = 20;

	final double memoryFraction = 1.0;

	// environment
	initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
	mockEnv.getExecutionConfig().enableObjectReuse();
	addInput(new UniformRecordGenerator(keyCnt, valCnt, false), 0);
	addOutput(this.outList);

	// chained reduce config
	{
		final TaskConfig reduceConfig = new TaskConfig(new Configuration());

		// input
		reduceConfig.addInputToGroup(0);
		reduceConfig.setInputSerializer(serFact, 0);

		// output
		reduceConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
		reduceConfig.setOutputSerializer(serFact);

		// driver
		reduceConfig.setDriverStrategy(DriverStrategy.ALL_REDUCE);
		reduceConfig.setDriverComparator(compFact, 0);
		reduceConfig.setDriverComparator(compFact, 1);
		reduceConfig.setRelativeMemoryDriver(memoryFraction);

		// udf
		reduceConfig.setStubWrapper(new UserCodeClassWrapper<>(MockReduceStub.class));

		getTaskConfig().addChainedTask(ChainedAllReduceDriver.class, reduceConfig, "reduce");
	}

	// chained map+reduce
	{
		registerTask(FlatMapDriver.class, MockMapStub.class);
		BatchTask<FlatMapFunction<Record, Record>, Record> testTask = new BatchTask<>(mockEnv);

		testTask.invoke();
	}

	int sumTotal = valCnt * keyCnt * (keyCnt - 1) / 2;

	Assert.assertEquals(1, this.outList.size());
	Assert.assertEquals(sumTotal, this.outList.get(0).getField(0, IntValue.class).getValue());
}

Example 11

Source File: TaskTestBase.java From flink with Apache License 2.0

4 votes

public void addOutput(List<Record> output) {
	this.mockEnv.addOutput(output);
	TaskConfig conf = new TaskConfig(this.mockEnv.getTaskConfiguration());
	conf.addOutputShipStrategy(ShipStrategyType.FORWARD);
	conf.setOutputSerializer(RecordSerializerFactory.get());
}