Java Code Examples for org.apache.flink.api.common.ExecutionConfig#disableObjectReuse()

The following examples show how to use org.apache.flink.api.common.ExecutionConfig#disableObjectReuse() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private void testExecuteOnCollection(FlatMapFunction<String, String> udf, List<String> input, boolean mutableSafe) throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	if (mutableSafe) {
		executionConfig.disableObjectReuse();
	} else {
		executionConfig.enableObjectReuse();
	}
	final TaskInfo taskInfo = new TaskInfo("Test UDF", 4, 0, 4, 0);
	// run on collections
	final List<String> result = getTestFlatMapOperator(udf)
			.executeOnCollections(input,
					new RuntimeUDFContext(
						taskInfo,  null, executionConfig, new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup()),
					executionConfig);

	Assert.assertEquals(input.size(), result.size());
	Assert.assertEquals(input, result);
}
 
Example 2
Source Project: flink   File: GenericDataSourceBaseTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDataSourcePlain() {
	try {
		TestNonRichInputFormat in = new TestNonRichInputFormat();
		GenericDataSourceBase<String, TestNonRichInputFormat> source =
				new GenericDataSourceBase<String, TestNonRichInputFormat>(
						in, new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO), "testSource");

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		List<String> resultMutableSafe = source.executeOnCollections(null, executionConfig);

		in.reset();
		executionConfig.enableObjectReuse();
		List<String> resultRegular = source.executeOnCollections(null, executionConfig);
		assertEquals(asList(TestIOData.NAMES), resultMutableSafe);
		assertEquals(asList(TestIOData.NAMES), resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 3
Source Project: flink   File: BinaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() throws IOException {
	LinkedList<Object[]> configs = new LinkedList<>();
	
	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();
	
	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();
	
	Object[] a = {withoutReuse};
	configs.add(a);
	Object[] b = {withReuse};
	configs.add(b);
	
	return configs;
}
 
Example 4
@Before
public void beforeTest() {
	ExecutionConfig config = new ExecutionConfig();
	config.disableObjectReuse();
	
	TupleTypeInfo<Tuple2<String, String>> typeInfo1 = TupleTypeInfo.getBasicTupleTypeInfo(String.class, String.class);
	TupleTypeInfo<Tuple2<String, Integer>> typeInfo2 = TupleTypeInfo.getBasicTupleTypeInfo(String.class, Integer.class);
	serializer1 = typeInfo1.createSerializer(config);
	serializer2 = typeInfo2.createSerializer(config);
	comparator1 = typeInfo1.createComparator(new int[]{0}, new boolean[]{true}, 0, config);
	comparator2 = typeInfo2.createComparator(new int[]{0}, new boolean[]{true}, 0, config);
	pairComp = new GenericPairComparator<>(comparator1, comparator2);

	this.memoryManager = new MemoryManager(MEMORY_SIZE, 1);
	this.ioManager = new IOManagerAsync();
}
 
Example 5
Source Project: flink   File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() {

	LinkedList<Object[]> configs = new LinkedList<Object[]>();

	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();

	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();

	Object[] a = { withoutReuse };
	configs.add(a);
	Object[] b = { withReuse };
	configs.add(b);

	return configs;
}
 
Example 6
@Test
public void testDataSourcePlain() {
	try {
		TestNonRichInputFormat in = new TestNonRichInputFormat();
		GenericDataSourceBase<String, TestNonRichInputFormat> source =
				new GenericDataSourceBase<String, TestNonRichInputFormat>(
						in, new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO), "testSource");

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		List<String> resultMutableSafe = source.executeOnCollections(null, executionConfig);

		in.reset();
		executionConfig.enableObjectReuse();
		List<String> resultRegular = source.executeOnCollections(null, executionConfig);
		assertEquals(asList(TestIOData.NAMES), resultMutableSafe);
		assertEquals(asList(TestIOData.NAMES), resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 7
@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() throws IOException {
	LinkedList<Object[]> configs = new LinkedList<>();
	
	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();
	
	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();
	
	Object[] a = {withoutReuse};
	configs.add(a);
	Object[] b = {withReuse};
	configs.add(b);
	
	return configs;
}
 
Example 8
private void testExecuteOnCollection(FlatMapFunction<String, String> udf, List<String> input, boolean mutableSafe) throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	if (mutableSafe) {
		executionConfig.disableObjectReuse();
	} else {
		executionConfig.enableObjectReuse();
	}
	final TaskInfo taskInfo = new TaskInfo("Test UDF", 4, 0, 4, 0);
	// run on collections
	final List<String> result = getTestFlatMapOperator(udf)
			.executeOnCollections(input,
					new RuntimeUDFContext(
						taskInfo,  null, executionConfig, new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup()),
					executionConfig);

	Assert.assertEquals(input.size(), result.size());
	Assert.assertEquals(input, result);
}
 
Example 9
Source Project: flink   File: InnerJoinOperatorBaseTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testJoinPlain(){
	final FlatJoinFunction<String, String, Integer> joiner = new FlatJoinFunction<String, String, Integer>() {

		@Override
		public void join(String first, String second, Collector<Integer> out) throws Exception {
			out.collect(first.length());
			out.collect(second.length());
		}
	};

	@SuppressWarnings({ "rawtypes", "unchecked" })
	InnerJoinOperatorBase<String, String, Integer,
					FlatJoinFunction<String, String,Integer> > base = new InnerJoinOperatorBase(joiner,
			new BinaryOperatorInformation(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
					BasicTypeInfo.INT_TYPE_INFO), new int[0], new int[0], "TestJoiner");

	List<String> inputData1 = new ArrayList<String>(Arrays.asList("foo", "bar", "foobar"));
	List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
	List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6 ,6));

	try {
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2, null, executionConfig);
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2, null, executionConfig);

		assertEquals(expected, resultSafe);
		assertEquals(expected, resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 10
Source Project: flink   File: GenericDataSinkBaseTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDataSourceWithRuntimeContext() {
	try {
		TestRichOutputFormat out = new TestRichOutputFormat();
		GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>(
				out,
				new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)),
				"test_sink");
		sink.setInput(source);

		ExecutionConfig executionConfig = new ExecutionConfig();
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
		final TaskInfo taskInfo = new TaskInfo("test_sink", 1, 0, 1, 0);
		executionConfig.disableObjectReuse();
		in.reset();
		
		sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(
				taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
	
			assertEquals(out.output, asList(TestIOData.RICH_NAMES));

		executionConfig.enableObjectReuse();
		out.clear();
		in.reset();
		
		sink.executeOnCollections(asList(TestIOData.NAMES), new RuntimeUDFContext(
				taskInfo, null, executionConfig, cpTasks, accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		assertEquals(out.output, asList(TestIOData.RICH_NAMES));
	} catch(Exception e){
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 11
Source Project: flink   File: GenericDataSinkBaseTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDataSourcePlain() {
	try {
		TestNonRichOutputFormat out = new TestNonRichOutputFormat();
		GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>(
				out,
				new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)),
				"test_sink");
		sink.setInput(source);

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		in.reset();
		sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig);
		assertEquals(out.output, asList(TestIOData.NAMES));

		executionConfig.enableObjectReuse();
		out.clear();
		in.reset();
		sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig);
		assertEquals(out.output, asList(TestIOData.NAMES));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 12
@Test(expected = IllegalArgumentException.class)
public void testThatExceptionIsThrownForOuterJoinTypeNull() throws Exception {
	final List<String> leftInput = Arrays.asList("foo", "bar", "foobar");
	final List<String> rightInput = Arrays.asList("bar", "foobar", "foo");

	baseOperator.setOuterJoinType(null);
	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.disableObjectReuse();
	baseOperator.executeOnCollections(leftInput, rightInput, runtimeContext, executionConfig);
}
 
Example 13
Source Project: flink   File: UnaryOperatorTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() {
	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();

	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();

	Object[] a = { withoutReuse };
	Object[] b = { withReuse };
	return Arrays.asList(a, b);
}
 
Example 14
Source Project: flink   File: GenericDataSinkBaseTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDataSourcePlain() {
	try {
		TestNonRichOutputFormat out = new TestNonRichOutputFormat();
		GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>(
				out,
				new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)),
				"test_sink");
		sink.setInput(source);

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		in.reset();
		sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig);
		assertEquals(out.output, asList(TestIOData.NAMES));

		executionConfig.enableObjectReuse();
		out.clear();
		in.reset();
		sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig);
		assertEquals(out.output, asList(TestIOData.NAMES));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 15
Source Project: flink   File: InnerJoinOperatorBaseTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testJoinRich(){
	final AtomicBoolean opened = new AtomicBoolean(false);
	final AtomicBoolean closed = new AtomicBoolean(false);
	final String taskName = "Test rich join function";

	final RichFlatJoinFunction<String, String, Integer> joiner = new RichFlatJoinFunction<String, String, Integer>() {
		@Override
		public void open(Configuration parameters) throws Exception {
			opened.compareAndSet(false, true);
			assertEquals(0, getRuntimeContext().getIndexOfThisSubtask());
			assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
		}

		@Override
		public void close() throws Exception{
			closed.compareAndSet(false, true);
		}

		@Override
		public void join(String first, String second, Collector<Integer> out) throws Exception {
			out.collect(first.length());
			out.collect(second.length());
		}
	};

	InnerJoinOperatorBase<String, String, Integer,
					RichFlatJoinFunction<String, String, Integer>> base = new InnerJoinOperatorBase<String, String, Integer,
									RichFlatJoinFunction<String, String, Integer>>(joiner, new BinaryOperatorInformation<String, String,
			Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.INT_TYPE_INFO), new int[0], new int[0], taskName);

	final List<String> inputData1 = new ArrayList<String>(Arrays.asList("foo", "bar", "foobar"));
	final List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
	final List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6, 6));


	try {
		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();

		ExecutionConfig executionConfig = new ExecutionConfig();
		
		executionConfig.disableObjectReuse();
		List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);

		assertEquals(expected, resultSafe);
		assertEquals(expected, resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}

	assertTrue(opened.get());
	assertTrue(closed.get());
}
 
Example 16
@Test
public void testReduceCollectionWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();

		final ReduceFunction<Tuple2<String, Integer>> reducer = new RichReduceFunction<Tuple2<String, Integer>>() {

			@Override
			public Tuple2<String, Integer> reduce(
					Tuple2<String, Integer> value1,
					Tuple2<String, Integer> value2) throws Exception {

				return new Tuple2<>(value1.f0, value1.f1 + value2.f1);
			}

			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}

			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};

		ReduceOperatorBase<Tuple2<String, Integer>, ReduceFunction<Tuple2<String, Integer>>> op =
				new ReduceOperatorBase<>(
						reducer,
						new UnaryOperatorInformation<>(STRING_INT_TUPLE, STRING_INT_TUPLE),
						new int[]{0},
						"TestReducer");

		List<Tuple2<String, Integer>> input = new ArrayList<>(asList(
				new Tuple2<>("foo", 1),
				new Tuple2<>("foo", 3),
				new Tuple2<>("bar", 2),
				new Tuple2<>("bar", 4)));

		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);

		ExecutionConfig executionConfig = new ExecutionConfig();

		executionConfig.disableObjectReuse();
		List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		executionConfig.enableObjectReuse();
		List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<>(resultMutableSafe);
		Set<Tuple2<String, Integer>> resultSetRegular = new HashSet<>(resultRegular);

		Set<Tuple2<String, Integer>> expectedResult = new HashSet<>(asList(
				new Tuple2<>("foo", 4),
				new Tuple2<>("bar", 6)));

		assertEquals(expectedResult, resultSetMutableSafe);
		assertEquals(expectedResult, resultSetRegular);

		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 17
@Test
public void testJoinRich(){
	final AtomicBoolean opened = new AtomicBoolean(false);
	final AtomicBoolean closed = new AtomicBoolean(false);
	final String taskName = "Test rich join function";

	final RichFlatJoinFunction<String, String, Integer> joiner = new RichFlatJoinFunction<String, String, Integer>() {
		@Override
		public void open(Configuration parameters) throws Exception {
			opened.compareAndSet(false, true);
			assertEquals(0, getRuntimeContext().getIndexOfThisSubtask());
			assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
		}

		@Override
		public void close() throws Exception{
			closed.compareAndSet(false, true);
		}

		@Override
		public void join(String first, String second, Collector<Integer> out) throws Exception {
			out.collect(first.length());
			out.collect(second.length());
		}
	};

	InnerJoinOperatorBase<String, String, Integer,
					RichFlatJoinFunction<String, String, Integer>> base = new InnerJoinOperatorBase<String, String, Integer,
									RichFlatJoinFunction<String, String, Integer>>(joiner, new BinaryOperatorInformation<String, String,
			Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.INT_TYPE_INFO), new int[0], new int[0], taskName);

	final List<String> inputData1 = new ArrayList<String>(Arrays.asList("foo", "bar", "foobar"));
	final List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
	final List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6, 6));


	try {
		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();

		ExecutionConfig executionConfig = new ExecutionConfig();
		
		executionConfig.disableObjectReuse();
		List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);

		assertEquals(expected, resultSafe);
		assertEquals(expected, resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}

	assertTrue(opened.get());
	assertTrue(closed.get());
}
 
Example 18
Source Project: flink   File: PartitionMapOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testMapPartitionWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();
		
		final MapPartitionFunction<String, Integer> parser = new RichMapPartitionFunction<String, Integer>() {
			
			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}
			
			@Override
			public void mapPartition(Iterable<String> values, Collector<Integer> out) {
				for (String s : values) {
					out.collect(Integer.parseInt(s));
				}
			}
			
			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};
		
		MapPartitionOperatorBase<String, Integer, MapPartitionFunction<String, Integer>> op = 
				new MapPartitionOperatorBase<String, Integer, MapPartitionFunction<String,Integer>>(
				parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), taskName);
		
		List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));

		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		
		List<Integer> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);
		
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
		
		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 19
Source Project: flink   File: PartitionMapOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testMapPartitionWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();
		
		final MapPartitionFunction<String, Integer> parser = new RichMapPartitionFunction<String, Integer>() {
			
			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}
			
			@Override
			public void mapPartition(Iterable<String> values, Collector<Integer> out) {
				for (String s : values) {
					out.collect(Integer.parseInt(s));
				}
			}
			
			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};
		
		MapPartitionOperatorBase<String, Integer, MapPartitionFunction<String, Integer>> op = 
				new MapPartitionOperatorBase<String, Integer, MapPartitionFunction<String,Integer>>(
				parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), taskName);
		
		List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));

		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		
		List<Integer> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);
		
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
		
		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 20
Source Project: flink   File: ReduceOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testReduceCollectionWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();

		final ReduceFunction<Tuple2<String, Integer>> reducer = new RichReduceFunction<Tuple2<String, Integer>>() {

			@Override
			public Tuple2<String, Integer> reduce(
					Tuple2<String, Integer> value1,
					Tuple2<String, Integer> value2) throws Exception {

				return new Tuple2<>(value1.f0, value1.f1 + value2.f1);
			}

			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}

			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};

		ReduceOperatorBase<Tuple2<String, Integer>, ReduceFunction<Tuple2<String, Integer>>> op =
				new ReduceOperatorBase<>(
						reducer,
						new UnaryOperatorInformation<>(STRING_INT_TUPLE, STRING_INT_TUPLE),
						new int[]{0},
						"TestReducer");

		List<Tuple2<String, Integer>> input = new ArrayList<>(asList(
				new Tuple2<>("foo", 1),
				new Tuple2<>("foo", 3),
				new Tuple2<>("bar", 2),
				new Tuple2<>("bar", 4)));

		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);

		ExecutionConfig executionConfig = new ExecutionConfig();

		executionConfig.disableObjectReuse();
		List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		executionConfig.enableObjectReuse();
		List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<>(resultMutableSafe);
		Set<Tuple2<String, Integer>> resultSetRegular = new HashSet<>(resultRegular);

		Set<Tuple2<String, Integer>> expectedResult = new HashSet<>(asList(
				new Tuple2<>("foo", 4),
				new Tuple2<>("bar", 6)));

		assertEquals(expectedResult, resultSetMutableSafe);
		assertEquals(expectedResult, resultSetRegular);

		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}