Java Code Examples for org.apache.flink.api.common.ExecutionConfig#enableObjectReuse()

The following examples show how to use org.apache.flink.api.common.ExecutionConfig#enableObjectReuse() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: BinaryOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() throws IOException {
	LinkedList<Object[]> configs = new LinkedList<>();
	
	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();
	
	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();
	
	Object[] a = {withoutReuse};
	configs.add(a);
	Object[] b = {withReuse};
	configs.add(b);
	
	return configs;
}
 
Example 2
Source Project: flink   File: DriverTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() {

	LinkedList<Object[]> configs = new LinkedList<Object[]>();

	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();

	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();

	Object[] a = { withoutReuse };
	configs.add(a);
	Object[] b = { withReuse };
	configs.add(b);

	return configs;
}
 
Example 3
@Test
public void testDataSourcePlain() {
	try {
		TestNonRichInputFormat in = new TestNonRichInputFormat();
		GenericDataSourceBase<String, TestNonRichInputFormat> source =
				new GenericDataSourceBase<String, TestNonRichInputFormat>(
						in, new OperatorInformation<String>(BasicTypeInfo.STRING_TYPE_INFO), "testSource");

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		List<String> resultMutableSafe = source.executeOnCollections(null, executionConfig);

		in.reset();
		executionConfig.enableObjectReuse();
		List<String> resultRegular = source.executeOnCollections(null, executionConfig);
		assertEquals(asList(TestIOData.NAMES), resultMutableSafe);
		assertEquals(asList(TestIOData.NAMES), resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 4
@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() throws IOException {
	LinkedList<Object[]> configs = new LinkedList<>();
	
	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();
	
	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();
	
	Object[] a = {withoutReuse};
	configs.add(a);
	Object[] b = {withReuse};
	configs.add(b);
	
	return configs;
}
 
Example 5
private void testExecuteOnCollection(FlatMapFunction<String, String> udf, List<String> input, boolean mutableSafe) throws Exception {
	ExecutionConfig executionConfig = new ExecutionConfig();
	if (mutableSafe) {
		executionConfig.disableObjectReuse();
	} else {
		executionConfig.enableObjectReuse();
	}
	final TaskInfo taskInfo = new TaskInfo("Test UDF", 4, 0, 4, 0);
	// run on collections
	final List<String> result = getTestFlatMapOperator(udf)
			.executeOnCollections(input,
					new RuntimeUDFContext(
						taskInfo,  null, executionConfig, new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(), new UnregisteredMetricsGroup()),
					executionConfig);

	Assert.assertEquals(input.size(), result.size());
	Assert.assertEquals(input, result);
}
 
Example 6
@Test
public void testDataSourcePlain() {
	try {
		TestNonRichOutputFormat out = new TestNonRichOutputFormat();
		GenericDataSinkBase<String> sink = new GenericDataSinkBase<String>(
				out,
				new UnaryOperatorInformation<String, Nothing>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.getInfoFor(Nothing.class)),
				"test_sink");
		sink.setInput(source);

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		in.reset();
		sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig);
		assertEquals(out.output, asList(TestIOData.NAMES));

		executionConfig.enableObjectReuse();
		out.clear();
		in.reset();
		sink.executeOnCollections(asList(TestIOData.NAMES), null, executionConfig);
		assertEquals(out.output, asList(TestIOData.NAMES));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 7
@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() {
	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();

	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();

	Object[] a = { withoutReuse };
	Object[] b = { withReuse };
	return Arrays.asList(a, b);
}
 
Example 8
Source Project: flink   File: MapOperatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMapPlain() {
	try {
		final MapFunction<String, Integer> parser = new MapFunction<String, Integer>() {
			@Override
			public Integer map(String value) {
				return Integer.parseInt(value);
			}
		};
		
		MapOperatorBase<String, Integer, MapFunction<String, Integer>> op = new MapOperatorBase<String, Integer, MapFunction<String,Integer>>(
				parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), "TestMapper");
		
		List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		List<Integer> resultMutableSafe = op.executeOnCollections(input, null, executionConfig);
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = op.executeOnCollections(input, null, executionConfig);
		
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 9
Source Project: flink   File: UnaryOperatorTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Parameterized.Parameters
public static Collection<Object[]> getConfigurations() {
	ExecutionConfig withReuse = new ExecutionConfig();
	withReuse.enableObjectReuse();

	ExecutionConfig withoutReuse = new ExecutionConfig();
	withoutReuse.disableObjectReuse();

	Object[] a = { withoutReuse };
	Object[] b = { withReuse };
	return Arrays.asList(a, b);
}
 
Example 10
Source Project: flink   File: ReduceOperatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReduceCollection() {
	try {
		final ReduceFunction<Tuple2<String, Integer>> reducer =
				(value1, value2) -> new Tuple2<>(value1.f0, value1.f1 + value2.f1);

		ReduceOperatorBase<Tuple2<String, Integer>, ReduceFunction<Tuple2<String, Integer>>> op =
				new ReduceOperatorBase<>(
						reducer,
						new UnaryOperatorInformation<>(STRING_INT_TUPLE, STRING_INT_TUPLE),
						new int[]{0},
						"TestReducer");

		List<Tuple2<String, Integer>> input = new ArrayList<>(asList(
				new Tuple2<>("foo", 1),
				new Tuple2<>("foo", 3),
				new Tuple2<>("bar", 2),
				new Tuple2<>("bar", 4)));

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input, null, executionConfig);
		executionConfig.enableObjectReuse();
		List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input, null, executionConfig);

		Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<>(resultMutableSafe);
		Set<Tuple2<String, Integer>> resultSetRegular = new HashSet<>(resultRegular);

		Set<Tuple2<String, Integer>> expectedResult = new HashSet<>(asList(
				new Tuple2<>("foo", 4),
				new Tuple2<>("bar", 6)));

		assertEquals(expectedResult, resultSetMutableSafe);
		assertEquals(expectedResult, resultSetRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 11
Source Project: flink   File: PartitionMapOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testMapPartitionWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();
		
		final MapPartitionFunction<String, Integer> parser = new RichMapPartitionFunction<String, Integer>() {
			
			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}
			
			@Override
			public void mapPartition(Iterable<String> values, Collector<Integer> out) {
				for (String s : values) {
					out.collect(Integer.parseInt(s));
				}
			}
			
			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};
		
		MapPartitionOperatorBase<String, Integer, MapPartitionFunction<String, Integer>> op = 
				new MapPartitionOperatorBase<String, Integer, MapPartitionFunction<String,Integer>>(
				parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), taskName);
		
		List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));

		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		
		List<Integer> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);
		
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
		
		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 12
@Test
public void testReduceCollectionWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();

		final ReduceFunction<Tuple2<String, Integer>> reducer = new RichReduceFunction<Tuple2<String, Integer>>() {

			@Override
			public Tuple2<String, Integer> reduce(
					Tuple2<String, Integer> value1,
					Tuple2<String, Integer> value2) throws Exception {

				return new Tuple2<>(value1.f0, value1.f1 + value2.f1);
			}

			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}

			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};

		ReduceOperatorBase<Tuple2<String, Integer>, ReduceFunction<Tuple2<String, Integer>>> op =
				new ReduceOperatorBase<>(
						reducer,
						new UnaryOperatorInformation<>(STRING_INT_TUPLE, STRING_INT_TUPLE),
						new int[]{0},
						"TestReducer");

		List<Tuple2<String, Integer>> input = new ArrayList<>(asList(
				new Tuple2<>("foo", 1),
				new Tuple2<>("foo", 3),
				new Tuple2<>("bar", 2),
				new Tuple2<>("bar", 4)));

		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);

		ExecutionConfig executionConfig = new ExecutionConfig();

		executionConfig.disableObjectReuse();
		List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		executionConfig.enableObjectReuse();
		List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<>(resultMutableSafe);
		Set<Tuple2<String, Integer>> resultSetRegular = new HashSet<>(resultRegular);

		Set<Tuple2<String, Integer>> expectedResult = new HashSet<>(asList(
				new Tuple2<>("foo", 4),
				new Tuple2<>("bar", 6)));

		assertEquals(expectedResult, resultSetMutableSafe);
		assertEquals(expectedResult, resultSetRegular);

		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 13
@Test
public void testTupleBaseJoiner(){
	final FlatJoinFunction<Tuple3<String, Double, Integer>, Tuple2<Integer, String>, Tuple2<Double, String>> joiner =
				new FlatJoinFunction<Tuple3<String, Double, Integer>, Tuple2<Integer, String>, Tuple2<Double, String>>() {
		@Override
		public void join(Tuple3<String, Double, Integer> first, Tuple2<Integer, String> second, Collector<Tuple2<Double, String>> out) {

			assertEquals(first.f0, second.f1);
			assertEquals(first.f2, second.f0);

			out.collect(new Tuple2<>(first.f1, second.f0.toString()));
		}
	};

	final TupleTypeInfo<Tuple3<String, Double, Integer>> leftTypeInfo = TupleTypeInfo.getBasicTupleTypeInfo
			(String.class, Double.class, Integer.class);
	final TupleTypeInfo<Tuple2<Integer, String>> rightTypeInfo = TupleTypeInfo.getBasicTupleTypeInfo(Integer.class,
			String.class);
	final TupleTypeInfo<Tuple2<Double, String>> outTypeInfo = TupleTypeInfo.getBasicTupleTypeInfo(Double.class,
			String.class);

	final int[] leftKeys = new int[]{0, 2};
	final int[] rightKeys = new int[]{1, 0};

	final String taskName = "Collection based tuple joiner";

	final BinaryOperatorInformation<Tuple3<String, Double, Integer>, Tuple2<Integer, String>, Tuple2<Double,
			String>> binaryOpInfo = new BinaryOperatorInformation<Tuple3<String, Double, Integer>, Tuple2<Integer,
			String>, Tuple2<Double, String>>(leftTypeInfo, rightTypeInfo, outTypeInfo);

	final InnerJoinOperatorBase<Tuple3<String, Double, Integer>, Tuple2<Integer,
					String>, Tuple2<Double, String>, FlatJoinFunction<Tuple3<String, Double, Integer>, Tuple2<Integer,
					String>, Tuple2<Double, String>>> base = new InnerJoinOperatorBase<Tuple3<String, Double, Integer>,
									Tuple2<Integer, String>, Tuple2<Double, String>, FlatJoinFunction<Tuple3<String, Double, Integer>,
									Tuple2<Integer, String>, Tuple2<Double, String>>>(joiner, binaryOpInfo, leftKeys, rightKeys, taskName);

	final List<Tuple3<String, Double, Integer> > inputData1 = new ArrayList<Tuple3<String, Double,
			Integer>>(Arrays.asList(
			new Tuple3<>("foo", 42.0, 1),
			new Tuple3<>("bar", 1.0, 2),
			new Tuple3<>("bar", 2.0, 3),
			new Tuple3<>("foobar", 3.0, 4),
			new Tuple3<>("bar", 3.0, 3)
	));

	final List<Tuple2<Integer, String>> inputData2 = new ArrayList<Tuple2<Integer, String>>(Arrays.asList(
			new Tuple2<>(3, "bar"),
			new Tuple2<>(4, "foobar"),
			new Tuple2<>(2, "foo")
	));
	final Set<Tuple2<Double, String>> expected = new HashSet<Tuple2<Double, String>>(Arrays.asList(
			new Tuple2<>(2.0, "3"),
			new Tuple2<>(3.0, "3"),
			new Tuple2<>(3.0, "4")
	));

	try {
		final TaskInfo taskInfo = new TaskInfo("op", 1, 0, 1, 0);
		ExecutionConfig executionConfig = new ExecutionConfig();

		executionConfig.disableObjectReuse();
		List<Tuple2<Double, String>> resultSafe = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		executionConfig.enableObjectReuse();
		List<Tuple2<Double, String>> resultRegular = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<String, Future<Path>>(),
						new HashMap<String, Accumulator<?, ?>>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		assertEquals(expected, new HashSet<>(resultSafe));
		assertEquals(expected, new HashSet<>(resultRegular));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 14
@Test
public void testJoinRich(){
	final AtomicBoolean opened = new AtomicBoolean(false);
	final AtomicBoolean closed = new AtomicBoolean(false);
	final String taskName = "Test rich join function";

	final RichFlatJoinFunction<String, String, Integer> joiner = new RichFlatJoinFunction<String, String, Integer>() {
		@Override
		public void open(Configuration parameters) throws Exception {
			opened.compareAndSet(false, true);
			assertEquals(0, getRuntimeContext().getIndexOfThisSubtask());
			assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
		}

		@Override
		public void close() throws Exception{
			closed.compareAndSet(false, true);
		}

		@Override
		public void join(String first, String second, Collector<Integer> out) throws Exception {
			out.collect(first.length());
			out.collect(second.length());
		}
	};

	InnerJoinOperatorBase<String, String, Integer,
					RichFlatJoinFunction<String, String, Integer>> base = new InnerJoinOperatorBase<String, String, Integer,
									RichFlatJoinFunction<String, String, Integer>>(joiner, new BinaryOperatorInformation<String, String,
			Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.INT_TYPE_INFO), new int[0], new int[0], taskName);

	final List<String> inputData1 = new ArrayList<String>(Arrays.asList("foo", "bar", "foobar"));
	final List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
	final List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6, 6));


	try {
		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();

		ExecutionConfig executionConfig = new ExecutionConfig();
		
		executionConfig.disableObjectReuse();
		List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);

		assertEquals(expected, resultSafe);
		assertEquals(expected, resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}

	assertTrue(opened.get());
	assertTrue(closed.get());
}
 
Example 15
Source Project: flink   File: MapOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testMapWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();
		
		final MapFunction<String, Integer> parser = new RichMapFunction<String, Integer>() {
			
			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}
			
			@Override
			public Integer map(String value) {
				return Integer.parseInt(value);
			}
			
			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};
		
		MapOperatorBase<String, Integer, MapFunction<String, Integer>> op = new MapOperatorBase<String, Integer, MapFunction<String,Integer>>(
				parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), taskName);
		
		List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		
		List<Integer> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
		
		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 16
Source Project: flink   File: InnerJoinOperatorBaseTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testJoinRich(){
	final AtomicBoolean opened = new AtomicBoolean(false);
	final AtomicBoolean closed = new AtomicBoolean(false);
	final String taskName = "Test rich join function";

	final RichFlatJoinFunction<String, String, Integer> joiner = new RichFlatJoinFunction<String, String, Integer>() {
		@Override
		public void open(Configuration parameters) throws Exception {
			opened.compareAndSet(false, true);
			assertEquals(0, getRuntimeContext().getIndexOfThisSubtask());
			assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
		}

		@Override
		public void close() throws Exception{
			closed.compareAndSet(false, true);
		}

		@Override
		public void join(String first, String second, Collector<Integer> out) throws Exception {
			out.collect(first.length());
			out.collect(second.length());
		}
	};

	InnerJoinOperatorBase<String, String, Integer,
					RichFlatJoinFunction<String, String, Integer>> base = new InnerJoinOperatorBase<String, String, Integer,
									RichFlatJoinFunction<String, String, Integer>>(joiner, new BinaryOperatorInformation<String, String,
			Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
			BasicTypeInfo.INT_TYPE_INFO), new int[0], new int[0], taskName);

	final List<String> inputData1 = new ArrayList<String>(Arrays.asList("foo", "bar", "foobar"));
	final List<String> inputData2 = new ArrayList<String>(Arrays.asList("foobar", "foo"));
	final List<Integer> expected = new ArrayList<Integer>(Arrays.asList(3, 3, 6, 6));


	try {
		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();

		ExecutionConfig executionConfig = new ExecutionConfig();
		
		executionConfig.disableObjectReuse();
		List<Integer> resultSafe = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = base.executeOnCollections(inputData1, inputData2,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);

		assertEquals(expected, resultSafe);
		assertEquals(expected, resultRegular);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}

	assertTrue(opened.get());
	assertTrue(closed.get());
}
 
Example 17
Source Project: Flink-CEPplus   File: MapOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testMapWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();
		
		final MapFunction<String, Integer> parser = new RichMapFunction<String, Integer>() {
			
			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}
			
			@Override
			public Integer map(String value) {
				return Integer.parseInt(value);
			}
			
			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};
		
		MapOperatorBase<String, Integer, MapFunction<String, Integer>> op = new MapOperatorBase<String, Integer, MapFunction<String,Integer>>(
				parser, new UnaryOperatorInformation<String, Integer>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO), taskName);
		
		List<String> input = new ArrayList<String>(asList("1", "2", "3", "4", "5", "6"));
		final HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<String, Accumulator<?, ?>>();
		final HashMap<String, Future<Path>> cpTasks = new HashMap<>();
		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.disableObjectReuse();
		
		List<Integer> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		executionConfig.enableObjectReuse();
		List<Integer> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
						accumulatorMap, new UnregisteredMetricsGroup()),
				executionConfig);
		
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultMutableSafe);
		assertEquals(asList(1, 2, 3, 4, 5, 6), resultRegular);
		
		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 18
@BeforeClass
public static void setupExecutionGraph() throws Exception {
	// -------------------------------------------------------------------------------------------------------------
	// Setup
	// -------------------------------------------------------------------------------------------------------------

	JobVertexID v1ID = new JobVertexID();
	JobVertexID v2ID = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", v1ID);
	JobVertex v2 = new JobVertex("v2", v2ID);

	v1.setParallelism(1);
	v2.setParallelism(2);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	JobGraph jobGraph = new JobGraph(v1, v2);
	ExecutionConfig config = new ExecutionConfig();

	config.setExecutionMode(ExecutionMode.BATCH_FORCED);
	config.setRestartStrategy(new RestartStrategies.NoRestartStrategyConfiguration());
	config.setParallelism(4);
	config.enableObjectReuse();
	config.setGlobalJobParameters(new TestJobParameters());

	jobGraph.setExecutionConfig(config);

	runtimeGraph = TestingExecutionGraphBuilder
		.newBuilder()
		.setJobGraph(jobGraph)
		.build();

	runtimeGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	List<ExecutionJobVertex> jobVertices = new ArrayList<>();
	jobVertices.add(runtimeGraph.getJobVertex(v1ID));
	jobVertices.add(runtimeGraph.getJobVertex(v2ID));

	CheckpointStatsTracker statsTracker = new CheckpointStatsTracker(
			0,
			jobVertices,
			mock(CheckpointCoordinatorConfiguration.class),
			new UnregisteredMetricsGroup());

	CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration(
		100,
		100,
		100,
		1,
		CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
		true,
		false,
		false,
		0);

	runtimeGraph.enableCheckpointing(
		chkConfig,
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<MasterTriggerRestoreHook<?>>emptyList(),
		new StandaloneCheckpointIDCounter(),
		new StandaloneCompletedCheckpointStore(1),
		new MemoryStateBackend(),
		statsTracker);

	runtimeGraph.setJsonPlan("{}");

	runtimeGraph.getJobVertex(v2ID).getTaskVertices()[0].getCurrentExecutionAttempt().fail(new RuntimeException("This exception was thrown on purpose."));
}
 
Example 19
Source Project: flink   File: ReduceOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testReduceCollectionWithRuntimeContext() {
	try {
		final String taskName = "Test Task";
		final AtomicBoolean opened = new AtomicBoolean();
		final AtomicBoolean closed = new AtomicBoolean();

		final ReduceFunction<Tuple2<String, Integer>> reducer = new RichReduceFunction<Tuple2<String, Integer>>() {

			@Override
			public Tuple2<String, Integer> reduce(
					Tuple2<String, Integer> value1,
					Tuple2<String, Integer> value2) throws Exception {

				return new Tuple2<>(value1.f0, value1.f1 + value2.f1);
			}

			@Override
			public void open(Configuration parameters) throws Exception {
				opened.set(true);
				RuntimeContext ctx = getRuntimeContext();
				assertEquals(0, ctx.getIndexOfThisSubtask());
				assertEquals(1, ctx.getNumberOfParallelSubtasks());
				assertEquals(taskName, ctx.getTaskName());
			}

			@Override
			public void close() throws Exception {
				closed.set(true);
			}
		};

		ReduceOperatorBase<Tuple2<String, Integer>, ReduceFunction<Tuple2<String, Integer>>> op =
				new ReduceOperatorBase<>(
						reducer,
						new UnaryOperatorInformation<>(STRING_INT_TUPLE, STRING_INT_TUPLE),
						new int[]{0},
						"TestReducer");

		List<Tuple2<String, Integer>> input = new ArrayList<>(asList(
				new Tuple2<>("foo", 1),
				new Tuple2<>("foo", 3),
				new Tuple2<>("bar", 2),
				new Tuple2<>("bar", 4)));

		final TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);

		ExecutionConfig executionConfig = new ExecutionConfig();

		executionConfig.disableObjectReuse();
		List<Tuple2<String, Integer>> resultMutableSafe = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		executionConfig.enableObjectReuse();
		List<Tuple2<String, Integer>> resultRegular = op.executeOnCollections(input,
				new RuntimeUDFContext(taskInfo, null, executionConfig,
						new HashMap<>(),
						new HashMap<>(),
						new UnregisteredMetricsGroup()),
				executionConfig);

		Set<Tuple2<String, Integer>> resultSetMutableSafe = new HashSet<>(resultMutableSafe);
		Set<Tuple2<String, Integer>> resultSetRegular = new HashSet<>(resultRegular);

		Set<Tuple2<String, Integer>> expectedResult = new HashSet<>(asList(
				new Tuple2<>("foo", 4),
				new Tuple2<>("bar", 6)));

		assertEquals(expectedResult, resultSetMutableSafe);
		assertEquals(expectedResult, resultSetRegular);

		assertTrue(opened.get());
		assertTrue(closed.get());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 20
@BeforeClass
public static void setupExecutionGraph() throws Exception {
	// -------------------------------------------------------------------------------------------------------------
	// Setup
	// -------------------------------------------------------------------------------------------------------------

	JobVertexID v1ID = new JobVertexID();
	JobVertexID v2ID = new JobVertexID();

	JobVertex v1 = new JobVertex("v1", v1ID);
	JobVertex v2 = new JobVertex("v2", v2ID);

	v1.setParallelism(1);
	v2.setParallelism(2);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);

	List<JobVertex> vertices = new ArrayList<>(Arrays.asList(v1, v2));

	ExecutionConfig config = new ExecutionConfig();

	config.setExecutionMode(ExecutionMode.BATCH_FORCED);
	config.setRestartStrategy(new RestartStrategies.NoRestartStrategyConfiguration());
	config.setParallelism(4);
	config.enableObjectReuse();
	config.setGlobalJobParameters(new TestJobParameters());

	runtimeGraph = new ExecutionGraph(
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		new JobID(),
		"test job",
		new Configuration(),
		new SerializedValue<>(config),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		mock(SlotProvider.class));

	runtimeGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	runtimeGraph.attachJobGraph(vertices);

	List<ExecutionJobVertex> jobVertices = new ArrayList<>();
	jobVertices.add(runtimeGraph.getJobVertex(v1ID));
	jobVertices.add(runtimeGraph.getJobVertex(v2ID));

	CheckpointStatsTracker statsTracker = new CheckpointStatsTracker(
			0,
			jobVertices,
			mock(CheckpointCoordinatorConfiguration.class),
			new UnregisteredMetricsGroup());

	CheckpointCoordinatorConfiguration chkConfig = new CheckpointCoordinatorConfiguration(
		100,
		100,
		100,
		1,
		CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
		true,
		false,
		0);

	runtimeGraph.enableCheckpointing(
		chkConfig,
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<ExecutionJobVertex>emptyList(),
		Collections.<MasterTriggerRestoreHook<?>>emptyList(),
		new StandaloneCheckpointIDCounter(),
		new StandaloneCompletedCheckpointStore(1),
		new MemoryStateBackend(),
		statsTracker);

	runtimeGraph.setJsonPlan("{}");

	runtimeGraph.getJobVertex(v2ID).getTaskVertices()[0].getCurrentExecutionAttempt().fail(new RuntimeException("This exception was thrown on purpose."));
}