org.apache.flink.api.common.functions.GroupReduceFunction Java Examples
The following examples show how to use
org.apache.flink.api.common.functions.GroupReduceFunction.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AllGroupReduceDriverTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testAllReduceDriverImmutableEmpty() { try { TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context = new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>(); List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData(); TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0)); MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get(); context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE); context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig())); context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>()); AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>(); driver.setup(context); driver.prepare(); driver.run(); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #2
Source File: AvroTypeExtractionTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testWithKryoGenericSer() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().enableForceKryo(); Path in = new Path(inFile.getAbsoluteFile().toURI()); AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class); DataSet<User> usersDS = env.createInput(users); DataSet<Tuple2<String, Integer>> res = usersDS .groupBy((KeySelector<User, String>) value -> String.valueOf(value.getName())) .reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> { for (User u : values) { out.collect(new Tuple2<>(u.getName().toString(), 1)); } }) .returns(Types.TUPLE(Types.STRING, Types.INT)); res.writeAsText(resultPath); env.execute("Avro Key selection"); expected = "(Charlie,1)\n(Alyssa,1)\n"; }
Example #3
Source File: AvroTypeExtractionTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testKeySelection() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().enableObjectReuse(); Path in = new Path(inFile.getAbsoluteFile().toURI()); AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class); DataSet<User> usersDS = env.createInput(users); DataSet<Tuple2<String, Integer>> res = usersDS .groupBy("name") .reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> { for (User u : values) { out.collect(new Tuple2<>(u.getName().toString(), 1)); } }) .returns(Types.TUPLE(Types.STRING, Types.INT)); res.writeAsText(resultPath); env.execute("Avro Key selection"); expected = "(Alyssa,1)\n(Charlie,1)\n"; }
Example #4
Source File: GroupReduceITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testGroupReduceWithAtomicValue() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Integer> ds = env.fromElements(1, 1, 2, 3, 4); DataSet<Integer> reduceDs = ds.groupBy("*").reduceGroup(new GroupReduceFunction<Integer, Integer>() { @Override public void reduce(Iterable<Integer> values, Collector<Integer> out) throws Exception { out.collect(values.iterator().next()); } }); List<Integer> result = reduceDs.collect(); String expected = "1\n" + "2\n" + "3\n" + "4"; compareResultAsText(result, expected); }
Example #5
Source File: GroupReduceITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Fix for FLINK-2158. * * @throws Exception */ @Test public void testDateNullException() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Integer, Date>> in = env.fromElements(new Tuple2<>(0, new Date(1230000000)), new Tuple2<Integer, Date>(1, null), new Tuple2<>(2, new Date(1230000000)) ); DataSet<String> r = in.groupBy(0).reduceGroup(new GroupReduceFunction<Tuple2<Integer, Date>, String>() { @Override public void reduce(Iterable<Tuple2<Integer, Date>> values, Collector<String> out) throws Exception { for (Tuple2<Integer, Date> e : values) { out.collect(Integer.toString(e.f0)); } } }); List<String> result = r.collect(); String expected = "0\n1\n2\n"; compareResultAsText(result, expected); }
Example #6
Source File: PlanUnwrappingSortedReduceGroupOperator.java From flink with Apache License 2.0 | 6 votes |
public PlanUnwrappingSortedReduceGroupOperator( GroupReduceFunction<IN, OUT> udf, Keys.SelectorFunctionKeys<IN, K1> groupingKey, Keys.SelectorFunctionKeys<IN, K2> sortingKey, String name, TypeInformation<OUT> outType, TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey, boolean combinable) { super( combinable ? new TupleUnwrappingGroupCombinableGroupReducer<IN, OUT, K1, K2>(udf) : new TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K1, K2>(udf), new UnaryOperatorInformation<>(typeInfoWithKey, outType), groupingKey.computeLogicalKeyPositions(), name); super.setCombinable(combinable); }
Example #7
Source File: AvroTypeExtractionTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testWithAvroGenericSer() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().enableForceAvro(); Path in = new Path(inFile.getAbsoluteFile().toURI()); AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class); DataSet<User> usersDS = env.createInput(users); DataSet<Tuple2<String, Integer>> res = usersDS .groupBy((KeySelector<User, String>) value -> String.valueOf(value.getName())) .reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> { for (User u : values) { out.collect(new Tuple2<>(u.getName().toString(), 1)); } }) .returns(Types.TUPLE(Types.STRING, Types.INT)); res.writeAsText(resultPath); env.execute("Avro Key selection"); expected = "(Charlie,1)\n(Alyssa,1)\n"; }
Example #8
Source File: GroupReduceOperator.java From flink with Apache License 2.0 | 6 votes |
@SuppressWarnings("unchecked") private static <IN, OUT, K1, K2> PlanUnwrappingSortedReduceGroupOperator<IN, OUT, K1, K2> translateSelectorFunctionSortedReducer( SelectorFunctionKeys<IN, ?> rawGroupingKey, SelectorFunctionKeys<IN, ?> rawSortingKey, Ordering groupOrdering, GroupReduceFunction<IN, OUT> function, TypeInformation<OUT> outputType, String name, Operator<IN> input, boolean combinable) { final SelectorFunctionKeys<IN, K1> groupingKey = (SelectorFunctionKeys<IN, K1>) rawGroupingKey; final SelectorFunctionKeys<IN, K2> sortingKey = (SelectorFunctionKeys<IN, K2>) rawSortingKey; TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(groupingKey, sortingKey); Operator<Tuple3<K1, K2, IN>> inputWithKey = KeyFunctions.appendKeyExtractor(input, groupingKey, sortingKey); PlanUnwrappingSortedReduceGroupOperator<IN, OUT, K1, K2> reducer = new PlanUnwrappingSortedReduceGroupOperator<>( function, groupingKey, sortingKey, name, outputType, typeInfoWithKey, combinable); reducer.setInput(inputWithKey); reducer.setGroupOrder(groupOrdering); return reducer; }
Example #9
Source File: GroupReduceOperator.java From flink with Apache License 2.0 | 6 votes |
@SuppressWarnings("unchecked") private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionReducer( SelectorFunctionKeys<IN, ?> rawKeys, GroupReduceFunction<IN, OUT> function, TypeInformation<OUT> outputType, String name, Operator<IN> input, boolean combinable) { SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys; TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer = new PlanUnwrappingReduceGroupOperator(function, keys, name, outputType, typeInfoWithKey, combinable); reducer.setInput(keyedInput); return reducer; }
Example #10
Source File: GroupReduceITCase.java From flink with Apache License 2.0 | 6 votes |
/** * Fix for FLINK-2158. * * @throws Exception */ @Test public void testDateNullException() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Integer, Date>> in = env.fromElements(new Tuple2<>(0, new Date(1230000000)), new Tuple2<Integer, Date>(1, null), new Tuple2<>(2, new Date(1230000000)) ); DataSet<String> r = in.groupBy(0).reduceGroup(new GroupReduceFunction<Tuple2<Integer, Date>, String>() { @Override public void reduce(Iterable<Tuple2<Integer, Date>> values, Collector<String> out) throws Exception { for (Tuple2<Integer, Date> e : values) { out.collect(Integer.toString(e.f0)); } } }); List<String> result = r.collect(); String expected = "0\n1\n2\n"; compareResultAsText(result, expected); }
Example #11
Source File: GroupReduceITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testGroupReduceWithAtomicValue() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Integer> ds = env.fromElements(1, 1, 2, 3, 4); DataSet<Integer> reduceDs = ds.groupBy("*").reduceGroup(new GroupReduceFunction<Integer, Integer>() { @Override public void reduce(Iterable<Integer> values, Collector<Integer> out) throws Exception { out.collect(values.iterator().next()); } }); List<Integer> result = reduceDs.collect(); String expected = "1\n" + "2\n" + "3\n" + "4"; compareResultAsText(result, expected); }
Example #12
Source File: AvroTypeExtractionTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testWithKryoGenericSer() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().enableForceKryo(); Path in = new Path(inFile.getAbsoluteFile().toURI()); AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class); DataSet<User> usersDS = env.createInput(users); DataSet<Tuple2<String, Integer>> res = usersDS .groupBy((KeySelector<User, String>) value -> String.valueOf(value.getName())) .reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> { for (User u : values) { out.collect(new Tuple2<>(u.getName().toString(), 1)); } }) .returns(Types.TUPLE(Types.STRING, Types.INT)); res.writeAsText(resultPath); env.execute("Avro Key selection"); expected = "(Charlie,1)\n(Alyssa,1)\n"; }
Example #13
Source File: AvroTypeExtractionTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testWithAvroGenericSer() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().enableForceAvro(); Path in = new Path(inFile.getAbsoluteFile().toURI()); AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class); DataSet<User> usersDS = env.createInput(users); DataSet<Tuple2<String, Integer>> res = usersDS .groupBy((KeySelector<User, String>) value -> String.valueOf(value.getName())) .reduceGroup((GroupReduceFunction<User, Tuple2<String, Integer>>) (values, out) -> { for (User u : values) { out.collect(new Tuple2<>(u.getName().toString(), 1)); } }) .returns(Types.TUPLE(Types.STRING, Types.INT)); res.writeAsText(resultPath); env.execute("Avro Key selection"); expected = "(Charlie,1)\n(Alyssa,1)\n"; }
Example #14
Source File: AllGroupReduceDriverTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testAllReduceDriverImmutableEmpty() { try { TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context = new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>(); List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData(); TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0)); MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get(); context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE); context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig())); context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>()); AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>(); driver.setup(context); driver.prepare(); driver.run(); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #15
Source File: GroupReduceDriverTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testAllReduceDriverMutable() { try { TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context = new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(); List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData(); TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0)); MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig())); TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig()); GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer(new ExecutionConfig())); context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE); context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig())); context.setComparator1(comparator); context.setCollector(result); context.setUdf(new ConcatSumMutableReducer()); GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>(); driver.setup(context); driver.prepare(); driver.run(); Object[] res = result.getList().toArray(); Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray(); DriverTestData.compareTupleArrays(expected, res); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #16
Source File: GroupReduceDriverTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testAllReduceDriverAccumulatingImmutable() { try { TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context = new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(); List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData(); TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0)); MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig())); TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig()); GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer(new ExecutionConfig())); context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE); context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig())); context.setComparator1(comparator); context.setCollector(result); context.setUdf(new ConcatSumMutableAccumulatingReducer()); context.setMutableObjectMode(false); GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>(); driver.setup(context); driver.prepare(); driver.run(); Object[] res = result.getList().toArray(); Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray(); DriverTestData.compareTupleArrays(expected, res); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #17
Source File: Flink.java From iteratorx with Apache License 2.0 | 5 votes |
public long forBatch(final int batchSize, final Consumer<List<JSONObject>> onNext) throws Exception { final DataSet<List<JSONObject>> groupDataSet = dataSet .reduceGroup(new GroupReduceFunction<JSONObject, List<JSONObject>>() { private static final long serialVersionUID = -1748951891205342549L; private final AtomicInteger count = new AtomicInteger(); private final List<JSONObject> group = new ArrayList<>(); @Override public void reduce(final Iterable<JSONObject> values, final Collector<List<JSONObject>> out) throws Exception { for (final JSONObject one : values) { group.add(one); count.getAndIncrement(); if (group.size() >= batchSize) { out.collect(new ArrayList<>(group)); group.clear(); } } if (group.size() > 0) { out.collect(new ArrayList<>(group)); group.clear(); } } }); return forEach(groupDataSet, onNext); }
Example #18
Source File: GroupReduceOperator.java From flink with Apache License 2.0 | 5 votes |
/** * Constructor for a grouped reduce. * * @param input The grouped input to be processed group-wise by the groupReduce function. * @param function The user-defined GroupReduce function. */ public GroupReduceOperator(Grouping<IN> input, TypeInformation<OUT> resultType, GroupReduceFunction<IN, OUT> function, String defaultName) { super(input != null ? input.getInputDataSet() : null, resultType); this.function = function; this.grouper = input; this.defaultName = defaultName; this.combinable = checkCombinability(); }
Example #19
Source File: AllGroupReduceDriverTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testAllReduceDriverMutable() { try { TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context = new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(); List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData(); TypeInformation<Tuple2<StringValue, IntValue>> typeInfo = TypeExtractor.getForObject(data.get(0)); MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig())); GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer(new ExecutionConfig())); context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE); context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig())); context.setCollector(result); context.setUdf(new ConcatSumMutableReducer()); AllGroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new AllGroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>(); driver.setup(context); driver.prepare(); driver.run(); Tuple2<StringValue, IntValue> res = result.getList().get(0); char[] foundString = res.f0.getValue().toCharArray(); Arrays.sort(foundString); char[] expectedString = "abcddeeeffff".toCharArray(); Arrays.sort(expectedString); Assert.assertArrayEquals(expectedString, foundString); Assert.assertEquals(78, res.f1.getValue()); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #20
Source File: PlanUnwrappingSortedReduceGroupOperator.java From flink with Apache License 2.0 | 5 votes |
private TupleUnwrappingGroupCombinableGroupReducer(GroupReduceFunction<IN, OUT> wrapped) { super(wrapped); if (!GroupCombineFunction.class.isAssignableFrom(wrappedFunction.getClass())) { throw new IllegalArgumentException("Wrapped reduce function does not implement the GroupCombineFunction interface."); } this.iter = new Tuple3UnwrappingIterator<>(); this.coll = new Tuple3WrappingCollector<>(this.iter); }
Example #21
Source File: PlanUnwrappingReduceGroupOperator.java From flink with Apache License 2.0 | 5 votes |
private TupleUnwrappingGroupCombinableGroupReducer(GroupReduceFunction<IN, OUT> wrapped) { super(wrapped); if (!GroupCombineFunction.class.isAssignableFrom(wrappedFunction.getClass())) { throw new IllegalArgumentException("Wrapped reduce function does not implement the GroupCombineFunction interface."); } this.iter = new TupleUnwrappingIterator<>(); this.coll = new TupleWrappingCollector<>(this.iter); }
Example #22
Source File: GroupReduceOperator.java From flink with Apache License 2.0 | 5 votes |
/** * Constructor for a non-grouped reduce (all reduce). * * @param input The input data set to the groupReduce function. * @param function The user-defined GroupReduce function. */ public GroupReduceOperator(DataSet<IN> input, TypeInformation<OUT> resultType, GroupReduceFunction<IN, OUT> function, String defaultName) { super(input, resultType); this.function = function; this.grouper = null; this.defaultName = defaultName; this.combinable = checkCombinability(); }
Example #23
Source File: GroupReduceDriverTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testAllReduceDriverImmutable() { try { TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context = new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>(); List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData(); TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0)); MutableObjectIterator<Tuple2<String, Integer>> input = new RegularToMutableObjectIterator<Tuple2<String, Integer>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig())); TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig()); GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer(new ExecutionConfig())); context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE); context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig())); context.setCollector(result); context.setComparator1(comparator); context.setUdf(new ConcatSumReducer()); GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>(); driver.setup(context); driver.prepare(); driver.run(); Object[] res = result.getList().toArray(); Object[] expected = DriverTestData.createReduceImmutableDataGroupedResult().toArray(); DriverTestData.compareTupleArrays(expected, res); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #24
Source File: GroupReduceDriverTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testAllReduceDriverImmutableEmpty() { try { TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context = new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>(); List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData(); TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0)); MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get(); TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig()); context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE); GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer(new ExecutionConfig())); context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig())); context.setComparator1(comparator); context.setCollector(result); GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>(); driver.setup(context); driver.prepare(); driver.run(); Assert.assertTrue(result.getList().isEmpty()); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #25
Source File: AllGroupReduceDriverTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testAllReduceDriverMutable() { try { TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context = new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>(); List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData(); TypeInformation<Tuple2<StringValue, IntValue>> typeInfo = TypeExtractor.getForObject(data.get(0)); MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig())); GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer(new ExecutionConfig())); context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE); context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig())); context.setCollector(result); context.setUdf(new ConcatSumMutableReducer()); AllGroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new AllGroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>(); driver.setup(context); driver.prepare(); driver.run(); Tuple2<StringValue, IntValue> res = result.getList().get(0); char[] foundString = res.f0.getValue().toCharArray(); Arrays.sort(foundString); char[] expectedString = "abcddeeeffff".toCharArray(); Arrays.sort(expectedString); Assert.assertArrayEquals(expectedString, foundString); Assert.assertEquals(78, res.f1.getValue()); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #26
Source File: AllGroupReduceDriverTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testAllReduceDriverImmutable() { try { TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context = new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>(); List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData(); TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0)); MutableObjectIterator<Tuple2<String, Integer>> input = new RegularToMutableObjectIterator<Tuple2<String, Integer>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig())); GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer(new ExecutionConfig())); context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE); context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig())); context.setCollector(result); context.setUdf(new ConcatSumReducer()); AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>(); driver.setup(context); driver.prepare(); driver.run(); Tuple2<String,Integer> res = result.getList().get(0); char[] foundString = res.f0.toCharArray(); Arrays.sort(foundString); char[] expectedString = "abcddeeeffff".toCharArray(); Arrays.sort(expectedString); Assert.assertArrayEquals(expectedString, foundString); Assert.assertEquals(78, res.f1.intValue()); } catch (Exception e) { System.err.println(e.getMessage()); e.printStackTrace(); Assert.fail(e.getMessage()); } }
Example #27
Source File: TypeExtractor.java From flink with Apache License 2.0 | 5 votes |
@PublicEvolving public static <IN, OUT> TypeInformation<OUT> getGroupReduceReturnTypes(GroupReduceFunction<IN, OUT> groupReduceInterface, TypeInformation<IN> inType, String functionName, boolean allowMissing) { return getUnaryOperatorReturnType( (Function) groupReduceInterface, GroupReduceFunction.class, 0, 1, new int[]{1, 0}, inType, functionName, allowMissing); }
Example #28
Source File: TypeExtractor.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@PublicEvolving public static <IN, OUT> TypeInformation<OUT> getGroupReduceReturnTypes(GroupReduceFunction<IN, OUT> groupReduceInterface, TypeInformation<IN> inType, String functionName, boolean allowMissing) { return getUnaryOperatorReturnType( (Function) groupReduceInterface, GroupReduceFunction.class, 0, 1, new int[]{1, 0}, inType, functionName, allowMissing); }
Example #29
Source File: PlanUnwrappingReduceGroupOperator.java From flink with Apache License 2.0 | 5 votes |
public PlanUnwrappingReduceGroupOperator( GroupReduceFunction<IN, OUT> udf, Keys.SelectorFunctionKeys<IN, K> key, String name, TypeInformation<OUT> outType, TypeInformation<Tuple2<K, IN>> typeInfoWithKey, boolean combinable) { super( combinable ? new TupleUnwrappingGroupCombinableGroupReducer<IN, OUT, K>(udf) : new TupleUnwrappingNonCombinableGroupReducer<IN, OUT, K>(udf), new UnaryOperatorInformation<>(typeInfoWithKey, outType), key.computeLogicalKeyPositions(), name); super.setCombinable(combinable); }
Example #30
Source File: GroupCombineChainedDriver.java From flink with Apache License 2.0 | 5 votes |
@Override public void setup(AbstractInvokable parent) { this.parent = parent; @SuppressWarnings("unchecked") final GroupReduceFunction<IN, OUT> combiner = BatchTask.instantiateUserCode(this.config, userCodeClassLoader, GroupReduceFunction.class); this.reducer = combiner; FunctionUtils.setFunctionRuntimeContext(combiner, getUdfRuntimeContext()); }