org.apache.flink.api.common.operators.Order Java Examples
The following examples show how to use
org.apache.flink.api.common.operators.Order.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SortPartitionITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testSortPartitionByKeyField() throws Exception { /* * Test sort partition on key field */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper<Tuple3<Integer, Long, String>>()).setParallelism(4) // parallelize input .sortPartition(1, Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new Tuple3Checker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
Example #2
Source File: SortPartitionITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testSortPartitionByTwoFieldExpressions() throws Exception { /* * Test sort partition on two field expressions */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(2); DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper<Tuple5<Integer, Long, Integer, String, Long>>()).setParallelism(2) // parallelize input .sortPartition("f4", Order.ASCENDING) .sortPartition("f2", Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new Tuple5Checker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
Example #3
Source File: GroupReduceWithCombineProperties.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) { super(groupKeys); // if we have an additional ordering, construct the ordering to have primarily the grouping fields if (additionalOrderKeys != null) { this.ordering = new Ordering(); for (Integer key : this.keyList) { this.ordering.appendOrdering(key, null, Order.ANY); } // and next the additional order fields for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) { Integer field = additionalOrderKeys.getFieldNumber(i); Order order = additionalOrderKeys.getOrder(i); this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order); } } else { this.ordering = null; } this.customPartitioner = customPartitioner; }
Example #4
Source File: SortPartitionITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testSortPartitionPojoByNestedFieldExpression() throws Exception { /* * Test sort partition on field expression */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(3); DataSet<POJO> ds = CollectionDataSets.getMixedPojoDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper<POJO>()).setParallelism(1) // parallelize input .sortPartition("nestedTupleWithCustom.f1.myString", Order.ASCENDING) .sortPartition("number", Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new PojoChecker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
Example #5
Source File: DataSinkTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testTupleTwoOrderIdx() { final ExecutionEnvironment env = ExecutionEnvironment .getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env .fromCollection(emptyTupleData, tupleTypeInfo); // should work try { tupleDs.writeAsText("/tmp/willNotHappen") .sortLocalOutput(0, Order.ASCENDING) .sortLocalOutput(3, Order.DESCENDING); } catch (Exception e) { Assert.fail(); } }
Example #6
Source File: SortPartitionITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testSortPartitionWithKeySelector2() throws Exception { /* * Test sort partition on an extracted key */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(4); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper<Tuple3<Integer, Long, String>>()).setParallelism(4) // parallelize input .sortPartition(new KeySelector<Tuple3<Integer, Long, String>, Tuple2<Integer, Long>>() { @Override public Tuple2<Integer, Long> getKey(Tuple3<Integer, Long, String> value) throws Exception { return new Tuple2<>(value.f0, value.f1); } }, Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new Tuple3Checker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
Example #7
Source File: GroupReduceITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testStringBasedDefinitionOnGroupSortForTwoGroupingKeysWithPojos() throws Exception { /* * Test string-based definition on group sort, for two grouping keys with Pojos */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); DataSet<PojoContainingTupleAndWritable> ds = CollectionDataSets.getGroupSortedPojoContainingTupleAndWritable(env); // f0.f0 is first integer DataSet<String> reduceDs = ds.groupBy("hadoopFan").sortGroup("theTuple.f0", Order.DESCENDING).sortGroup("theTuple.f1", Order.DESCENDING) .reduceGroup(new GroupReducer5()); List<String> result = reduceDs.collect(); String expected = "1---(10,100)-\n" + "2---(30,600)-(30,400)-(30,200)-(20,201)-(20,200)-\n"; compareResultAsText(result, expected); }
Example #8
Source File: GroupReduceITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testIntBasedDefinitionOnGroupSortForFullNestedTuple() throws Exception { /* * Test int-based definition on group sort, for (full) nested Tuple */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env); DataSet<String> reduceDs = ds.groupBy("f1").sortGroup(0, Order.DESCENDING).reduceGroup(new NestedTupleReducer()); List<String> result = reduceDs.collect(); String expected = "a--(2,1)-(1,3)-(1,2)-\n" + "b--(2,2)-\n" + "c--(4,9)-(3,6)-(3,3)-\n"; compareResultAsText(result, expected); }
Example #9
Source File: DataSinkTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testTupleTwoOrderExp() { final ExecutionEnvironment env = ExecutionEnvironment .getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env .fromCollection(emptyTupleData, tupleTypeInfo); // should work try { tupleDs.writeAsText("/tmp/willNotHappen") .sortLocalOutput("f1", Order.ASCENDING) .sortLocalOutput("f4", Order.DESCENDING); } catch (Exception e) { Assert.fail(); } }
Example #10
Source File: GroupReduceITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testTupleKeySelectorSortCombineOnTuple() throws Exception { /* * check correctness of sorted groupReduceon with Tuple2 keyselector sorting */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env); DataSet<Tuple5<Integer, Long, Integer, String, Long>> reduceDs = ds .groupBy(new IntFieldExtractor<Tuple5<Integer, Long, Integer, String, Long>>(0)) .sortGroup(new FiveToTwoTupleExtractor(), Order.DESCENDING) .reduceGroup(new Tuple5SortedGroupReduce()); List<Tuple5<Integer, Long, Integer, String, Long>> result = reduceDs.collect(); String expected = "1,1,0,Hallo,1\n" + "2,5,0,Hallo Welt-Hallo Welt wie,1\n" + "3,15,0,BCD-ABC-Hallo Welt wie gehts?,2\n" + "4,34,0,FGH-CDE-EFG-DEF,1\n" + "5,65,0,IJK-HIJ-KLM-JKL-GHI,1\n"; compareResultAsTuples(result, expected); }
Example #11
Source File: SortPartitionITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testSortPartitionParallelismChange() throws Exception { /* * Test sort partition with parallelism change */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(3); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); List<Tuple1<Boolean>> result = ds .sortPartition(1, Order.DESCENDING).setParallelism(3) // change parallelism .mapPartition(new OrderCheckMapper<>(new Tuple3Checker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
Example #12
Source File: DataSinkITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testStringSortingParallelism1() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<String> ds = CollectionDataSets.getStringDataSet(env); ds.writeAsText(resultPath).sortLocalOutput("*", Order.ASCENDING).setParallelism(1); env.execute(); String expected = "Hello\n" + "Hello world\n" + "Hello world, how are you?\n" + "Hi\n" + "I am fine.\n" + "LOL\n" + "Luke Skywalker\n" + "Random comment\n"; compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath); }
Example #13
Source File: DataSinkITCase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testPojoSortingSingleParallelism1() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<CollectionDataSets.POJO> ds = CollectionDataSets.getMixedPojoDataSet(env); ds.writeAsText(resultPath).sortLocalOutput("number", Order.ASCENDING).setParallelism(1); env.execute(); String expected = "1 First (10,100,1000,One) 10100\n" + "2 First_ (10,105,1000,One) 10200\n" + "3 First (11,102,3000,One) 10200\n" + "4 First_ (11,106,1000,One) 10300\n" + "5 First (11,102,2000,One) 10100\n" + "6 Second_ (20,200,2000,Two) 10100\n" + "7 Third (31,301,2000,Three) 10200\n" + "8 Third_ (30,300,1000,Three) 10100\n"; compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath); }
Example #14
Source File: GroupCombineITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testIdentityWithGroupByAndSort() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); DataSet<Tuple3<Integer, Long, String>> reduceDs = ds .groupBy(1) .sortGroup(1, Order.DESCENDING) // reduce partially .combineGroup(new IdentityFunction()) .groupBy(1) .sortGroup(1, Order.DESCENDING) // fully reduce .reduceGroup(new IdentityFunction()); List<Tuple3<Integer, Long, String>> result = reduceDs.collect(); compareResultAsTuples(result, identityResult); }
Example #15
Source File: LocalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testSortingErased() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;5"}, null, null, tupleInfo, tupleInfo); Ordering o = new Ordering(); o.appendOrdering(2, IntValue.class, Order.ASCENDING); o.appendOrdering(0, StringValue.class, Order.DESCENDING); o.appendOrdering(5, LongValue.class, Order.DESCENDING); LocalProperties lProps = LocalProperties.forOrdering(o); LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0); FieldList gFields = filtered.getGroupedFields(); Ordering order = filtered.getOrdering(); assertNull(gFields); assertNull(order); assertNull(filtered.getUniqueFields()); }
Example #16
Source File: GroupReduceProperties.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public GroupReduceProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) { super(groupKeys); // if we have an additional ordering, construct the ordering to have primarily the grouping fields if (additionalOrderKeys != null) { this.ordering = new Ordering(); for (Integer key : this.keyList) { this.ordering.appendOrdering(key, null, Order.ANY); } // and next the additional order fields for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) { Integer field = additionalOrderKeys.getFieldNumber(i); Order order = additionalOrderKeys.getOrder(i); this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order); } } else { this.ordering = null; } this.customPartitioner = customPartitioner; }
Example #17
Source File: DataSinkITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testTupleSortingNestedParallelism1_2() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple3<Tuple2<Integer, Integer>, String, Integer>> ds = CollectionDataSets.getGroupSortedNestedTupleDataSet2(env); ds.writeAsText(resultPath) .sortLocalOutput(1, Order.ASCENDING) .sortLocalOutput(2, Order.DESCENDING) .setParallelism(1); env.execute(); String expected = "((2,1),a,3)\n" + "((1,3),a,2)\n" + "((1,2),a,1)\n" + "((2,2),b,4)\n" + "((4,9),c,7)\n" + "((3,6),c,6)\n" + "((3,3),c,5)\n"; compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath); }
Example #18
Source File: GroupingTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testGroupSortByKeyExpression3() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo); // should work try { tupleDs.groupBy("f0") .sortGroup("f2.myString", Order.ASCENDING) .sortGroup("f1", Order.DESCENDING); } catch (Exception e) { Assert.fail(); } }
Example #19
Source File: SortPartitionITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testSortPartitionPojoByNestedFieldExpression() throws Exception { /* * Test sort partition on field expression */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(3); DataSet<POJO> ds = CollectionDataSets.getMixedPojoDataSet(env); List<Tuple1<Boolean>> result = ds .map(new IdMapper<POJO>()).setParallelism(1) // parallelize input .sortPartition("nestedTupleWithCustom.f1.myString", Order.ASCENDING) .sortPartition("number", Order.DESCENDING) .mapPartition(new OrderCheckMapper<>(new PojoChecker())) .distinct().collect(); String expected = "(true)\n"; compareResultAsText(result, expected); }
Example #20
Source File: GroupingTest.java From flink with Apache License 2.0 | 6 votes |
@Test(expected = InvalidProgramException.class) public void testGroupSortKeyFields3() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO); // should not work: sorted groups on groupings by key selectors longDs.groupBy(new KeySelector<Long, Long>() { private static final long serialVersionUID = 1L; @Override public Long getKey(Long value) { return value; } }).sortGroup(0, Order.ASCENDING); }
Example #21
Source File: DataSinkITCase.java From flink with Apache License 2.0 | 6 votes |
@Test public void testPojoSortingSingleParallelism1() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<CollectionDataSets.POJO> ds = CollectionDataSets.getMixedPojoDataSet(env); ds.writeAsText(resultPath).sortLocalOutput("number", Order.ASCENDING).setParallelism(1); env.execute(); String expected = "1 First (10,100,1000,One) 10100\n" + "2 First_ (10,105,1000,One) 10200\n" + "3 First (11,102,3000,One) 10200\n" + "4 First_ (11,106,1000,One) 10300\n" + "5 First (11,102,2000,One) 10100\n" + "6 Second_ (20,200,2000,Two) 10100\n" + "7 Third (31,301,2000,Three) 10200\n" + "8 Third_ (30,300,1000,Three) 10100\n"; compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath); }
Example #22
Source File: LocalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testSortingPreserved4() { SingleInputSemanticProperties sp = new SingleInputSemanticProperties(); SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"2->7;5"}, null, null, tupleInfo, tupleInfo); Ordering o = new Ordering(); o.appendOrdering(2, IntValue.class, Order.ASCENDING); o.appendOrdering(0, StringValue.class, Order.DESCENDING); o.appendOrdering(5, LongValue.class, Order.DESCENDING); LocalProperties lProps = LocalProperties.forOrdering(o); LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0); FieldList gFields = filtered.getGroupedFields(); Ordering order = filtered.getOrdering(); assertNotNull(gFields); assertEquals(1, gFields.size()); assertTrue(gFields.contains(7)); assertNotNull(order); assertEquals(1, order.getNumberOfFields()); assertEquals(7, order.getFieldNumber(0).intValue()); assertEquals(Order.ASCENDING, order.getOrder(0)); assertEquals(IntValue.class, order.getType(0)); assertNull(filtered.getUniqueFields()); }
Example #23
Source File: PartitionOperator.java From flink with Apache License 2.0 | 6 votes |
@SuppressWarnings("unchecked") private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner( SelectorFunctionKeys<T, ?> rawKeys, PartitionMethod pMethod, String name, Operator<T> input, int partitionDop, Partitioner<?> customPartitioner, Order[] orders) { final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys; TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys); Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys); PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput = new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[]{0}, name); keyedPartitionedInput.setInput(keyedInput); keyedPartitionedInput.setCustomPartitioner(customPartitioner); keyedPartitionedInput.setParallelism(partitionDop); keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING)); return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys); }
Example #24
Source File: DataSinkTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testTupleTwoOrderIdx() { final ExecutionEnvironment env = ExecutionEnvironment .getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env .fromCollection(emptyTupleData, tupleTypeInfo); // should work try { tupleDs.writeAsText("/tmp/willNotHappen") .sortLocalOutput(0, Order.ASCENDING) .sortLocalOutput(3, Order.DESCENDING); } catch (Exception e) { Assert.fail(); } }
Example #25
Source File: PropertyDataSourceTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void checkSinglePartitionedOrderedSource6() { ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(); env.setParallelism(DEFAULT_PARALLELISM); DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType); data.getSplitDataProperties() .splitsPartitionedBy("f1.intField") .splitsOrderedBy("f1", new Order[]{Order.DESCENDING}); data.output(new DiscardingOutputFormat<Tuple3<Long, SomePojo, String>>()); Plan plan = env.createProgramPlan(); // submit the plan to the compiler OptimizedPlan oPlan = compileNoStats(plan); // check the optimized Plan SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next(); SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor(); GlobalProperties gprops = sourceNode.getGlobalProperties(); LocalProperties lprops = sourceNode.getLocalProperties(); Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2))); Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING); Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(1,2,3))); Assert.assertTrue(lprops.getOrdering() == null); }
Example #26
Source File: GroupCombineITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test // check if all API methods are callable public void testAPI() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple1<String>> ds = CollectionDataSets.getStringDataSet(env).map(new MapFunction<String, Tuple1<String>>() { @Override public Tuple1<String> map(String value) throws Exception { return new Tuple1<>(value); } }); // all methods on DataSet ds.combineGroup(new GroupCombineFunctionExample()) .output(new DiscardingOutputFormat<Tuple1<String>>()); // all methods on UnsortedGrouping ds.groupBy(0).combineGroup(new GroupCombineFunctionExample()) .output(new DiscardingOutputFormat<Tuple1<String>>()); // all methods on SortedGrouping ds.groupBy(0).sortGroup(0, Order.ASCENDING).combineGroup(new GroupCombineFunctionExample()) .output(new DiscardingOutputFormat<Tuple1<String>>()); env.execute(); }
Example #27
Source File: GroupingTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGroupSortByKeyExpression2() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo); // should work try { tupleDs.groupBy("f0").sortGroup("f2.myString", Order.ASCENDING); } catch (Exception e) { Assert.fail(); } }
Example #28
Source File: GroupReduceITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTupleKeySelectorGroupSort() throws Exception { /* * check correctness of sorted groupReduce on tuples with keyselector sorting */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(1); DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env); DataSet<Tuple3<Integer, Long, String>> reduceDs = ds .groupBy(new LongFieldExtractor<Tuple3<Integer, Long, String>>(1)) .sortGroup(new StringFieldExtractor<Tuple3<Integer, Long, String>>(2), Order.DESCENDING) .reduceGroup(new Tuple3SortedGroupReduce()); List<Tuple3<Integer, Long, String>> result = reduceDs.collect(); String expected = "1,1,Hi\n" + "5,2,Hello world-Hello\n" + "15,3,Luke Skywalker-I am fine.-Hello world, how are you?\n" + "34,4,Comment#4-Comment#3-Comment#2-Comment#1\n" + "65,5,Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" + "111,6,Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n"; compareResultAsTuples(result, expected); }
Example #29
Source File: GroupingTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testChainedGroupSortKeyFields() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo); // should work try { tupleDs.groupBy(0).sortGroup(0, Order.ASCENDING).sortGroup(2, Order.DESCENDING); } catch (Exception e) { Assert.fail(); } }
Example #30
Source File: GroupingTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGroupSortKeyFields1() { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo); // should work try { tupleDs.groupBy(0).sortGroup(0, Order.ASCENDING); } catch (Exception e) { Assert.fail(); } }