org.apache.flink.api.common.operators.Order Java Examples

The following examples show how to use org.apache.flink.api.common.operators.Order. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: Flink-CEPplus   Author: ljygz   File: SortPartitionITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSortPartitionByKeyField() throws Exception {
	/*
	 * Test sort partition on key field
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	List<Tuple1<Boolean>> result = ds
			.map(new IdMapper<Tuple3<Integer, Long, String>>()).setParallelism(4) // parallelize input
			.sortPartition(1, Order.DESCENDING)
			.mapPartition(new OrderCheckMapper<>(new Tuple3Checker()))
			.distinct().collect();

	String expected = "(true)\n";

	compareResultAsText(result, expected);
}
 
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: SortPartitionITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSortPartitionByTwoFieldExpressions() throws Exception {
	/*
	 * Test sort partition on two field expressions
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(2);

	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env);
	List<Tuple1<Boolean>> result = ds
			.map(new IdMapper<Tuple5<Integer, Long, Integer, String, Long>>()).setParallelism(2) // parallelize input
			.sortPartition("f4", Order.ASCENDING)
			.sortPartition("f2", Order.DESCENDING)
			.mapPartition(new OrderCheckMapper<>(new Tuple5Checker()))
			.distinct().collect();

	String expected = "(true)\n";

	compareResultAsText(result, expected);
}
 
Example #3
Source Project: Flink-CEPplus   Author: ljygz   File: GroupReduceWithCombineProperties.java    License: Apache License 2.0 6 votes vote down vote up
public GroupReduceWithCombineProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
	super(groupKeys);
	
	// if we have an additional ordering, construct the ordering to have primarily the grouping fields
	if (additionalOrderKeys != null) {
		this.ordering = new Ordering();
		for (Integer key : this.keyList) {
			this.ordering.appendOrdering(key, null, Order.ANY);
		}
	
		// and next the additional order fields
		for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
			Integer field = additionalOrderKeys.getFieldNumber(i);
			Order order = additionalOrderKeys.getOrder(i);
			this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
		}
	} else {
		this.ordering = null;
	}
	
	this.customPartitioner = customPartitioner;
}
 
Example #4
Source Project: Flink-CEPplus   Author: ljygz   File: SortPartitionITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSortPartitionPojoByNestedFieldExpression() throws Exception {
	/*
	 * Test sort partition on field expression
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	DataSet<POJO> ds = CollectionDataSets.getMixedPojoDataSet(env);
	List<Tuple1<Boolean>> result = ds
			.map(new IdMapper<POJO>()).setParallelism(1) // parallelize input
			.sortPartition("nestedTupleWithCustom.f1.myString", Order.ASCENDING)
			.sortPartition("number", Order.DESCENDING)
			.mapPartition(new OrderCheckMapper<>(new PojoChecker()))
			.distinct().collect();

	String expected = "(true)\n";

	compareResultAsText(result, expected);
}
 
Example #5
Source Project: flink   Author: flink-tpc-ds   File: DataSinkTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTupleTwoOrderIdx() {

	final ExecutionEnvironment env = ExecutionEnvironment
			.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env
			.fromCollection(emptyTupleData, tupleTypeInfo);

	// should work
	try {
		tupleDs.writeAsText("/tmp/willNotHappen")
			.sortLocalOutput(0, Order.ASCENDING)
			.sortLocalOutput(3, Order.DESCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 
Example #6
Source Project: Flink-CEPplus   Author: ljygz   File: SortPartitionITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSortPartitionWithKeySelector2() throws Exception {
	/*
	 * Test sort partition on an extracted key
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	List<Tuple1<Boolean>> result = ds
		.map(new IdMapper<Tuple3<Integer, Long, String>>()).setParallelism(4) // parallelize input
		.sortPartition(new KeySelector<Tuple3<Integer, Long, String>, Tuple2<Integer, Long>>() {
			@Override
			public Tuple2<Integer, Long> getKey(Tuple3<Integer, Long, String> value) throws Exception {
				return new Tuple2<>(value.f0, value.f1);
			}
		}, Order.DESCENDING)
		.mapPartition(new OrderCheckMapper<>(new Tuple3Checker()))
		.distinct().collect();

	String expected = "(true)\n";

	compareResultAsText(result, expected);
}
 
Example #7
Source Project: flink   Author: apache   File: GroupReduceITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStringBasedDefinitionOnGroupSortForTwoGroupingKeysWithPojos() throws Exception {
	/*
	 * Test string-based definition on group sort, for two grouping keys with Pojos
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	DataSet<PojoContainingTupleAndWritable> ds = CollectionDataSets.getGroupSortedPojoContainingTupleAndWritable(env);
	// f0.f0 is first integer
	DataSet<String> reduceDs = ds.groupBy("hadoopFan").sortGroup("theTuple.f0", Order.DESCENDING).sortGroup("theTuple.f1", Order.DESCENDING)
			.reduceGroup(new GroupReducer5());
	List<String> result = reduceDs.collect();

	String expected = "1---(10,100)-\n"
			+
			"2---(30,600)-(30,400)-(30,200)-(20,201)-(20,200)-\n";

	compareResultAsText(result, expected);
}
 
Example #8
Source Project: flink   Author: apache   File: GroupReduceITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testIntBasedDefinitionOnGroupSortForFullNestedTuple() throws Exception {
	/*
	 * Test int-based definition on group sort, for (full) nested Tuple
	 */
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	DataSet<Tuple2<Tuple2<Integer, Integer>, String>> ds = CollectionDataSets.getGroupSortedNestedTupleDataSet(env);
	DataSet<String> reduceDs = ds.groupBy("f1").sortGroup(0, Order.DESCENDING).reduceGroup(new NestedTupleReducer());
	List<String> result = reduceDs.collect();

	String expected = "a--(2,1)-(1,3)-(1,2)-\n" +
			"b--(2,2)-\n" +
			"c--(4,9)-(3,6)-(3,3)-\n";

	compareResultAsText(result, expected);
}
 
Example #9
Source Project: flink   Author: apache   File: DataSinkTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTupleTwoOrderExp() {

	final ExecutionEnvironment env = ExecutionEnvironment
			.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env
			.fromCollection(emptyTupleData, tupleTypeInfo);

	// should work
	try {
		tupleDs.writeAsText("/tmp/willNotHappen")
			.sortLocalOutput("f1", Order.ASCENDING)
			.sortLocalOutput("f4", Order.DESCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 
Example #10
Source Project: Flink-CEPplus   Author: ljygz   File: GroupReduceITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTupleKeySelectorSortCombineOnTuple() throws Exception {
	/*
	 * check correctness of sorted groupReduceon with Tuple2 keyselector sorting
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env);
	DataSet<Tuple5<Integer, Long, Integer, String, Long>> reduceDs = ds
			.groupBy(new IntFieldExtractor<Tuple5<Integer, Long, Integer, String, Long>>(0))
			.sortGroup(new FiveToTwoTupleExtractor(), Order.DESCENDING)
			.reduceGroup(new Tuple5SortedGroupReduce());

	List<Tuple5<Integer, Long, Integer, String, Long>> result = reduceDs.collect();

	String expected = "1,1,0,Hallo,1\n"
			+
			"2,5,0,Hallo Welt-Hallo Welt wie,1\n" +
			"3,15,0,BCD-ABC-Hallo Welt wie gehts?,2\n" +
			"4,34,0,FGH-CDE-EFG-DEF,1\n" +
			"5,65,0,IJK-HIJ-KLM-JKL-GHI,1\n";

	compareResultAsTuples(result, expected);
}
 
Example #11
Source Project: flink   Author: flink-tpc-ds   File: SortPartitionITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSortPartitionParallelismChange() throws Exception {
	/*
	 * Test sort partition with parallelism change
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	List<Tuple1<Boolean>> result = ds
			.sortPartition(1, Order.DESCENDING).setParallelism(3) // change parallelism
			.mapPartition(new OrderCheckMapper<>(new Tuple3Checker()))
			.distinct().collect();

	String expected = "(true)\n";

	compareResultAsText(result, expected);
}
 
Example #12
Source Project: Flink-CEPplus   Author: ljygz   File: DataSinkITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStringSortingParallelism1() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<String> ds = CollectionDataSets.getStringDataSet(env);
	ds.writeAsText(resultPath).sortLocalOutput("*", Order.ASCENDING).setParallelism(1);

	env.execute();

	String expected = "Hello\n" +
			"Hello world\n" +
			"Hello world, how are you?\n" +
			"Hi\n" +
			"I am fine.\n" +
			"LOL\n" +
			"Luke Skywalker\n" +
			"Random comment\n";

	compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath);
}
 
Example #13
Source Project: Flink-CEPplus   Author: ljygz   File: DataSinkITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPojoSortingSingleParallelism1() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CollectionDataSets.POJO> ds = CollectionDataSets.getMixedPojoDataSet(env);
	ds.writeAsText(resultPath).sortLocalOutput("number", Order.ASCENDING).setParallelism(1);

	env.execute();

	String expected = "1 First (10,100,1000,One) 10100\n" +
			"2 First_ (10,105,1000,One) 10200\n" +
			"3 First (11,102,3000,One) 10200\n" +
			"4 First_ (11,106,1000,One) 10300\n" +
			"5 First (11,102,2000,One) 10100\n" +
			"6 Second_ (20,200,2000,Two) 10100\n" +
			"7 Third (31,301,2000,Three) 10200\n" +
			"8 Third_ (30,300,1000,Three) 10100\n";

	compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath);
}
 
Example #14
Source Project: flink   Author: apache   File: GroupCombineITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testIdentityWithGroupByAndSort() throws Exception {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);

	DataSet<Tuple3<Integer, Long, String>> reduceDs = ds
			.groupBy(1)
			.sortGroup(1, Order.DESCENDING)
			// reduce partially
			.combineGroup(new IdentityFunction())
			.groupBy(1)
			.sortGroup(1, Order.DESCENDING)
			// fully reduce
			.reduceGroup(new IdentityFunction());

	List<Tuple3<Integer, Long, String>> result = reduceDs.collect();

	compareResultAsTuples(result, identityResult);
}
 
Example #15
Source Project: Flink-CEPplus   Author: ljygz   File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSortingErased() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNull(gFields);
	assertNull(order);
	assertNull(filtered.getUniqueFields());
}
 
Example #16
Source Project: Flink-CEPplus   Author: ljygz   File: GroupReduceProperties.java    License: Apache License 2.0 6 votes vote down vote up
public GroupReduceProperties(FieldSet groupKeys, Ordering additionalOrderKeys, Partitioner<?> customPartitioner) {
	super(groupKeys);
	
	// if we have an additional ordering, construct the ordering to have primarily the grouping fields
	if (additionalOrderKeys != null) {
		this.ordering = new Ordering();
		for (Integer key : this.keyList) {
			this.ordering.appendOrdering(key, null, Order.ANY);
		}
	
		// and next the additional order fields
		for (int i = 0; i < additionalOrderKeys.getNumberOfFields(); i++) {
			Integer field = additionalOrderKeys.getFieldNumber(i);
			Order order = additionalOrderKeys.getOrder(i);
			this.ordering.appendOrdering(field, additionalOrderKeys.getType(i), order);
		}
	}
	else {
		this.ordering = null;
	}
	
	this.customPartitioner = customPartitioner;
}
 
Example #17
Source Project: flink   Author: flink-tpc-ds   File: DataSinkITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTupleSortingNestedParallelism1_2() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple3<Tuple2<Integer, Integer>, String, Integer>> ds =
			CollectionDataSets.getGroupSortedNestedTupleDataSet2(env);
	ds.writeAsText(resultPath)
		.sortLocalOutput(1, Order.ASCENDING)
		.sortLocalOutput(2, Order.DESCENDING)
		.setParallelism(1);

	env.execute();

	String expected =
			"((2,1),a,3)\n" +
			"((1,3),a,2)\n" +
			"((1,2),a,1)\n" +
			"((2,2),b,4)\n" +
			"((4,9),c,7)\n" +
			"((3,6),c,6)\n" +
			"((3,3),c,5)\n";

	compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath);
}
 
Example #18
Source Project: Flink-CEPplus   Author: ljygz   File: GroupingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGroupSortByKeyExpression3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs =
			env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// should work
	try {
		tupleDs.groupBy("f0")
				.sortGroup("f2.myString", Order.ASCENDING)
				.sortGroup("f1", Order.DESCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 
Example #19
Source Project: flink   Author: apache   File: SortPartitionITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSortPartitionPojoByNestedFieldExpression() throws Exception {
	/*
	 * Test sort partition on field expression
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(3);

	DataSet<POJO> ds = CollectionDataSets.getMixedPojoDataSet(env);
	List<Tuple1<Boolean>> result = ds
			.map(new IdMapper<POJO>()).setParallelism(1) // parallelize input
			.sortPartition("nestedTupleWithCustom.f1.myString", Order.ASCENDING)
			.sortPartition("number", Order.DESCENDING)
			.mapPartition(new OrderCheckMapper<>(new PojoChecker()))
			.distinct().collect();

	String expected = "(true)\n";

	compareResultAsText(result, expected);
}
 
Example #20
Source Project: flink   Author: apache   File: GroupingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = InvalidProgramException.class)
public void testGroupSortKeyFields3() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Long> longDs = env.fromCollection(emptyLongData, BasicTypeInfo.LONG_TYPE_INFO);

	// should not work: sorted groups on groupings by key selectors
	longDs.groupBy(new KeySelector<Long, Long>() {
		private static final long serialVersionUID = 1L;

		@Override
		public Long getKey(Long value) {
			return value;
		}

	}).sortGroup(0, Order.ASCENDING);

}
 
Example #21
Source Project: flink   Author: flink-tpc-ds   File: DataSinkITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPojoSortingSingleParallelism1() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<CollectionDataSets.POJO> ds = CollectionDataSets.getMixedPojoDataSet(env);
	ds.writeAsText(resultPath).sortLocalOutput("number", Order.ASCENDING).setParallelism(1);

	env.execute();

	String expected = "1 First (10,100,1000,One) 10100\n" +
			"2 First_ (10,105,1000,One) 10200\n" +
			"3 First (11,102,3000,One) 10200\n" +
			"4 First_ (11,106,1000,One) 10300\n" +
			"5 First (11,102,2000,One) 10100\n" +
			"6 Second_ (20,200,2000,Two) 10100\n" +
			"7 Third (31,301,2000,Three) 10200\n" +
			"8 Third_ (30,300,1000,Three) 10100\n";

	compareResultsByLinesInMemoryWithStrictOrder(expected, resultPath);
}
 
Example #22
Source Project: Flink-CEPplus   Author: ljygz   File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSortingPreserved4() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"2->7;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(1, gFields.size());
	assertTrue(gFields.contains(7));
	assertNotNull(order);
	assertEquals(1, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(IntValue.class, order.getType(0));
	assertNull(filtered.getUniqueFields());
}
 
Example #23
Source Project: flink   Author: flink-tpc-ds   File: PartitionOperator.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner(
	SelectorFunctionKeys<T, ?> rawKeys,
	PartitionMethod pMethod,
	String name,
	Operator<T> input,
	int partitionDop,
	Partitioner<?> customPartitioner,
	Order[] orders) {
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput =
		new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[]{0}, name);
	keyedPartitionedInput.setInput(keyedInput);
	keyedPartitionedInput.setCustomPartitioner(customPartitioner);
	keyedPartitionedInput.setParallelism(partitionDop);
	keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING));

	return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys);
}
 
Example #24
Source Project: Flink-CEPplus   Author: ljygz   File: DataSinkTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTupleTwoOrderIdx() {

	final ExecutionEnvironment env = ExecutionEnvironment
			.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env
			.fromCollection(emptyTupleData, tupleTypeInfo);

	// should work
	try {
		tupleDs.writeAsText("/tmp/willNotHappen")
			.sortLocalOutput(0, Order.ASCENDING)
			.sortLocalOutput(3, Order.DESCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 
Example #25
Source Project: flink   Author: apache   File: PropertyDataSourceTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void checkSinglePartitionedOrderedSource6() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);

	data.getSplitDataProperties()
			.splitsPartitionedBy("f1.intField")
			.splitsOrderedBy("f1", new Order[]{Order.DESCENDING});

	data.output(new DiscardingOutputFormat<Tuple3<Long, SomePojo, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
	Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(1,2,3)));
	Assert.assertTrue(lprops.getOrdering() == null);

}
 
Example #26
Source Project: Flink-CEPplus   Author: ljygz   File: GroupCombineITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
// check if all API methods are callable
public void testAPI() throws Exception {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<Tuple1<String>> ds = CollectionDataSets.getStringDataSet(env).map(new MapFunction<String, Tuple1<String>>() {
		@Override
		public Tuple1<String> map(String value) throws Exception {
			return new Tuple1<>(value);
		}
	});

	// all methods on DataSet
	ds.combineGroup(new GroupCombineFunctionExample())
	.output(new DiscardingOutputFormat<Tuple1<String>>());

	// all methods on UnsortedGrouping
	ds.groupBy(0).combineGroup(new GroupCombineFunctionExample())
	.output(new DiscardingOutputFormat<Tuple1<String>>());

	// all methods on SortedGrouping
	ds.groupBy(0).sortGroup(0, Order.ASCENDING).combineGroup(new GroupCombineFunctionExample())
	.output(new DiscardingOutputFormat<Tuple1<String>>());

	env.execute();
}
 
Example #27
Source Project: flink   Author: flink-tpc-ds   File: GroupingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGroupSortByKeyExpression2() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs =
			env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);

	// should work
	try {
		tupleDs.groupBy("f0").sortGroup("f2.myString", Order.ASCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 
Example #28
Source Project: flink   Author: flink-tpc-ds   File: GroupReduceITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTupleKeySelectorGroupSort() throws Exception {
	/*
	 * check correctness of sorted groupReduce on tuples with keyselector sorting
	 */

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);

	DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env);
	DataSet<Tuple3<Integer, Long, String>> reduceDs = ds
			.groupBy(new LongFieldExtractor<Tuple3<Integer, Long, String>>(1))
			.sortGroup(new StringFieldExtractor<Tuple3<Integer, Long, String>>(2), Order.DESCENDING)
			.reduceGroup(new Tuple3SortedGroupReduce());

	List<Tuple3<Integer, Long, String>> result = reduceDs.collect();

	String expected = "1,1,Hi\n"
			+
			"5,2,Hello world-Hello\n" +
			"15,3,Luke Skywalker-I am fine.-Hello world, how are you?\n" +
			"34,4,Comment#4-Comment#3-Comment#2-Comment#1\n" +
			"65,5,Comment#9-Comment#8-Comment#7-Comment#6-Comment#5\n" +
			"111,6,Comment#15-Comment#14-Comment#13-Comment#12-Comment#11-Comment#10\n";

	compareResultAsTuples(result, expected);
}
 
Example #29
Source Project: Flink-CEPplus   Author: ljygz   File: GroupingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testChainedGroupSortKeyFields() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);

	// should work
	try {
		tupleDs.groupBy(0).sortGroup(0, Order.ASCENDING).sortGroup(2, Order.DESCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}
 
Example #30
Source Project: flink   Author: flink-tpc-ds   File: GroupingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGroupSortKeyFields1() {

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);

	// should work
	try {
		tupleDs.groupBy(0).sortGroup(0, Order.ASCENDING);
	} catch (Exception e) {
		Assert.fail();
	}
}