Java Code Examples for org.apache.flink.api.common.operators.util.FieldList

The following examples show how to use org.apache.flink.api.common.operators.util.FieldList. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) {

		TypeComparator<T> comparator;
		if (typeInfo instanceof CompositeType) {
			comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
		}
		else if (typeInfo instanceof AtomicType) {
			// handle grouping of atomic types
			comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
		}
		else {
			throw new RuntimeException("Unrecognized type: " + typeInfo);
		}

		return new RuntimeComparatorFactory<>(comparator);
	}
 
Example 2
Source Project: flink   Source File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGroupingPreserved1() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;3"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNotNull(filtered.getGroupedFields());
	assertEquals(3, filtered.getGroupedFields().size());
	assertTrue(filtered.getGroupedFields().contains(0));
	assertTrue(filtered.getGroupedFields().contains(2));
	assertTrue(filtered.getGroupedFields().contains(3));
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}
 
Example 3
Source Project: flink   Source File: WorksetIterationNode.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new node with a single input for the optimizer plan.
 * 
 * @param iteration The iteration operator that the node represents.
 */
public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) {
	super(iteration);
	
	final int[] ssKeys = iteration.getSolutionSetKeyFields();
	if (ssKeys == null || ssKeys.length == 0) {
		throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
	}
	this.solutionSetKeyFields = new FieldList(ssKeys);
	this.partitionedProperties = new GlobalProperties();
	this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
	
	int weight = iteration.getMaximumNumberOfIterations() > 0 ? 
		iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;
		
	if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) {
		weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT;
	}
	this.costWeight = weight; 
	
	this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields));
}
 
Example 4
Source Project: flink   Source File: GlobalPropertiesFilteringTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCustomPartitioningPreserved1() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
	gprops.setCustomPartitioned(new FieldList(0, 4), myP);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(2, pFields.size());
	assertTrue(pFields.contains(0));
	assertTrue(pFields.contains(4));
	assertEquals(myP, result.getCustomPartitioner());
}
 
Example 5
Source Project: flink   Source File: GlobalPropertiesFilteringTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testHashPartitioningPreserved1() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1, 4));

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.HASH_PARTITIONED, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(3, pFields.size());
	assertTrue(pFields.contains(0));
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(4));
}
 
Example 6
Source Project: flink   Source File: JavaApiPostPass.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
	
	TypeComparator<T> comparator;
	if (typeInfo instanceof CompositeType) {
		comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
	}
	else if (typeInfo instanceof AtomicType) {
		// handle grouping of atomic types
		comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
	}
	else {
		throw new RuntimeException("Unrecognized type: " + typeInfo);
	}

	return new RuntimeComparatorFactory<T>(comparator);
}
 
Example 7
@Test
public void testCustomPartitioningErased() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
	gprops.setCustomPartitioned(new FieldList(0, 4), myP);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningFields());
	assertNull(result.getCustomPartitioner());
}
 
Example 8
Source Project: flink   Source File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSortingErased() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNull(gFields);
	assertNull(order);
	assertNull(filtered.getUniqueFields());
}
 
Example 9
Source Project: flink   Source File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllErased1() {

	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 1, 2));
	lProps = lProps.addUniqueFields(new FieldSet(3,4));
	lProps = lProps.addUniqueFields(new FieldSet(5,6));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}
 
Example 10
Source Project: flink   Source File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSortingPreserved4() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"2->7;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(1, gFields.size());
	assertTrue(gFields.contains(7));
	assertNotNull(order);
	assertEquals(1, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(IntValue.class, order.getType(0));
	assertNull(filtered.getUniqueFields());
}
 
Example 11
Source Project: flink   Source File: GlobalPropertiesFilteringTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllErased1() {

	SingleInputSemanticProperties semProps = new SingleInputSemanticProperties();

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1));
	gprops.addUniqueFieldCombination(new FieldSet(3, 4));
	gprops.addUniqueFieldCombination(new FieldSet(5, 6));

	GlobalProperties result = gprops.filterBySemanticProperties(semProps, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningFields());
	assertNull(result.getPartitioningOrdering());
	assertNull(result.getUniqueFieldCombination());
}
 
Example 12
@Test
public void testCustomPartitioningPreserved2() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
	gprops.setCustomPartitioned(new FieldList(0, 4), myP);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(2, pFields.size());
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(3));
	assertEquals(myP, result.getCustomPartitioner());
}
 
Example 13
@Test
public void testAllErased2() {

	SingleInputSemanticProperties semProps = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(semProps, new String[]{"2"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1));
	gprops.addUniqueFieldCombination(new FieldSet(3, 4));
	gprops.addUniqueFieldCombination(new FieldSet(5, 6));

	GlobalProperties result = gprops.filterBySemanticProperties(semProps, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningFields());
	assertNull(result.getPartitioningOrdering());
	assertNull(result.getUniqueFieldCombination());
}
 
Example 14
Source Project: Flink-CEPplus   Source File: FieldListTest.java    License: Apache License 2.0 5 votes vote down vote up
private static void check(FieldList set, int... elements) {
	if (elements == null) {
		assertEquals(0, set.size());
		return;
	}
	
	assertEquals(elements.length, set.size());
	
	// test contains
	for (int i : elements) {
		set.contains(i);
	}
	
	// test to array
	{
		int[] arr = set.toArray();
		assertTrue(Arrays.equals(arr, elements));
	}
	
	{
		int[] fromIter = new int[set.size()];
		Iterator<Integer> iter = set.iterator();
		
		for (int i = 0; i < fromIter.length; i++) {
			fromIter[i] = iter.next();
		}
		assertFalse(iter.hasNext());
		assertTrue(Arrays.equals(fromIter, elements));
	}
}
 
Example 15
Source Project: flink   Source File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSortingPreserved1() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(3, gFields.size());
	assertTrue(gFields.contains(0));
	assertTrue(gFields.contains(2));
	assertTrue(gFields.contains(5));
	assertNotNull(order);
	assertEquals(3, order.getNumberOfFields());
	assertEquals(2, order.getFieldNumber(0).intValue());
	assertEquals(0, order.getFieldNumber(1).intValue());
	assertEquals(5, order.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(Order.DESCENDING, order.getOrder(1));
	assertEquals(Order.DESCENDING, order.getOrder(2));
	assertEquals(IntValue.class, order.getType(0));
	assertEquals(StringValue.class, order.getType(1));
	assertEquals(LongValue.class, order.getType(2));
	assertNull(filtered.getUniqueFields());
}
 
Example 16
Source Project: flink   Source File: SingleInputPlanNode.java    License: Apache License 2.0 5 votes vote down vote up
public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input, 
		DriverStrategy driverStrategy, FieldList driverKeyFields, boolean[] driverSortOrders)
{
	super(template, nodeName, driverStrategy);
	this.input = input;
	
	this.comparators = new TypeComparatorFactory<?>[driverStrategy.getNumRequiredComparators()];
	this.driverKeys = new FieldList[driverStrategy.getNumRequiredComparators()];
	this.driverSortOrders = new boolean[driverStrategy.getNumRequiredComparators()][];
	
	if(driverStrategy.getNumRequiredComparators() > 0) {
		this.driverKeys[0] = driverKeyFields;
		this.driverSortOrders[0] = driverSortOrders;
	}
	
	if (this.input.getShipStrategy() == ShipStrategyType.BROADCAST) {
		this.input.setReplicationFactor(getParallelism());
	}
	
	final PlanNode predNode = input.getSource();
	
	if (predNode.branchPlan != null && !predNode.branchPlan.isEmpty()) {
		
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		this.branchPlan.putAll(predNode.branchPlan);
	}
}
 
Example 17
Source Project: flink   Source File: JavaApiPostPass.java    License: Apache License 2.0 5 votes vote down vote up
private static final boolean[] getSortOrders(FieldList keys, boolean[] orders) {
	if (orders == null) {
		orders = new boolean[keys.size()];
		Arrays.fill(orders, true);
	}
	return orders;
}
 
Example 18
Source Project: flink   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
public static Ordering createOrdering(FieldList fields, boolean[] directions) {
	final Ordering o = new Ordering();
	for (int i = 0; i < fields.size(); i++) {
		o.appendOrdering(fields.get(i), null, directions == null || directions[i] ? Order.ASCENDING : Order.DESCENDING);
	}
	return o;
}
 
Example 19
@Test
public void testRangePartitioningPreserved2() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1->3; 2->0; 5->1"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	o.appendOrdering(2, StringValue.class, Order.ASCENDING);
	GlobalProperties gprops = new GlobalProperties();
	gprops.setRangePartitioned(o);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.RANGE_PARTITIONED, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(3, pFields.size());
	assertEquals(3, pFields.get(0).intValue());
	assertEquals(1, pFields.get(1).intValue());
	assertEquals(0, pFields.get(2).intValue());
	Ordering pOrder = result.getPartitioningOrdering();
	assertEquals(3, pOrder.getNumberOfFields());
	assertEquals(3, pOrder.getFieldNumber(0).intValue());
	assertEquals(1, pOrder.getFieldNumber(1).intValue());
	assertEquals(0, pOrder.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, pOrder.getOrder(0));
	assertEquals(Order.DESCENDING, pOrder.getOrder(1));
	assertEquals(Order.ASCENDING, pOrder.getOrder(2));
	assertEquals(IntValue.class, pOrder.getType(0));
	assertEquals(LongValue.class, pOrder.getType(1));
	assertEquals(StringValue.class, pOrder.getType(2));
}
 
Example 20
Source Project: flink   Source File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSortingPreserved1() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(3, gFields.size());
	assertTrue(gFields.contains(0));
	assertTrue(gFields.contains(2));
	assertTrue(gFields.contains(5));
	assertNotNull(order);
	assertEquals(3, order.getNumberOfFields());
	assertEquals(2, order.getFieldNumber(0).intValue());
	assertEquals(0, order.getFieldNumber(1).intValue());
	assertEquals(5, order.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(Order.DESCENDING, order.getOrder(1));
	assertEquals(Order.DESCENDING, order.getOrder(2));
	assertEquals(IntValue.class, order.getType(0));
	assertEquals(StringValue.class, order.getType(1));
	assertEquals(LongValue.class, order.getType(2));
	assertNull(filtered.getUniqueFields());
}
 
Example 21
Source Project: flink   Source File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSortingPreserved2() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->3;2->7;5->1"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(3, gFields.size());
	assertTrue(gFields.contains(3));
	assertTrue(gFields.contains(7));
	assertTrue(gFields.contains(1));
	assertNotNull(order);
	assertEquals(3, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(3, order.getFieldNumber(1).intValue());
	assertEquals(1, order.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(Order.DESCENDING, order.getOrder(1));
	assertEquals(Order.DESCENDING, order.getOrder(2));
	assertEquals(IntValue.class, order.getType(0));
	assertEquals(StringValue.class, order.getType(1));
	assertEquals(LongValue.class, order.getType(2));
	assertNull(filtered.getUniqueFields());
}
 
Example 22
Source Project: flink   Source File: GlobalPropertiesFilteringTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHashPartitioningErased() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1, 4));

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningFields());
}
 
Example 23
Source Project: flink   Source File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAllErased2() {

	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"5"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 1, 2));
	lProps = lProps.addUniqueFields(new FieldSet(3,4));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}
 
Example 24
Source Project: flink   Source File: LocalPropertiesFilteringTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGroupingErased() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->4;2->0"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}
 
Example 25
@Test
public void testSortingPreserved3() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(2, gFields.size());
	assertTrue(gFields.contains(0));
	assertTrue(gFields.contains(2));
	assertNotNull(order);
	assertEquals(2, order.getNumberOfFields());
	assertEquals(2, order.getFieldNumber(0).intValue());
	assertEquals(0, order.getFieldNumber(1).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(Order.DESCENDING, order.getOrder(1));
	assertEquals(IntValue.class, order.getType(0));
	assertEquals(StringValue.class, order.getType(1));
	assertNull(filtered.getUniqueFields());
}
 
Example 26
Source Project: flink   Source File: PartitionPushdownTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPartitioningNotPushedDown() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		
		@SuppressWarnings("unchecked")
		DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
		
		input
			.groupBy(0, 1).sum(2)
			.groupBy(0).sum(1)
			.output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		
		SingleInputPlanNode agg2Reducer = (SingleInputPlanNode) sink.getInput().getSource();
		SingleInputPlanNode agg2Combiner = (SingleInputPlanNode) agg2Reducer.getInput().getSource();
		SingleInputPlanNode agg1Reducer = (SingleInputPlanNode) agg2Combiner.getInput().getSource();
		
		assertEquals(ShipStrategyType.PARTITION_HASH, agg2Reducer.getInput().getShipStrategy());
		assertEquals(new FieldList(0), agg2Reducer.getInput().getShipStrategyKeys());
		
		assertEquals(ShipStrategyType.FORWARD, agg2Combiner.getInput().getShipStrategy());
		
		assertEquals(ShipStrategyType.PARTITION_HASH, agg1Reducer.getInput().getShipStrategy());
		assertEquals(new FieldList(0, 1), agg1Reducer.getInput().getShipStrategyKeys());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 27
Source Project: flink   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
private static boolean[] getSortOrders(FieldList keys, boolean[] orders) {
	if (orders == null) {
		orders = new boolean[keys.size()];
		Arrays.fill(orders, true);
	}
	return orders;
}
 
Example 28
Source Project: Flink-CEPplus   Source File: RequestedGlobalProperties.java    License: Apache License 2.0 5 votes vote down vote up
private boolean checkCompatiblePartitioningFields(GlobalProperties props) {
	if(this.partitioningFields instanceof FieldList) {
		// partitioningFields as FieldList requires strict checking!
		return props.isExactlyPartitionedOnFields((FieldList)this.partitioningFields);
	} else {
		return props.isPartitionedOnFields(this.partitioningFields);
	}
}
 
Example 29
Source Project: flink   Source File: OperatorDescriptorDual.java    License: Apache License 2.0 5 votes vote down vote up
protected boolean checkEquivalentFieldPositionsInKeyFields(FieldList fields1, FieldList fields2) {

		// check number of produced partitioning fields
		if(fields1.size() != fields2.size()) {
			return false;
		} else {
			return checkEquivalentFieldPositionsInKeyFields(fields1, fields2, fields1.size());
		}
	}
 
Example 30
Source Project: Flink-CEPplus   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
public static FieldList createOrderedFromSet(FieldSet set) {
	if (set instanceof FieldList) {
		return (FieldList) set;
	} else {
		final int[] cols = set.toArray();
		Arrays.sort(cols);
		return new FieldList(cols);
	}
}