org.apache.flink.api.common.operators.util.FieldList Java Examples

The following examples show how to use org.apache.flink.api.common.operators.util.FieldList. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LocalPropertiesFilteringTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSortingErased() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNull(gFields);
	assertNull(order);
	assertNull(filtered.getUniqueFields());
}
 
Example #2
Source File: GlobalPropertiesFilteringTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testCustomPartitioningPreserved1() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
	gprops.setCustomPartitioned(new FieldList(0, 4), myP);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(2, pFields.size());
	assertTrue(pFields.contains(0));
	assertTrue(pFields.contains(4));
	assertEquals(myP, result.getCustomPartitioner());
}
 
Example #3
Source File: LocalPropertiesFilteringTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testGroupingPreserved1() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;3"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNotNull(filtered.getGroupedFields());
	assertEquals(3, filtered.getGroupedFields().size());
	assertTrue(filtered.getGroupedFields().contains(0));
	assertTrue(filtered.getGroupedFields().contains(2));
	assertTrue(filtered.getGroupedFields().contains(3));
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}
 
Example #4
Source File: GlobalPropertiesFilteringTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testCustomPartitioningErased() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
	gprops.setCustomPartitioned(new FieldList(0, 4), myP);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningFields());
	assertNull(result.getCustomPartitioner());
}
 
Example #5
Source File: WorksetIterationNode.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new node with a single input for the optimizer plan.
 * 
 * @param iteration The iteration operator that the node represents.
 */
public WorksetIterationNode(DeltaIterationBase<?, ?> iteration) {
	super(iteration);
	
	final int[] ssKeys = iteration.getSolutionSetKeyFields();
	if (ssKeys == null || ssKeys.length == 0) {
		throw new CompilerException("Invalid WorksetIteration: No key fields defined for the solution set.");
	}
	this.solutionSetKeyFields = new FieldList(ssKeys);
	this.partitionedProperties = new GlobalProperties();
	this.partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
	
	int weight = iteration.getMaximumNumberOfIterations() > 0 ? 
		iteration.getMaximumNumberOfIterations() : DEFAULT_COST_WEIGHT;
		
	if (weight > OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) {
		weight = OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT;
	}
	this.costWeight = weight; 
	
	this.dataProperties = Collections.<OperatorDescriptorDual>singletonList(new WorksetOpDescriptor(this.solutionSetKeyFields));
}
 
Example #6
Source File: LocalPropertiesFilteringTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testAllErased1() {

	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 1, 2));
	lProps = lProps.addUniqueFields(new FieldSet(3,4));
	lProps = lProps.addUniqueFields(new FieldSet(5,6));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}
 
Example #7
Source File: GlobalPropertiesFilteringTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testHashPartitioningPreserved1() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1, 4));

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.HASH_PARTITIONED, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(3, pFields.size());
	assertTrue(pFields.contains(0));
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(4));
}
 
Example #8
Source File: GlobalPropertiesFilteringTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testAllErased2() {

	SingleInputSemanticProperties semProps = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(semProps, new String[]{"2"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1));
	gprops.addUniqueFieldCombination(new FieldSet(3, 4));
	gprops.addUniqueFieldCombination(new FieldSet(5, 6));

	GlobalProperties result = gprops.filterBySemanticProperties(semProps, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningFields());
	assertNull(result.getPartitioningOrdering());
	assertNull(result.getUniqueFieldCombination());
}
 
Example #9
Source File: Utils.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) {

		TypeComparator<T> comparator;
		if (typeInfo instanceof CompositeType) {
			comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
		}
		else if (typeInfo instanceof AtomicType) {
			// handle grouping of atomic types
			comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
		}
		else {
			throw new RuntimeException("Unrecognized type: " + typeInfo);
		}

		return new RuntimeComparatorFactory<>(comparator);
	}
 
Example #10
Source File: GlobalPropertiesFilteringTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testCustomPartitioningPreserved2() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
	gprops.setCustomPartitioned(new FieldList(0, 4), myP);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(2, pFields.size());
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(3));
	assertEquals(myP, result.getCustomPartitioner());
}
 
Example #11
Source File: GlobalPropertiesFilteringTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testAllErased1() {

	SingleInputSemanticProperties semProps = new SingleInputSemanticProperties();

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1));
	gprops.addUniqueFieldCombination(new FieldSet(3, 4));
	gprops.addUniqueFieldCombination(new FieldSet(5, 6));

	GlobalProperties result = gprops.filterBySemanticProperties(semProps, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningFields());
	assertNull(result.getPartitioningOrdering());
	assertNull(result.getUniqueFieldCombination());
}
 
Example #12
Source File: JavaApiPostPass.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
	
	TypeComparator<T> comparator;
	if (typeInfo instanceof CompositeType) {
		comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
	}
	else if (typeInfo instanceof AtomicType) {
		// handle grouping of atomic types
		comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
	}
	else {
		throw new RuntimeException("Unrecognized type: " + typeInfo);
	}

	return new RuntimeComparatorFactory<T>(comparator);
}
 
Example #13
Source File: LocalPropertiesFilteringTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSortingPreserved4() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"2->7;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(1, gFields.size());
	assertTrue(gFields.contains(7));
	assertNotNull(order);
	assertEquals(1, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(IntValue.class, order.getType(0));
	assertNull(filtered.getUniqueFields());
}
 
Example #14
Source File: LocalPropertiesFilteringTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSortingPreserved1() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(3, gFields.size());
	assertTrue(gFields.contains(0));
	assertTrue(gFields.contains(2));
	assertTrue(gFields.contains(5));
	assertNotNull(order);
	assertEquals(3, order.getNumberOfFields());
	assertEquals(2, order.getFieldNumber(0).intValue());
	assertEquals(0, order.getFieldNumber(1).intValue());
	assertEquals(5, order.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(Order.DESCENDING, order.getOrder(1));
	assertEquals(Order.DESCENDING, order.getOrder(2));
	assertEquals(IntValue.class, order.getType(0));
	assertEquals(StringValue.class, order.getType(1));
	assertEquals(LongValue.class, order.getType(2));
	assertNull(filtered.getUniqueFields());
}
 
Example #15
Source File: LocalPropertiesFilteringTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testAllErased2() {

	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"5"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 1, 2));
	lProps = lProps.addUniqueFields(new FieldSet(3,4));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}
 
Example #16
Source File: LocalPropertiesFilteringTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSortingPreserved1() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(3, gFields.size());
	assertTrue(gFields.contains(0));
	assertTrue(gFields.contains(2));
	assertTrue(gFields.contains(5));
	assertNotNull(order);
	assertEquals(3, order.getNumberOfFields());
	assertEquals(2, order.getFieldNumber(0).intValue());
	assertEquals(0, order.getFieldNumber(1).intValue());
	assertEquals(5, order.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(Order.DESCENDING, order.getOrder(1));
	assertEquals(Order.DESCENDING, order.getOrder(2));
	assertEquals(IntValue.class, order.getType(0));
	assertEquals(StringValue.class, order.getType(1));
	assertEquals(LongValue.class, order.getType(2));
	assertNull(filtered.getUniqueFields());
}
 
Example #17
Source File: JavaApiPostPass.java    From flink with Apache License 2.0 5 votes vote down vote up
private static final boolean[] getSortOrders(FieldList keys, boolean[] orders) {
	if (orders == null) {
		orders = new boolean[keys.size()];
		Arrays.fill(orders, true);
	}
	return orders;
}
 
Example #18
Source File: Channel.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public void setShipStrategy(ShipStrategyType strategy, FieldList keys,
							boolean[] sortDirection, Partitioner<?> partitioner,
							DataExchangeMode dataExchangeMode) {
	this.shipStrategy = strategy;
	this.shipKeys = keys;
	this.shipSortOrder = sortDirection;
	this.partitioner = partitioner;
	this.dataExchangeMode = dataExchangeMode;
	this.globalProps = null;		// reset the global properties
}
 
Example #19
Source File: SingleInputPlanNode.java    From flink with Apache License 2.0 5 votes vote down vote up
public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input, 
		DriverStrategy driverStrategy, FieldList driverKeyFields, boolean[] driverSortOrders)
{
	super(template, nodeName, driverStrategy);
	this.input = input;
	
	this.comparators = new TypeComparatorFactory<?>[driverStrategy.getNumRequiredComparators()];
	this.driverKeys = new FieldList[driverStrategy.getNumRequiredComparators()];
	this.driverSortOrders = new boolean[driverStrategy.getNumRequiredComparators()][];
	
	if(driverStrategy.getNumRequiredComparators() > 0) {
		this.driverKeys[0] = driverKeyFields;
		this.driverSortOrders[0] = driverSortOrders;
	}
	
	if (this.input.getShipStrategy() == ShipStrategyType.BROADCAST) {
		this.input.setReplicationFactor(getParallelism());
	}
	
	final PlanNode predNode = input.getSource();
	
	if (predNode.branchPlan != null && !predNode.branchPlan.isEmpty()) {
		
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		this.branchPlan.putAll(predNode.branchPlan);
	}
}
 
Example #20
Source File: SingleInputPlanNode.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Sets the key field information for the specified driver comparator.
 * 
 * @param keys The key field indexes for the specified driver comparator.
 * @param sortOrder The key sort order for the specified driver comparator.
 * @param id The ID of the driver comparator.
 */
public void setDriverKeyInfo(FieldList keys, boolean[] sortOrder, int id) {
	if(id < 0 || id >= driverKeys.length) {
		throw new CompilerException("Invalid id for driver key information. DriverStrategy requires only "
										+super.getDriverStrategy().getNumRequiredComparators()+" comparators.");
	}
	this.driverKeys[id] = keys;
	this.driverSortOrders[id] = sortOrder;
}
 
Example #21
Source File: FieldListTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private static void check(FieldList set, int... elements) {
	if (elements == null) {
		assertEquals(0, set.size());
		return;
	}
	
	assertEquals(elements.length, set.size());
	
	// test contains
	for (int i : elements) {
		set.contains(i);
	}
	
	// test to array
	{
		int[] arr = set.toArray();
		assertTrue(Arrays.equals(arr, elements));
	}
	
	{
		int[] fromIter = new int[set.size()];
		Iterator<Integer> iter = set.iterator();
		
		for (int i = 0; i < fromIter.length; i++) {
			fromIter[i] = iter.next();
		}
		assertFalse(iter.hasNext());
		assertTrue(Arrays.equals(fromIter, elements));
	}
}
 
Example #22
Source File: Utils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public static FieldList createOrderedFromSet(FieldSet set) {
	if (set instanceof FieldList) {
		return (FieldList) set;
	} else {
		final int[] cols = set.toArray();
		Arrays.sort(cols);
		return new FieldList(cols);
	}
}
 
Example #23
Source File: LocalPropertiesFilteringTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSortingPreserved2() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->3;2->7;5->1"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(3, gFields.size());
	assertTrue(gFields.contains(3));
	assertTrue(gFields.contains(7));
	assertTrue(gFields.contains(1));
	assertNotNull(order);
	assertEquals(3, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(3, order.getFieldNumber(1).intValue());
	assertEquals(1, order.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(Order.DESCENDING, order.getOrder(1));
	assertEquals(Order.DESCENDING, order.getOrder(2));
	assertEquals(IntValue.class, order.getType(0));
	assertEquals(StringValue.class, order.getType(1));
	assertEquals(LongValue.class, order.getType(2));
	assertNull(filtered.getUniqueFields());
}
 
Example #24
Source File: GlobalPropertiesFilteringTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testHashPartitioningErased() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1, 4));

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningFields());
}
 
Example #25
Source File: Utils.java    From flink with Apache License 2.0 5 votes vote down vote up
private static boolean[] getSortOrders(FieldList keys, boolean[] orders) {
	if (orders == null) {
		orders = new boolean[keys.size()];
		Arrays.fill(orders, true);
	}
	return orders;
}
 
Example #26
Source File: OperatorDescriptorDual.java    From flink with Apache License 2.0 5 votes vote down vote up
protected boolean checkEquivalentFieldPositionsInKeyFields(FieldList fields1, FieldList fields2) {

		// check number of produced partitioning fields
		if(fields1.size() != fields2.size()) {
			return false;
		} else {
			return checkEquivalentFieldPositionsInKeyFields(fields1, fields2, fields1.size());
		}
	}
 
Example #27
Source File: LocalPropertiesFilteringTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGroupingErased() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->4;2->0"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}
 
Example #28
Source File: LocalPropertiesFilteringTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testSortingPreserved3() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;2"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(2, gFields.size());
	assertTrue(gFields.contains(0));
	assertTrue(gFields.contains(2));
	assertNotNull(order);
	assertEquals(2, order.getNumberOfFields());
	assertEquals(2, order.getFieldNumber(0).intValue());
	assertEquals(0, order.getFieldNumber(1).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(Order.DESCENDING, order.getOrder(1));
	assertEquals(IntValue.class, order.getType(0));
	assertEquals(StringValue.class, order.getType(1));
	assertNull(filtered.getUniqueFields());
}
 
Example #29
Source File: RequestedGlobalProperties.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private boolean checkCompatiblePartitioningFields(GlobalProperties props) {
	if(this.partitioningFields instanceof FieldList) {
		// partitioningFields as FieldList requires strict checking!
		return props.isExactlyPartitionedOnFields((FieldList)this.partitioningFields);
	} else {
		return props.isPartitionedOnFields(this.partitioningFields);
	}
}
 
Example #30
Source File: PartitionPushdownTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPartitioningNotPushedDown() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		
		@SuppressWarnings("unchecked")
		DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
		
		input
			.groupBy(0, 1).sum(2)
			.groupBy(0).sum(1)
			.output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		
		SingleInputPlanNode agg2Reducer = (SingleInputPlanNode) sink.getInput().getSource();
		SingleInputPlanNode agg2Combiner = (SingleInputPlanNode) agg2Reducer.getInput().getSource();
		SingleInputPlanNode agg1Reducer = (SingleInputPlanNode) agg2Combiner.getInput().getSource();
		
		assertEquals(ShipStrategyType.PARTITION_HASH, agg2Reducer.getInput().getShipStrategy());
		assertEquals(new FieldList(0), agg2Reducer.getInput().getShipStrategyKeys());
		
		assertEquals(ShipStrategyType.FORWARD, agg2Combiner.getInput().getShipStrategy());
		
		assertEquals(ShipStrategyType.PARTITION_HASH, agg1Reducer.getInput().getShipStrategy());
		assertEquals(new FieldList(0, 1), agg1Reducer.getInput().getShipStrategyKeys());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}