org.apache.flink.api.java.functions.SemanticPropUtil Java Examples

The following examples show how to use org.apache.flink.api.java.functions.SemanticPropUtil. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testAnyPartitioningPreserved1() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setAnyPartitioning(new FieldList(0, 1, 4));

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.ANY_PARTITIONING, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(3, pFields.size());
	assertTrue(pFields.contains(0));
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(4));
}

Example #2

Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSortingPreserved4() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"2->7;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(1, gFields.size());
	assertTrue(gFields.contains(7));
	assertNotNull(order);
	assertEquals(1, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(IntValue.class, order.getType(0));
	assertNull(filtered.getUniqueFields());
}

Example #3

Source File: ReduceOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}
		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}

Example #4

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testRangePartitioningErased() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	o.appendOrdering(2, StringValue.class, Order.ASCENDING);
	GlobalProperties gprops = new GlobalProperties();
	gprops.setRangePartitioned(o);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningOrdering());
	assertNull(result.getPartitioningFields());
}

Example #5

Source File: CoGroupOperator.java From flink with Apache License 2.0

6 votes

@Override
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
				(this.keys1 instanceof SelectorFunctionKeys ||
				this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}

Example #6

Source File: RequestedLocalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testGroupingPreserved1() {

	SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"0;2;3"}, null, null, tupleInfo, tupleInfo);

	RequestedLocalProperties rlProp = new RequestedLocalProperties();
	rlProp.setGroupedFields(new FieldSet(0, 2, 3));

	RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);

	assertNotNull(filtered);
	assertNotNull(filtered.getGroupedFields());
	assertEquals(3, filtered.getGroupedFields().size());
	assertTrue(filtered.getGroupedFields().contains(0));
	assertTrue(filtered.getGroupedFields().contains(2));
	assertTrue(filtered.getGroupedFields().contains(3));
	assertNull(filtered.getOrdering());
}

Example #7

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testCustomPartitioningPreserved1() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
	gprops.setCustomPartitioned(new FieldList(0, 4), myP);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(2, pFields.size());
	assertTrue(pFields.contains(0));
	assertTrue(pFields.contains(4));
	assertEquals(myP, result.getCustomPartitioner());
}

Example #8

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testHashPartitioningPreserved2() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1, 4));

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.HASH_PARTITIONED, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(3, pFields.size());
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(2));
	assertTrue(pFields.contains(3));
}

Example #9

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testCustomPartitioningPreserved2() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
	gprops.setCustomPartitioned(new FieldList(0, 4), myP);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(2, pFields.size());
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(3));
	assertEquals(myP, result.getCustomPartitioner());
}

Example #10

Source File: ReduceOperator.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}
		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}

Example #11

Source File: UdfAnalyzerTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@SuppressWarnings({ "rawtypes", "unchecked" })
public static void compareAnalyzerResultWithAnnotationsDualInputWithKeys(Class<?> baseClass, Class<?> clazz,
	TypeInformation<?> in1Type, TypeInformation<?> in2Type, TypeInformation<?> outType, String[] keys1, String[] keys2) {
	// expected
	final Set<Annotation> annotations = FunctionAnnotation.readDualForwardAnnotations(clazz);
	final DualInputSemanticProperties expected = SemanticPropUtil.getSemanticPropsDual(annotations, in1Type,
			in2Type, outType);

	// actual
	final UdfAnalyzer ua = new UdfAnalyzer(baseClass, clazz, "operator", in1Type, in2Type, outType, (keys1 == null) ? null
			: new Keys.ExpressionKeys(keys1, in1Type), (keys2 == null) ? null : new Keys.ExpressionKeys(
					keys2, in2Type), true);
	ua.analyze();
	final DualInputSemanticProperties actual = (DualInputSemanticProperties) ua.getSemanticProperties();

	assertEquals(expected.toString(), actual.toString());
}

Example #12

Source File: LocalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testAllErased1() {

	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 1, 2));
	lProps = lProps.addUniqueFields(new FieldSet(3,4));
	lProps = lProps.addUniqueFields(new FieldSet(5,6));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}

Example #13

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testUniqueFieldGroupsErased() {
	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0; 3; 5; 6; 7"}, null, null, tupleInfo, tupleInfo);

	FieldSet set1 = new FieldSet(0, 1, 2);
	FieldSet set2 = new FieldSet(3, 4);
	FieldSet set3 = new FieldSet(4, 5, 6, 7);
	GlobalProperties gprops = new GlobalProperties();
	gprops.addUniqueFieldCombination(set1);
	gprops.addUniqueFieldCombination(set2);
	gprops.addUniqueFieldCombination(set3);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
	Assert.assertNull(result.getUniqueFieldCombination());
}

Example #14

Source File: RequestedGlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testHashPartitioningPreserved2() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"2->0;1->3;7->4"}, null, null, tupleInfo, tupleInfo);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setHashPartitioned(new FieldSet(0, 3, 4));

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered.getPartitioning());
	assertNotNull(filtered.getPartitionedFields());
	assertEquals(3, filtered.getPartitionedFields().size());
	assertTrue(filtered.getPartitionedFields().contains(1));
	assertTrue(filtered.getPartitionedFields().contains(2));
	assertTrue(filtered.getPartitionedFields().contains(7));
	assertNull(filtered.getDataDistribution());
	assertNull(filtered.getCustomPartitioner());
	assertNull(filtered.getOrdering());
}

Example #15

Source File: LocalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testGroupingPreserved2() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->4;2->0;3->7"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNotNull(filtered.getGroupedFields());
	assertEquals(3, filtered.getGroupedFields().size());
	assertTrue(filtered.getGroupedFields().contains(4));
	assertTrue(filtered.getGroupedFields().contains(0));
	assertTrue(filtered.getGroupedFields().contains(7));
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}

Example #16

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testCustomPartitioningErased() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
	gprops.setCustomPartitioned(new FieldList(0, 4), myP);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningFields());
	assertNull(result.getCustomPartitioner());
}

Example #17

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testUniqueFieldGroupsPreserved2() {
	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->5;1;2;3->6;4"}, null, null, tupleInfo, tupleInfo);

	FieldSet set1 = new FieldSet(0, 1, 2);
	FieldSet set2 = new FieldSet(3, 4);
	FieldSet set3 = new FieldSet(4, 5, 6, 7);
	GlobalProperties gprops = new GlobalProperties();
	gprops.addUniqueFieldCombination(set1);
	gprops.addUniqueFieldCombination(set2);
	gprops.addUniqueFieldCombination(set3);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
	Set<FieldSet> unique = result.getUniqueFieldCombination();
	FieldSet expected1 = new FieldSet(1, 2, 5);
	FieldSet expected2 = new FieldSet(4, 6);

	Assert.assertTrue(unique.size() == 2);
	Assert.assertTrue(unique.contains(expected1));
	Assert.assertTrue(unique.contains(expected2));
}

Example #18

Source File: GroupReduceOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}
		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}

Example #19

Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testUniqueFieldsPreserved3() {

	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->7;1->6;2->5;3->4;4->3"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = new LocalProperties();
	lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
	lProps = lProps.addUniqueFields(new FieldSet(3,4));
	lProps = lProps.addUniqueFields(new FieldSet(4,5,6));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldSet expected1 = new FieldSet(5,6,7);
	FieldSet expected2 = new FieldSet(3,4);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNotNull(filtered.getUniqueFields());
	assertEquals(2, filtered.getUniqueFields().size());
	assertTrue(filtered.getUniqueFields().contains(expected1));
	assertTrue(filtered.getUniqueFields().contains(expected2));
}

Example #20

Source File: LocalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testSortingPreserved4() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"2->7;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(1, gFields.size());
	assertTrue(gFields.contains(7));
	assertNotNull(order);
	assertEquals(1, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(IntValue.class, order.getType(0));
	assertNull(filtered.getUniqueFields());
}

Example #21

Source File: LocalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testSortingErased() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNull(gFields);
	assertNull(order);
	assertNull(filtered.getUniqueFields());
}

Example #22

Source File: GroupCombineOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}

		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}

Example #23

Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testUniqueFieldsErased() {

	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = new LocalProperties();
	lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
	lProps = lProps.addUniqueFields(new FieldSet(3,4));
	lProps = lProps.addUniqueFields(new FieldSet(4,5,6));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}

Example #24

Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testUniqueFieldsPreserved1() {

	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;1;2;3;4"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = new LocalProperties();
	lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
	lProps = lProps.addUniqueFields(new FieldSet(3,4));
	lProps = lProps.addUniqueFields(new FieldSet(4,5,6));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldSet expected1 = new FieldSet(0,1,2);
	FieldSet expected2 = new FieldSet(3,4);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNotNull(filtered.getUniqueFields());
	assertEquals(2, filtered.getUniqueFields().size());
	assertTrue(filtered.getUniqueFields().contains(expected1));
	assertTrue(filtered.getUniqueFields().contains(expected2));
}

Example #25

Source File: GlobalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testRangePartitioningErased() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	o.appendOrdering(2, StringValue.class, Order.ASCENDING);
	GlobalProperties gprops = new GlobalProperties();
	gprops.setRangePartitioned(o);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningOrdering());
	assertNull(result.getPartitioningFields());
}

Example #26

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testUniqueFieldGroupsPreserved1() {
	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;2;3;4"}, null, null, tupleInfo, tupleInfo);

	FieldSet set1 = new FieldSet(0, 1, 2);
	FieldSet set2 = new FieldSet(3, 4);
	FieldSet set3 = new FieldSet(4, 5, 6, 7);
	GlobalProperties gprops = new GlobalProperties();
	gprops.addUniqueFieldCombination(set1);
	gprops.addUniqueFieldCombination(set2);
	gprops.addUniqueFieldCombination(set3);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
	Set<FieldSet> unique = result.getUniqueFieldCombination();
	FieldSet expected1 = new FieldSet(0, 1, 2);
	FieldSet expected2 = new FieldSet(3, 4);

	Assert.assertTrue(unique.size() == 2);
	Assert.assertTrue(unique.contains(expected1));
	Assert.assertTrue(unique.contains(expected2));
}

Example #27

Source File: GlobalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testHashPartitioningPreserved1() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1, 4));

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.HASH_PARTITIONED, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(3, pFields.size());
	assertTrue(pFields.contains(0));
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(4));
}

Example #28

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testHashPartitioningPreserved1() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1, 4));

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.HASH_PARTITIONED, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(3, pFields.size());
	assertTrue(pFields.contains(0));
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(4));
}

Example #29

Source File: GroupReduceOperator.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}
		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}

Example #30

Source File: GlobalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testCustomPartitioningPreserved2() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
	gprops.setCustomPartitioned(new FieldList(0, 4), myP);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(2, pFields.size());
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(3));
	assertEquals(myP, result.getCustomPartitioner());
}