org.apache.flink.api.common.operators.SingleInputSemanticProperties Java Examples

The following examples show how to use org.apache.flink.api.common.operators.SingleInputSemanticProperties. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: SemanticPropertiesTranslationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testUnaryFunctionMovingForwardedAnnotation() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	@SuppressWarnings("unchecked")
	DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L));
	input.map(new ShufflingMapper<Long>()).output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
	Plan plan = env.createProgramPlan();

	GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
	MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput();

	SingleInputSemanticProperties semantics = mapper.getSemanticProperties();

	FieldSet fw1 = semantics.getForwardingTargetFields(0, 0);
	FieldSet fw2 = semantics.getForwardingTargetFields(0, 1);
	FieldSet fw3 = semantics.getForwardingTargetFields(0, 2);
	assertNotNull(fw1);
	assertNotNull(fw2);
	assertNotNull(fw3);
	assertTrue(fw1.contains(2));
	assertTrue(fw2.contains(0));
	assertTrue(fw3.contains(1));
}

Example #2

Source File: GroupReduceOperator.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}
		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}

Example #3

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testRangePartitioningErased() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	o.appendOrdering(2, StringValue.class, Order.ASCENDING);
	GlobalProperties gprops = new GlobalProperties();
	gprops.setRangePartitioned(o);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningOrdering());
	assertNull(result.getPartitioningFields());
}

Example #4

Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSortingPreserved4() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"2->7;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(1, gFields.size());
	assertTrue(gFields.contains(7));
	assertNotNull(order);
	assertEquals(1, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(IntValue.class, order.getType(0));
	assertNull(filtered.getUniqueFields());
}

Example #5

Source File: GroupCombineOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}

		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}

Example #6

Source File: SemanticPropertiesTranslationTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testUnaryFunctionReadFieldsAnnotation() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	@SuppressWarnings("unchecked")
	DataSet<Tuple3<Long, Long, Long>> input = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L));
	input.map(new ReadSetMapper<Tuple3<Long, Long, Long>>()).output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
	Plan plan = env.createProgramPlan();

	GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
	MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput();

	SingleInputSemanticProperties semantics = mapper.getSemanticProperties();

	FieldSet read = semantics.getReadFields(0);
	assertNotNull(read);
	assertEquals(2, read.size());
	assertTrue(read.contains(0));
	assertTrue(read.contains(2));
}

Example #7

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testUniqueFieldGroupsErased() {
	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0; 3; 5; 6; 7"}, null, null, tupleInfo, tupleInfo);

	FieldSet set1 = new FieldSet(0, 1, 2);
	FieldSet set2 = new FieldSet(3, 4);
	FieldSet set3 = new FieldSet(4, 5, 6, 7);
	GlobalProperties gprops = new GlobalProperties();
	gprops.addUniqueFieldCombination(set1);
	gprops.addUniqueFieldCombination(set2);
	gprops.addUniqueFieldCombination(set3);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
	Assert.assertNull(result.getUniqueFieldCombination());
}

Example #8

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testCustomPartitioningPreserved2() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	Partitioner<Tuple2<Long, Integer>> myP = new MockPartitioner();
	gprops.setCustomPartitioned(new FieldList(0, 4), myP);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.CUSTOM_PARTITIONING, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(2, pFields.size());
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(3));
	assertEquals(myP, result.getCustomPartitioner());
}

Example #9

Source File: RequestedGlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testAnyPartitioningPreserved2() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"2->0;1->3;7->4"}, null, null, tupleInfo, tupleInfo);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setAnyPartitioning(new FieldSet(0, 3, 4));

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.ANY_PARTITIONING, filtered.getPartitioning());
	assertNotNull(filtered.getPartitionedFields());
	assertEquals(3, filtered.getPartitionedFields().size());
	assertTrue(filtered.getPartitionedFields().contains(1));
	assertTrue(filtered.getPartitionedFields().contains(2));
	assertTrue(filtered.getPartitionedFields().contains(7));
	assertNull(filtered.getDataDistribution());
	assertNull(filtered.getCustomPartitioner());
	assertNull(filtered.getOrdering());
}

Example #10

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testHashPartitioningPreserved2() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0->1; 1->2; 4->3"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setHashPartitioned(new FieldList(0, 1, 4));

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.HASH_PARTITIONED, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(3, pFields.size());
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(2));
	assertTrue(pFields.contains(3));
}

Example #11

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testAnyPartitioningPreserved1() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setAnyPartitioning(new FieldList(0, 1, 4));

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.ANY_PARTITIONING, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(3, pFields.size());
	assertTrue(pFields.contains(0));
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(4));
}

Example #12

Source File: GlobalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testUniqueFieldGroupsPreserved1() {
	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;2;3;4"}, null, null, tupleInfo, tupleInfo);

	FieldSet set1 = new FieldSet(0, 1, 2);
	FieldSet set2 = new FieldSet(3, 4);
	FieldSet set3 = new FieldSet(4, 5, 6, 7);
	GlobalProperties gprops = new GlobalProperties();
	gprops.addUniqueFieldCombination(set1);
	gprops.addUniqueFieldCombination(set2);
	gprops.addUniqueFieldCombination(set3);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);
	Set<FieldSet> unique = result.getUniqueFieldCombination();
	FieldSet expected1 = new FieldSet(0, 1, 2);
	FieldSet expected2 = new FieldSet(3, 4);

	Assert.assertTrue(unique.size() == 2);
	Assert.assertTrue(unique.contains(expected1));
	Assert.assertTrue(unique.contains(expected2));
}

Example #13

Source File: LocalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testGroupingPreserved2() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->4;2->0;3->7"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = LocalProperties.forGrouping(new FieldList(0, 2, 3));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNotNull(filtered.getGroupedFields());
	assertEquals(3, filtered.getGroupedFields().size());
	assertTrue(filtered.getGroupedFields().contains(4));
	assertTrue(filtered.getGroupedFields().contains(0));
	assertTrue(filtered.getGroupedFields().contains(7));
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}

Example #14

Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSortingPreserved4() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"2->7;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(1, gFields.size());
	assertTrue(gFields.contains(7));
	assertNotNull(order);
	assertEquals(1, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(IntValue.class, order.getType(0));
	assertNull(filtered.getUniqueFields());
}

Example #15

Source File: RequestedLocalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testGroupingPreserved1() {

	SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"0;2;3"}, null, null, tupleInfo, tupleInfo);

	RequestedLocalProperties rlProp = new RequestedLocalProperties();
	rlProp.setGroupedFields(new FieldSet(0, 2, 3));

	RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);

	assertNotNull(filtered);
	assertNotNull(filtered.getGroupedFields());
	assertEquals(3, filtered.getGroupedFields().size());
	assertTrue(filtered.getGroupedFields().contains(0));
	assertTrue(filtered.getGroupedFields().contains(2));
	assertTrue(filtered.getGroupedFields().contains(3));
	assertNull(filtered.getOrdering());
}

Example #16

Source File: LocalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testUniqueFieldsErased() {

	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = new LocalProperties();
	lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
	lProps = lProps.addUniqueFields(new FieldSet(3,4));
	lProps = lProps.addUniqueFields(new FieldSet(4,5,6));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNull(filtered.getUniqueFields());
}

Example #17

Source File: LocalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testUniqueFieldsPreserved1() {

	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0;1;2;3;4"}, null, null, tupleInfo, tupleInfo);

	LocalProperties lProps = new LocalProperties();
	lProps = lProps.addUniqueFields(new FieldSet(0,1,2));
	lProps = lProps.addUniqueFields(new FieldSet(3,4));
	lProps = lProps.addUniqueFields(new FieldSet(4,5,6));

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldSet expected1 = new FieldSet(0,1,2);
	FieldSet expected2 = new FieldSet(3,4);

	assertNull(filtered.getGroupedFields());
	assertNull(filtered.getOrdering());
	assertNotNull(filtered.getUniqueFields());
	assertEquals(2, filtered.getUniqueFields().size());
	assertTrue(filtered.getUniqueFields().contains(expected1));
	assertTrue(filtered.getUniqueFields().contains(expected2));
}

Example #18

Source File: SemanticPropertiesTranslationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testUnaryFunctionWildcardForwardedAnnotation() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	@SuppressWarnings("unchecked")
	DataSet<Tuple3<Long, String, Integer>> input = env.fromElements(new Tuple3<Long, String, Integer>(3L, "test", 42));
	input.map(new WildcardForwardedMapper<Tuple3<Long, String, Integer>>()).output(new DiscardingOutputFormat<Tuple3<Long, String, Integer>>());
	Plan plan = env.createProgramPlan();

	GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
	MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput();

	SingleInputSemanticProperties semantics = mapper.getSemanticProperties();

	FieldSet fw1 = semantics.getForwardingTargetFields(0, 0);
	FieldSet fw2 = semantics.getForwardingTargetFields(0, 1);
	FieldSet fw3 = semantics.getForwardingTargetFields(0, 2);
	assertNotNull(fw1);
	assertNotNull(fw2);
	assertNotNull(fw3);
	assertTrue(fw1.contains(0));
	assertTrue(fw2.contains(1));
	assertTrue(fw3.contains(2));
}

Example #19

Source File: RequestedGlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testAnyPartitioningPreserved2() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"2->0;1->3;7->4"}, null, null, tupleInfo, tupleInfo);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setAnyPartitioning(new FieldSet(0, 3, 4));

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.ANY_PARTITIONING, filtered.getPartitioning());
	assertNotNull(filtered.getPartitionedFields());
	assertEquals(3, filtered.getPartitionedFields().size());
	assertTrue(filtered.getPartitionedFields().contains(1));
	assertTrue(filtered.getPartitionedFields().contains(2));
	assertTrue(filtered.getPartitionedFields().contains(7));
	assertNull(filtered.getDataDistribution());
	assertNull(filtered.getCustomPartitioner());
	assertNull(filtered.getOrdering());
}

Example #20

Source File: GlobalPropertiesFilteringTest.java From flink with Apache License 2.0

6 votes

@Test
public void testAnyPartitioningPreserved1() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"0;1;4"}, null, null, tupleInfo, tupleInfo);

	GlobalProperties gprops = new GlobalProperties();
	gprops.setAnyPartitioning(new FieldList(0, 1, 4));

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.ANY_PARTITIONING, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(3, pFields.size());
	assertTrue(pFields.contains(0));
	assertTrue(pFields.contains(1));
	assertTrue(pFields.contains(4));
}

Example #21

Source File: SemanticPropertiesTranslationTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testUnaryFunctionInPlaceForwardedAnnotation() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	@SuppressWarnings("unchecked")
	DataSet<Tuple3<Long, String, Integer>> input = env.fromElements(new Tuple3<Long, String, Integer>(3L, "test", 42));
	input.map(new IndividualForwardedMapper<Long, String, Integer>()).output(new DiscardingOutputFormat<Tuple3<Long, String, Integer>>());
	Plan plan = env.createProgramPlan();

	GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
	MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput();

	SingleInputSemanticProperties semantics = mapper.getSemanticProperties();

	FieldSet fw1 = semantics.getForwardingTargetFields(0, 0);
	FieldSet fw2 = semantics.getForwardingTargetFields(0, 2);
	assertNotNull(fw1);
	assertNotNull(fw2);
	assertTrue(fw1.contains(0));
	assertTrue(fw2.contains(2));
}

Example #22

Source File: SemanticPropUtilTest.java From flink with Apache License 2.0

5 votes

@Test
public void testReadFieldsSpaces() {
	String[] readFields = { "  f1  ; f2   " };
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, null, readFields, threeIntTupleType, threeIntTupleType);

	FieldSet fs = sp.getReadFields(0);
	assertTrue(fs.size() == 2);
	assertTrue(fs.contains(2));
	assertTrue(fs.contains(1));
}

Example #23

Source File: RequestedGlobalPropertiesFilteringTest.java From flink with Apache License 2.0

5 votes

@Test
public void testRangePartitioningPreserved2() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"7->3;1->1;2->6"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(3, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRangePartitioned(o);

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
	assertNotNull(filtered.getOrdering());
	assertEquals(3, filtered.getOrdering().getNumberOfFields());
	assertEquals(7, filtered.getOrdering().getFieldNumber(0).intValue());
	assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
	assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
	assertEquals(LongValue.class, filtered.getOrdering().getType(0));
	assertEquals(IntValue.class, filtered.getOrdering().getType(1));
	assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
	assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
	assertNull(filtered.getPartitionedFields());
	assertNull(filtered.getDataDistribution());
	assertNull(filtered.getCustomPartitioner());
}

Example #24

Source File: RequestedLocalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test(expected = IndexOutOfBoundsException.class)
public void testInvalidInputIndex() {

	SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"1; 4"}, null, null, tupleInfo, tupleInfo);

	RequestedLocalProperties rlProp = new RequestedLocalProperties();
	rlProp.setGroupedFields(new FieldSet(1, 4));

	rlProp.filterBySemanticProperties(sProps, 1);
}

Example #25

Source File: GroupCombineNodeTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testGetSemanticProperties() {

	SingleInputSemanticProperties origProps = new SingleInputSemanticProperties();
	origProps.addForwardedField(0, 1);
	origProps.addForwardedField(2, 2);
	origProps.addForwardedField(3, 4);
	origProps.addForwardedField(6, 0);
	origProps.addReadFields(new FieldSet(0, 2, 4, 7));

	GroupCombineOperatorBase<?,?,?> op = mock(GroupCombineOperatorBase.class);
	when(op.getSemanticProperties()).thenReturn(origProps);
	when(op.getKeyColumns(0)).thenReturn(new int[]{3,2});
	when(op.getParameters()).thenReturn(new Configuration());

	GroupCombineNode node = new GroupCombineNode(op);

	SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();

	assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
	assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
	assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
	assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
	assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
	assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
	assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
	assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
	assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
	assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);

	assertTrue(filteredProps.getReadFields(0).size() == 4);
	assertTrue(filteredProps.getReadFields(0).contains(0));
	assertTrue(filteredProps.getReadFields(0).contains(2));
	assertTrue(filteredProps.getReadFields(0).contains(4));
	assertTrue(filteredProps.getReadFields(0).contains(7));

}

Example #26

Source File: SemanticPropertiesPrecedenceTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testFunctionSkipCodeAnalysisAnnotationPrecedence() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().setCodeAnalysisMode(CodeAnalysisMode.OPTIMIZE);

	@SuppressWarnings("unchecked")
	DataSet<Tuple3<Long, String, Integer>> input = env.fromElements(Tuple3.of(3L, "test", 42));
	input
			.map(new WildcardForwardedMapperWithSkipAnnotation<Tuple3<Long, String, Integer>>())
			.output(new DiscardingOutputFormat<Tuple3<Long, String, Integer>>());
	Plan plan = env.createProgramPlan();

	GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
	MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput();

	SingleInputSemanticProperties semantics = mapper.getSemanticProperties();

	FieldSet fw1 = semantics.getForwardingTargetFields(0, 0);
	FieldSet fw2 = semantics.getForwardingTargetFields(0, 1);
	FieldSet fw3 = semantics.getForwardingTargetFields(0, 2);
	assertNotNull(fw1);
	assertNotNull(fw2);
	assertNotNull(fw3);
	assertFalse(fw1.contains(0));
	assertFalse(fw2.contains(1));
	assertFalse(fw3.contains(2));
}

Example #27

Source File: GroupReduceNodeTest.java From flink with Apache License 2.0

5 votes

@Test
public void testGetSemanticProperties() {

	SingleInputSemanticProperties origProps = new SingleInputSemanticProperties();
	origProps.addForwardedField(0, 1);
	origProps.addForwardedField(2, 2);
	origProps.addForwardedField(3, 4);
	origProps.addForwardedField(6, 0);
	origProps.addReadFields(new FieldSet(0, 2, 4, 7));

	GroupReduceOperatorBase<?,?,?> op = mock(GroupReduceOperatorBase.class);
	when(op.getSemanticProperties()).thenReturn(origProps);
	when(op.getKeyColumns(0)).thenReturn(new int[]{3,2});
	when(op.getParameters()).thenReturn(new Configuration());

	GroupReduceNode node = new GroupReduceNode(op);

	SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();

	assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
	assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
	assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
	assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
	assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
	assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
	assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
	assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
	assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
	assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);

	assertTrue(filteredProps.getReadFields(0).size() == 4);
	assertTrue(filteredProps.getReadFields(0).contains(0));
	assertTrue(filteredProps.getReadFields(0).contains(2));
	assertTrue(filteredProps.getReadFields(0).contains(4));
	assertTrue(filteredProps.getReadFields(0).contains(7));

}

Example #28

Source File: SemanticPropUtilTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testNonForwardedNone() {
	String[] nonForwardedFields = { "" };
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, nonForwardedFields, null, threeIntTupleType, threeIntTupleType);

	assertTrue(sp.getForwardingTargetFields(0, 0).contains(0));
	assertTrue(sp.getForwardingTargetFields(0, 1).contains(1));
	assertTrue(sp.getForwardingTargetFields(0, 2).contains(2));
}

Example #29

Source File: SemanticPropertiesPrecedenceTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testFunctionApiPrecedence() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().setCodeAnalysisMode(CodeAnalysisMode.OPTIMIZE);

	@SuppressWarnings("unchecked")
	DataSet<Tuple3<Long, String, Integer>> input = env.fromElements(Tuple3.of(3L, "test", 42));
	input
			.map(new WildcardForwardedMapper<Tuple3<Long, String, Integer>>())
			.withForwardedFields("f0")
			.output(new DiscardingOutputFormat<Tuple3<Long, String, Integer>>());
	Plan plan = env.createProgramPlan();

	GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
	MapOperatorBase<?, ?, ?> mapper = (MapOperatorBase<?, ?, ?>) sink.getInput();

	SingleInputSemanticProperties semantics = mapper.getSemanticProperties();

	FieldSet fw1 = semantics.getForwardingTargetFields(0, 0);
	FieldSet fw2 = semantics.getForwardingTargetFields(0, 1);
	FieldSet fw3 = semantics.getForwardingTargetFields(0, 2);
	assertNotNull(fw1);
	assertNotNull(fw2);
	assertNotNull(fw3);
	assertTrue(fw1.contains(0));
	assertFalse(fw2.contains(1));
	assertFalse(fw3.contains(2));
}

Example #30

Source File: SemanticPropUtilTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testNonForwardedIndividualStrings() {
	String[] nonForwardedFields = { "f1", "f2" };
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, null, nonForwardedFields, null, threeIntTupleType, threeIntTupleType);

	assertTrue(sp.getForwardingTargetFields(0, 0).contains(0));
	assertTrue(sp.getForwardingTargetFields(0, 1).size() == 0);
	assertTrue(sp.getForwardingTargetFields(0, 2).size() == 0);
}