org.apache.flink.api.common.operators.DualInputSemanticProperties Java Examples

The following examples show how to use org.apache.flink.api.common.operators.DualInputSemanticProperties. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: CoGroupOperator.java From flink with Apache License 2.0

6 votes

@Override
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
				(this.keys1 instanceof SelectorFunctionKeys ||
				this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}

Example #2

Source File: JoinOperator.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
@Internal
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			(this.keys1 instanceof SelectorFunctionKeys ||
					this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}

Example #3

Source File: CoGroupOperator.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
				(this.keys1 instanceof SelectorFunctionKeys ||
				this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}

Example #4

Source File: SemanticPropUtilTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testForwardedReadDual() {
	String[] forwardedFieldsFirst = { "f1->f2; f2->f3" };
	String[] forwardedFieldsSecond = { "f1->f1; f2->f0" };
	String[] readFieldsFirst = {"0;2"};
	String[] readFieldsSecond = {"1"};
	DualInputSemanticProperties dsp = new DualInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null,
			null, readFieldsFirst, readFieldsSecond, fourIntTupleType, fourIntTupleType, fourIntTupleType);

	assertTrue(dsp.getForwardingTargetFields(0, 1).contains(2));
	assertTrue(dsp.getForwardingTargetFields(0, 2).contains(3));
	assertTrue(dsp.getForwardingTargetFields(1, 1).contains(1));
	assertTrue(dsp.getForwardingTargetFields(1, 2).contains(0));
	assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0);
	assertTrue(dsp.getForwardingTargetFields(1, 3).size() == 0);
	assertTrue(dsp.getReadFields(0).size() == 2);
	assertTrue(dsp.getReadFields(0).contains(0));
	assertTrue(dsp.getReadFields(0).contains(2));
	assertTrue(dsp.getReadFields(1).size() == 1);
	assertTrue(dsp.getReadFields(1).contains(1));

}

Example #5

Source File: UdfAnalyzerTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@SuppressWarnings({ "rawtypes", "unchecked" })
public static void compareAnalyzerResultWithAnnotationsDualInputWithKeys(Class<?> baseClass, Class<?> clazz,
	TypeInformation<?> in1Type, TypeInformation<?> in2Type, TypeInformation<?> outType, String[] keys1, String[] keys2) {
	// expected
	final Set<Annotation> annotations = FunctionAnnotation.readDualForwardAnnotations(clazz);
	final DualInputSemanticProperties expected = SemanticPropUtil.getSemanticPropsDual(annotations, in1Type,
			in2Type, outType);

	// actual
	final UdfAnalyzer ua = new UdfAnalyzer(baseClass, clazz, "operator", in1Type, in2Type, outType, (keys1 == null) ? null
			: new Keys.ExpressionKeys(keys1, in1Type), (keys2 == null) ? null : new Keys.ExpressionKeys(
					keys2, in2Type), true);
	ua.analyze();
	final DualInputSemanticProperties actual = (DualInputSemanticProperties) ua.getSemanticProperties();

	assertEquals(expected.toString(), actual.toString());
}

Example #6

Source File: SemanticPropUtilTest.java From flink with Apache License 2.0

6 votes

@Test
public void testForwardedReadDual() {
	String[] forwardedFieldsFirst = { "f1->f2; f2->f3" };
	String[] forwardedFieldsSecond = { "f1->f1; f2->f0" };
	String[] readFieldsFirst = {"0;2"};
	String[] readFieldsSecond = {"1"};
	DualInputSemanticProperties dsp = new DualInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null,
			null, readFieldsFirst, readFieldsSecond, fourIntTupleType, fourIntTupleType, fourIntTupleType);

	assertTrue(dsp.getForwardingTargetFields(0, 1).contains(2));
	assertTrue(dsp.getForwardingTargetFields(0, 2).contains(3));
	assertTrue(dsp.getForwardingTargetFields(1, 1).contains(1));
	assertTrue(dsp.getForwardingTargetFields(1, 2).contains(0));
	assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0);
	assertTrue(dsp.getForwardingTargetFields(1, 3).size() == 0);
	assertTrue(dsp.getReadFields(0).size() == 2);
	assertTrue(dsp.getReadFields(0).contains(0));
	assertTrue(dsp.getReadFields(0).contains(2));
	assertTrue(dsp.getReadFields(1).size() == 1);
	assertTrue(dsp.getReadFields(1).contains(1));

}

Example #7

Source File: SemanticPropUtilTest.java From flink with Apache License 2.0

6 votes

@Test
public void testForwardedReadDual() {
	String[] forwardedFieldsFirst = { "f1->f2; f2->f3" };
	String[] forwardedFieldsSecond = { "f1->f1; f2->f0" };
	String[] readFieldsFirst = {"0;2"};
	String[] readFieldsSecond = {"1"};
	DualInputSemanticProperties dsp = new DualInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsDualFromString(dsp, forwardedFieldsFirst, forwardedFieldsSecond, null,
			null, readFieldsFirst, readFieldsSecond, fourIntTupleType, fourIntTupleType, fourIntTupleType);

	assertTrue(dsp.getForwardingTargetFields(0, 1).contains(2));
	assertTrue(dsp.getForwardingTargetFields(0, 2).contains(3));
	assertTrue(dsp.getForwardingTargetFields(1, 1).contains(1));
	assertTrue(dsp.getForwardingTargetFields(1, 2).contains(0));
	assertTrue(dsp.getForwardingTargetFields(0, 0).size() == 0);
	assertTrue(dsp.getForwardingTargetFields(1, 3).size() == 0);
	assertTrue(dsp.getReadFields(0).size() == 2);
	assertTrue(dsp.getReadFields(0).contains(0));
	assertTrue(dsp.getReadFields(0).contains(2));
	assertTrue(dsp.getReadFields(1).size() == 1);
	assertTrue(dsp.getReadFields(1).contains(1));

}

Example #8

Source File: JoinOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			(this.keys1 instanceof SelectorFunctionKeys ||
					this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}

Example #9

Source File: JoinOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			(this.keys1 instanceof SelectorFunctionKeys ||
					this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}

Example #10

Source File: CoGroupOperator.java From flink with Apache License 2.0

6 votes

@Override
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
				(this.keys1 instanceof SelectorFunctionKeys ||
				this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}

Example #11

Source File: RequestedGlobalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testDualHashPartitioningPreserved() {

	DualInputSemanticProperties dprops = new DualInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[]{"0;2;4"}, new String[]{"1->3;4->6;3->7"},
			null, null, null, null, tupleInfo, tupleInfo, tupleInfo);

	RequestedGlobalProperties gprops1 = new RequestedGlobalProperties();
	RequestedGlobalProperties gprops2 = new RequestedGlobalProperties();
	gprops1.setHashPartitioned(new FieldSet(2, 0, 4));
	gprops2.setHashPartitioned(new FieldSet(3, 6, 7));
	RequestedGlobalProperties filtered1 = gprops1.filterBySemanticProperties(dprops, 0);
	RequestedGlobalProperties filtered2 = gprops2.filterBySemanticProperties(dprops, 1);

	assertNotNull(filtered1);
	assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered1.getPartitioning());
	assertNotNull(filtered1.getPartitionedFields());
	assertEquals(3, filtered1.getPartitionedFields().size());
	assertTrue(filtered1.getPartitionedFields().contains(0));
	assertTrue(filtered1.getPartitionedFields().contains(2));
	assertTrue(filtered1.getPartitionedFields().contains(4));
	assertNull(filtered1.getOrdering());
	assertNull(filtered1.getCustomPartitioner());
	assertNull(filtered1.getDataDistribution());

	assertNotNull(filtered2);
	assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered2.getPartitioning());
	assertNotNull(filtered2.getPartitionedFields());
	assertEquals(3, filtered2.getPartitionedFields().size());
	assertTrue(filtered2.getPartitionedFields().contains(1));
	assertTrue(filtered2.getPartitionedFields().contains(3));
	assertTrue(filtered2.getPartitionedFields().contains(4));
	assertNull(filtered2.getOrdering());
	assertNull(filtered2.getCustomPartitioner());
	assertNull(filtered2.getDataDistribution());
}

Example #12

Source File: TwoInputUdfOperator.java From flink with Apache License 2.0

5 votes

@Override
@Internal
public DualInputSemanticProperties getSemanticProperties() {
	if (this.udfSemantics == null || analyzedUdfSemantics) {
		DualInputSemanticProperties props = extractSemanticAnnotationsFromUdf(getFunction().getClass());
		if (props != null) {
			setSemanticProperties(props);
		}
	}
	if (this.udfSemantics == null) {
		setSemanticProperties(new DualInputSemanticProperties());
	}
	return this.udfSemantics;
}

Example #13

Source File: TwoInputUdfOperator.java From flink with Apache License 2.0

5 votes

@Override
@Internal
public DualInputSemanticProperties getSemanticProperties() {
	if (this.udfSemantics == null || analyzedUdfSemantics) {
		DualInputSemanticProperties props = extractSemanticAnnotationsFromUdf(getFunction().getClass());
		if (props != null) {
			setSemanticProperties(props);
		}
	}
	if (this.udfSemantics == null) {
		setSemanticProperties(new DualInputSemanticProperties());
	}
	return this.udfSemantics;
}

Example #14

Source File: JoinOperator.java From flink with Apache License 2.0

5 votes

@Override
protected DualInputSemanticProperties extractSemanticAnnotationsFromUdf(Class<?> udfClass) {
	if (function instanceof DefaultJoin.WrappingFlatJoinFunction) {
		return super.extractSemanticAnnotationsFromUdf(((WrappingFunction<?>) function).getWrappedFunction().getClass());
	} else {
		return super.extractSemanticAnnotationsFromUdf(function.getClass());
	}
}

Example #15

Source File: SemanticPropUtilTest.java From flink with Apache License 2.0

5 votes

@Test(expected = InvalidSemanticAnnotationException.class)
public void testForwardedNonForwardedFirstCheck() {
	String[] forwarded = { "1" };
	String[] nonForwarded = { "1" };
	SemanticPropUtil.getSemanticPropsDualFromString(new DualInputSemanticProperties(), forwarded, null, nonForwarded, null, null, null,
			threeIntTupleType, threeIntTupleType, threeIntTupleType);
}

Example #16

Source File: SemanticPropUtil.java From Flink-CEPplus with Apache License 2.0

5 votes

public static void getSemanticPropsDualFromString(DualInputSemanticProperties result,
		String[] forwardedFirst, String[] forwardedSecond,
		String[] nonForwardedFirst, String[] nonForwardedSecond, String[]
		readFieldsFirst, String[] readFieldsSecond,
		TypeInformation<?> inType1, TypeInformation<?> inType2, TypeInformation<?> outType) {
	getSemanticPropsDualFromString(result, forwardedFirst, forwardedSecond, nonForwardedFirst,
		nonForwardedSecond, readFieldsFirst, readFieldsSecond, inType1, inType2, outType,
		false);
}

Example #17

Source File: SemanticPropUtilTest.java From flink with Apache License 2.0

5 votes

@Test(expected = InvalidSemanticAnnotationException.class)
public void testNonForwardedDualInvalidTypes2() {

	String[] nonForwardedFieldsSecond = { "f1" };
	DualInputSemanticProperties dsp = new DualInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null,
			null, nonForwardedFieldsSecond, null, null, threeIntTupleType, pojoInTupleType, threeIntTupleType);
}

Example #18

Source File: SemanticPropUtilTest.java From flink with Apache License 2.0

5 votes

@Test(expected = InvalidSemanticAnnotationException.class)
public void testNonForwardedDualInvalidTypes1() {

	String[] nonForwardedFieldsFirst = { "f1" };
	DualInputSemanticProperties dsp = new DualInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null,
			nonForwardedFieldsFirst, null, null, null, fiveIntTupleType, threeIntTupleType, threeIntTupleType);
}

Example #19

Source File: RequestedLocalPropertiesFilteringTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testDualGroupingPreserved() {

	DualInputSemanticProperties dprops = new DualInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[]{"1->0;3;2->4"}, new String[]{"0->7;1"},
			null, null, null, null, tupleInfo, tupleInfo, tupleInfo);

	RequestedLocalProperties lprops1 = new RequestedLocalProperties();
	lprops1.setGroupedFields(new FieldSet(0,3,4));

	RequestedLocalProperties lprops2 = new RequestedLocalProperties();
	lprops2.setGroupedFields(new FieldSet(7, 1));

	RequestedLocalProperties filtered1 = lprops1.filterBySemanticProperties(dprops, 0);
	RequestedLocalProperties filtered2 = lprops2.filterBySemanticProperties(dprops, 1);

	assertNotNull(filtered1);
	assertNotNull(filtered1.getGroupedFields());
	assertEquals(3, filtered1.getGroupedFields().size());
	assertTrue(filtered1.getGroupedFields().contains(1));
	assertTrue(filtered1.getGroupedFields().contains(2));
	assertTrue(filtered1.getGroupedFields().contains(3));
	assertNull(filtered1.getOrdering());

	assertNotNull(filtered2);
	assertNotNull(filtered2.getGroupedFields());
	assertEquals(2, filtered2.getGroupedFields().size());
	assertTrue(filtered2.getGroupedFields().contains(0));
	assertTrue(filtered2.getGroupedFields().contains(1));
	assertNull(filtered2.getOrdering());
}

Example #20

Source File: RequestedLocalPropertiesFilteringTest.java From flink with Apache License 2.0

5 votes

@Test
public void testDualGroupingPreserved() {

	DualInputSemanticProperties dprops = new DualInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[]{"1->0;3;2->4"}, new String[]{"0->7;1"},
			null, null, null, null, tupleInfo, tupleInfo, tupleInfo);

	RequestedLocalProperties lprops1 = new RequestedLocalProperties();
	lprops1.setGroupedFields(new FieldSet(0,3,4));

	RequestedLocalProperties lprops2 = new RequestedLocalProperties();
	lprops2.setGroupedFields(new FieldSet(7, 1));

	RequestedLocalProperties filtered1 = lprops1.filterBySemanticProperties(dprops, 0);
	RequestedLocalProperties filtered2 = lprops2.filterBySemanticProperties(dprops, 1);

	assertNotNull(filtered1);
	assertNotNull(filtered1.getGroupedFields());
	assertEquals(3, filtered1.getGroupedFields().size());
	assertTrue(filtered1.getGroupedFields().contains(1));
	assertTrue(filtered1.getGroupedFields().contains(2));
	assertTrue(filtered1.getGroupedFields().contains(3));
	assertNull(filtered1.getOrdering());

	assertNotNull(filtered2);
	assertNotNull(filtered2.getGroupedFields());
	assertEquals(2, filtered2.getGroupedFields().size());
	assertTrue(filtered2.getGroupedFields().contains(0));
	assertTrue(filtered2.getGroupedFields().contains(1));
	assertNull(filtered2.getOrdering());
}

Example #21

Source File: JoinOperator.java From flink with Apache License 2.0

5 votes

@Override
protected DualInputSemanticProperties extractSemanticAnnotationsFromUdf(Class<?> udfClass) {
	if (function instanceof DefaultJoin.WrappingFlatJoinFunction) {
		return super.extractSemanticAnnotationsFromUdf(((WrappingFunction<?>) function).getWrappedFunction().getClass());
	} else {
		return super.extractSemanticAnnotationsFromUdf(function.getClass());
	}
}

Example #22

Source File: SemanticPropUtil.java From flink with Apache License 2.0

5 votes

public static void getSemanticPropsDualFromString(DualInputSemanticProperties result,
		String[] forwardedFirst, String[] forwardedSecond,
		String[] nonForwardedFirst, String[] nonForwardedSecond, String[]
		readFieldsFirst, String[] readFieldsSecond,
		TypeInformation<?> inType1, TypeInformation<?> inType2, TypeInformation<?> outType) {
	getSemanticPropsDualFromString(result, forwardedFirst, forwardedSecond, nonForwardedFirst,
		nonForwardedSecond, readFieldsFirst, readFieldsSecond, inType1, inType2, outType,
		false);
}

Example #23

Source File: SemanticPropertiesProjectionTest.java From flink with Apache License 2.0

5 votes

@Test
public void testCrossProjectionSemProps1() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);

	tupleDs.cross(tupleDs)
			.projectFirst(2, 3)
			.projectSecond(1, 4)
			.output(new DiscardingOutputFormat<Tuple>());

	Plan plan = env.createProgramPlan();

	GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
	CrossOperatorBase<?, ?, ?, ?> projectCrossOperator = ((CrossOperatorBase<?, ?, ?, ?>) sink.getInput());

	DualInputSemanticProperties props = projectCrossOperator.getSemanticProperties();

	assertEquals(1, props.getForwardingTargetFields(0, 2).size());
	assertEquals(1, props.getForwardingTargetFields(0, 3).size());
	assertEquals(1, props.getForwardingTargetFields(1, 1).size());
	assertEquals(1, props.getForwardingTargetFields(1, 4).size());

	assertTrue(props.getForwardingTargetFields(0, 2).contains(0));
	assertTrue(props.getForwardingTargetFields(0, 3).contains(1));
	assertTrue(props.getForwardingTargetFields(1, 1).contains(2));
	assertTrue(props.getForwardingTargetFields(1, 4).contains(3));
}

Example #24

Source File: RequestedLocalPropertiesFilteringTest.java From flink with Apache License 2.0

5 votes

@Test
public void testDualGroupingPreserved() {

	DualInputSemanticProperties dprops = new DualInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[]{"1->0;3;2->4"}, new String[]{"0->7;1"},
			null, null, null, null, tupleInfo, tupleInfo, tupleInfo);

	RequestedLocalProperties lprops1 = new RequestedLocalProperties();
	lprops1.setGroupedFields(new FieldSet(0,3,4));

	RequestedLocalProperties lprops2 = new RequestedLocalProperties();
	lprops2.setGroupedFields(new FieldSet(7, 1));

	RequestedLocalProperties filtered1 = lprops1.filterBySemanticProperties(dprops, 0);
	RequestedLocalProperties filtered2 = lprops2.filterBySemanticProperties(dprops, 1);

	assertNotNull(filtered1);
	assertNotNull(filtered1.getGroupedFields());
	assertEquals(3, filtered1.getGroupedFields().size());
	assertTrue(filtered1.getGroupedFields().contains(1));
	assertTrue(filtered1.getGroupedFields().contains(2));
	assertTrue(filtered1.getGroupedFields().contains(3));
	assertNull(filtered1.getOrdering());

	assertNotNull(filtered2);
	assertNotNull(filtered2.getGroupedFields());
	assertEquals(2, filtered2.getGroupedFields().size());
	assertTrue(filtered2.getGroupedFields().contains(0));
	assertTrue(filtered2.getGroupedFields().contains(1));
	assertNull(filtered2.getOrdering());
}

Example #25

Source File: SemanticPropUtilTest.java From flink with Apache License 2.0

5 votes

@Test(expected = InvalidSemanticAnnotationException.class)
public void testNonForwardedDualInvalidTypes1() {

	String[] nonForwardedFieldsFirst = { "f1" };
	DualInputSemanticProperties dsp = new DualInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null,
			nonForwardedFieldsFirst, null, null, null, fiveIntTupleType, threeIntTupleType, threeIntTupleType);
}

Example #26

Source File: SemanticPropUtilTest.java From flink with Apache License 2.0

5 votes

@Test(expected = InvalidSemanticAnnotationException.class)
public void testNonForwardedDualInvalidTypes2() {

	String[] nonForwardedFieldsSecond = { "f1" };
	DualInputSemanticProperties dsp = new DualInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsDualFromString(dsp, null, null,
			null, nonForwardedFieldsSecond, null, null, threeIntTupleType, pojoInTupleType, threeIntTupleType);
}

Example #27

Source File: SemanticPropUtilTest.java From flink with Apache License 2.0

5 votes

@Test(expected = InvalidSemanticAnnotationException.class)
public void testForwardedNonForwardedFirstCheck() {
	String[] forwarded = { "1" };
	String[] nonForwarded = { "1" };
	SemanticPropUtil.getSemanticPropsDualFromString(new DualInputSemanticProperties(), forwarded, null, nonForwarded, null, null, null,
			threeIntTupleType, threeIntTupleType, threeIntTupleType);
}

Example #28

Source File: SemanticPropUtilTest.java From flink with Apache License 2.0

5 votes

@Test(expected = InvalidSemanticAnnotationException.class)
public void testForwardedNonForwardedSecondCheck() {
	String[] forwarded = { "1" };
	String[] nonForwarded = { "1" };
	SemanticPropUtil.getSemanticPropsDualFromString(new DualInputSemanticProperties(), null, forwarded, null, nonForwarded, null, null,
			threeIntTupleType, threeIntTupleType, threeIntTupleType);
}

Example #29

Source File: SemanticPropertiesProjectionTest.java From flink with Apache License 2.0

5 votes

@Test
public void testJoinProjectionSemProps1() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	DataSet<Tuple5<Integer, Long, String, Long, Integer>> tupleDs = env.fromCollection(emptyTupleData, tupleTypeInfo);

	tupleDs.join(tupleDs).where(0).equalTo(0)
			.projectFirst(2, 3)
			.projectSecond(1, 4)
			.output(new DiscardingOutputFormat<Tuple>());

	Plan plan = env.createProgramPlan();

	GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
	InnerJoinOperatorBase<?, ?, ?, ?> projectJoinOperator = ((InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput());

	DualInputSemanticProperties props = projectJoinOperator.getSemanticProperties();

	assertEquals(1, props.getForwardingTargetFields(0, 2).size());
	assertEquals(1, props.getForwardingTargetFields(0, 3).size());
	assertEquals(1, props.getForwardingTargetFields(1, 1).size());
	assertEquals(1, props.getForwardingTargetFields(1, 4).size());

	assertTrue(props.getForwardingTargetFields(0, 2).contains(0));
	assertTrue(props.getForwardingTargetFields(0, 3).contains(1));
	assertTrue(props.getForwardingTargetFields(1, 1).contains(2));
	assertTrue(props.getForwardingTargetFields(1, 4).contains(3));
}

Example #30

Source File: RequestedGlobalPropertiesFilteringTest.java From flink with Apache License 2.0

5 votes

@Test
public void testDualHashPartitioningPreserved() {

	DualInputSemanticProperties dprops = new DualInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsDualFromString(dprops, new String[]{"0;2;4"}, new String[]{"1->3;4->6;3->7"},
			null, null, null, null, tupleInfo, tupleInfo, tupleInfo);

	RequestedGlobalProperties gprops1 = new RequestedGlobalProperties();
	RequestedGlobalProperties gprops2 = new RequestedGlobalProperties();
	gprops1.setHashPartitioned(new FieldSet(2, 0, 4));
	gprops2.setHashPartitioned(new FieldSet(3, 6, 7));
	RequestedGlobalProperties filtered1 = gprops1.filterBySemanticProperties(dprops, 0);
	RequestedGlobalProperties filtered2 = gprops2.filterBySemanticProperties(dprops, 1);

	assertNotNull(filtered1);
	assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered1.getPartitioning());
	assertNotNull(filtered1.getPartitionedFields());
	assertEquals(3, filtered1.getPartitionedFields().size());
	assertTrue(filtered1.getPartitionedFields().contains(0));
	assertTrue(filtered1.getPartitionedFields().contains(2));
	assertTrue(filtered1.getPartitionedFields().contains(4));
	assertNull(filtered1.getOrdering());
	assertNull(filtered1.getCustomPartitioner());
	assertNull(filtered1.getDataDistribution());

	assertNotNull(filtered2);
	assertEquals(PartitioningProperty.HASH_PARTITIONED, filtered2.getPartitioning());
	assertNotNull(filtered2.getPartitionedFields());
	assertEquals(3, filtered2.getPartitionedFields().size());
	assertTrue(filtered2.getPartitionedFields().contains(1));
	assertTrue(filtered2.getPartitionedFields().contains(3));
	assertTrue(filtered2.getPartitionedFields().contains(4));
	assertNull(filtered2.getOrdering());
	assertNull(filtered2.getCustomPartitioner());
	assertNull(filtered2.getDataDistribution());
}