Java Code Examples for org.apache.flink.api.common.operators.Ordering#appendOrdering()

The following examples show how to use org.apache.flink.api.common.operators.Ordering#appendOrdering() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LocalPropertiesFilteringTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testSortingPreserved4() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"2->7;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(1, gFields.size());
	assertTrue(gFields.contains(7));
	assertNotNull(order);
	assertEquals(1, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(IntValue.class, order.getType(0));
	assertNull(filtered.getUniqueFields());
}
Example 2
Source File: RequestedGlobalPropertiesFilteringTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRangePartitioningErased() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;2"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(3, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRangePartitioned(o);

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNull(filtered);
}
Example 3
Source File: PartitionOperator.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private static <T> Ordering computeOrdering(Keys<T> pKeys, Order[] orders) {
	Ordering ordering = new Ordering();
	final int[] logicalKeyPositions = pKeys.computeLogicalKeyPositions();

	if (orders == null) {
		for (int key : logicalKeyPositions) {
			ordering.appendOrdering(key, null, Order.ASCENDING);
		}
	} else {
		final TypeInformation<?>[] originalKeyFieldTypes = pKeys.getOriginalKeyFieldTypes();
		int index = 0;
		for (int i = 0; i < originalKeyFieldTypes.length; i++) {
			final int typeTotalFields = originalKeyFieldTypes[i].getTotalFields();
			for (int j = index; j < index + typeTotalFields; j++) {
				ordering.appendOrdering(logicalKeyPositions[j], null, orders[i]);
			}
			index += typeTotalFields;
		}
	}

	return ordering;
}
Example 4
Source File: RequestedLocalPropertiesFilteringTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testOrderErased() {

	SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"1; 4"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(4, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedLocalProperties rlProp = new RequestedLocalProperties();
	rlProp.setOrdering(o);

	RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);

	assertNull(filtered);
}
Example 5
Source File: GlobalPropertiesFilteringTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testRangePartitioningErased() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	o.appendOrdering(2, StringValue.class, Order.ASCENDING);
	GlobalProperties gprops = new GlobalProperties();
	gprops.setRangePartitioned(o);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.RANDOM_PARTITIONED, result.getPartitioning());
	assertNull(result.getPartitioningOrdering());
	assertNull(result.getPartitioningFields());
}
Example 6
Source File: PartitionOperator.java    From flink with Apache License 2.0 6 votes vote down vote up 
private static <T> Ordering computeOrdering(Keys<T> pKeys, Order[] orders) {
	Ordering ordering = new Ordering();
	final int[] logicalKeyPositions = pKeys.computeLogicalKeyPositions();

	if (orders == null) {
		for (int key : logicalKeyPositions) {
			ordering.appendOrdering(key, null, Order.ASCENDING);
		}
	} else {
		final TypeInformation<?>[] originalKeyFieldTypes = pKeys.getOriginalKeyFieldTypes();
		int index = 0;
		for (int i = 0; i < originalKeyFieldTypes.length; i++) {
			final int typeTotalFields = originalKeyFieldTypes[i].getTotalFields();
			for (int j = index; j < index + typeTotalFields; j++) {
				ordering.appendOrdering(logicalKeyPositions[j], null, orders[i]);
			}
			index += typeTotalFields;
		}
	}

	return ordering;
}
Example 7
Source File: RequestedLocalPropertiesFilteringTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testOrderErased() {

	SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"1; 4"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(4, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedLocalProperties rlProp = new RequestedLocalProperties();
	rlProp.setOrdering(o);

	RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);

	assertNull(filtered);
}
Example 8
Source File: RequestedGlobalPropertiesFilteringTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRangePartitioningPreserved2() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"7->3;1->1;2->6"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(3, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRangePartitioned(o);

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
	assertNotNull(filtered.getOrdering());
	assertEquals(3, filtered.getOrdering().getNumberOfFields());
	assertEquals(7, filtered.getOrdering().getFieldNumber(0).intValue());
	assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
	assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
	assertEquals(LongValue.class, filtered.getOrdering().getType(0));
	assertEquals(IntValue.class, filtered.getOrdering().getType(1));
	assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
	assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
	assertNull(filtered.getPartitionedFields());
	assertNull(filtered.getDataDistribution());
	assertNull(filtered.getCustomPartitioner());
}
Example 9
Source File: DataSink.java    From flink with Apache License 2.0 5 votes vote down vote up 
protected GenericDataSinkBase<T> translateToDataFlow(Operator<T> input) {
	// select the name (or create a default one)
	String name = this.name != null ? this.name : this.format.toString();
	GenericDataSinkBase<T> sink = new GenericDataSinkBase<>(this.format, new UnaryOperatorInformation<>(this.type, new NothingTypeInfo()), name);
	// set input
	sink.setInput(input);
	// set parameters
	if (this.parameters != null) {
		sink.getParameters().addAll(this.parameters);
	}
	// set parallelism
	if (this.parallelism > 0) {
		// use specified parallelism
		sink.setParallelism(this.parallelism);
	} else {
		// if no parallelism has been specified, use parallelism of input operator to enable chaining
		sink.setParallelism(input.getParallelism());
	}

	if (this.sortKeyPositions != null) {
		// configure output sorting
		Ordering ordering = new Ordering();
		for (int i = 0; i < this.sortKeyPositions.length; i++) {
			ordering.appendOrdering(this.sortKeyPositions[i], null, this.sortOrders[i]);
		}
		sink.setLocalOrder(ordering);
	}

	return sink;
}
Example 10
Source File: GlobalPropertiesFilteringTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRangePartitioningPreserved2() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1->3; 2->0; 5->1"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	o.appendOrdering(2, StringValue.class, Order.ASCENDING);
	GlobalProperties gprops = new GlobalProperties();
	gprops.setRangePartitioned(o);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.RANGE_PARTITIONED, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(3, pFields.size());
	assertEquals(3, pFields.get(0).intValue());
	assertEquals(1, pFields.get(1).intValue());
	assertEquals(0, pFields.get(2).intValue());
	Ordering pOrder = result.getPartitioningOrdering();
	assertEquals(3, pOrder.getNumberOfFields());
	assertEquals(3, pOrder.getFieldNumber(0).intValue());
	assertEquals(1, pOrder.getFieldNumber(1).intValue());
	assertEquals(0, pOrder.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, pOrder.getOrder(0));
	assertEquals(Order.DESCENDING, pOrder.getOrder(1));
	assertEquals(Order.ASCENDING, pOrder.getOrder(2));
	assertEquals(IntValue.class, pOrder.getType(0));
	assertEquals(LongValue.class, pOrder.getType(1));
	assertEquals(StringValue.class, pOrder.getType(2));
}
Example 11
Source File: Utils.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static Ordering createOrdering(FieldList fields) {
	final Ordering o = new Ordering();
	for (int i = 0; i < fields.size(); i++) {
		o.appendOrdering(fields.get(i), null, Order.ANY);
	}
	return o;
}
Example 12
Source File: RequestedLocalPropertiesFilteringTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testOrderPreserved1() {

	SingleInputSemanticProperties sProps = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProps, new String[]{"1;4;6"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(4, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedLocalProperties rlProp = new RequestedLocalProperties();
	rlProp.setOrdering(o);

	RequestedLocalProperties filtered = rlProp.filterBySemanticProperties(sProps, 0);

	assertNotNull(filtered);
	assertNotNull(filtered.getOrdering());
	assertEquals(3, filtered.getOrdering().getNumberOfFields());
	assertEquals(4, filtered.getOrdering().getFieldNumber(0).intValue());
	assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
	assertEquals(6, filtered.getOrdering().getFieldNumber(2).intValue());
	assertEquals(LongValue.class, filtered.getOrdering().getType(0));
	assertEquals(IntValue.class, filtered.getOrdering().getType(1));
	assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
	assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
	assertNull(filtered.getGroupedFields());
}
Example 13
Source File: SortPartitionOperator.java    From flink with Apache License 2.0 5 votes vote down vote up 
private <K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlowWithKeyExtractor(
	Operator<T> input, Keys.SelectorFunctionKeys<T, K> keys, Order order, String name) {
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Keys.ExpressionKeys<Tuple2<K, T>> newKey = new Keys.ExpressionKeys<>(0, typeInfoWithKey);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	int[] sortKeyPositions = newKey.computeLogicalKeyPositions();
	Ordering partitionOrdering = new Ordering();
	for (int keyPosition : sortKeyPositions) {
		partitionOrdering.appendOrdering(keyPosition, null, order);
	}

	// distinguish between partition types
	UnaryOperatorInformation<Tuple2<K, T>, Tuple2<K, T>> operatorInfo = new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey);
	SortPartitionOperatorBase<Tuple2<K, T>> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
	noop.setInput(keyedInput);
	if (this.getParallelism() < 0) {
		// use parallelism of input if not explicitly specified
		noop.setParallelism(input.getParallelism());
	} else {
		// use explicitly specified parallelism
		noop.setParallelism(this.getParallelism());
	}

	return KeyFunctions.appendKeyRemover(noop, keys);
}
Example 14
Source File: RequestedGlobalPropertiesFilteringTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRangePartitioningPreserved2() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"7->3;1->1;2->6"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(3, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRangePartitioned(o);

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
	assertNotNull(filtered.getOrdering());
	assertEquals(3, filtered.getOrdering().getNumberOfFields());
	assertEquals(7, filtered.getOrdering().getFieldNumber(0).intValue());
	assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
	assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
	assertEquals(LongValue.class, filtered.getOrdering().getType(0));
	assertEquals(IntValue.class, filtered.getOrdering().getType(1));
	assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
	assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
	assertNull(filtered.getPartitionedFields());
	assertNull(filtered.getDataDistribution());
	assertNull(filtered.getCustomPartitioner());
}
Example 15
Source File: RequestedGlobalPropertiesFilteringTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRangePartitioningPreserved1() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"1;3;6"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(3, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRangePartitioned(o);

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
	assertNotNull(filtered.getOrdering());
	assertEquals(3, filtered.getOrdering().getNumberOfFields());
	assertEquals(3, filtered.getOrdering().getFieldNumber(0).intValue());
	assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
	assertEquals(6, filtered.getOrdering().getFieldNumber(2).intValue());
	assertEquals(LongValue.class, filtered.getOrdering().getType(0));
	assertEquals(IntValue.class, filtered.getOrdering().getType(1));
	assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
	assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
	assertNull(filtered.getPartitionedFields());
	assertNull(filtered.getDataDistribution());
	assertNull(filtered.getCustomPartitioner());
}
Example 16
Source File: LocalPropertiesFilteringTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSortingPreserved2() {
	SingleInputSemanticProperties sp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sp, new String[]{"0->3;2->7;5->1"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(2, IntValue.class, Order.ASCENDING);
	o.appendOrdering(0, StringValue.class, Order.DESCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	LocalProperties lProps = LocalProperties.forOrdering(o);

	LocalProperties filtered = lProps.filterBySemanticProperties(sp, 0);
	FieldList gFields = filtered.getGroupedFields();
	Ordering order = filtered.getOrdering();

	assertNotNull(gFields);
	assertEquals(3, gFields.size());
	assertTrue(gFields.contains(3));
	assertTrue(gFields.contains(7));
	assertTrue(gFields.contains(1));
	assertNotNull(order);
	assertEquals(3, order.getNumberOfFields());
	assertEquals(7, order.getFieldNumber(0).intValue());
	assertEquals(3, order.getFieldNumber(1).intValue());
	assertEquals(1, order.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, order.getOrder(0));
	assertEquals(Order.DESCENDING, order.getOrder(1));
	assertEquals(Order.DESCENDING, order.getOrder(2));
	assertEquals(IntValue.class, order.getType(0));
	assertEquals(StringValue.class, order.getType(1));
	assertEquals(LongValue.class, order.getType(2));
	assertNull(filtered.getUniqueFields());
}
Example 17
Source File: GlobalPropertiesFilteringTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRangePartitioningPreserved1() {

	SingleInputSemanticProperties sprops = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sprops, new String[]{"1;2;5"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(5, LongValue.class, Order.DESCENDING);
	o.appendOrdering(2, StringValue.class, Order.ASCENDING);
	GlobalProperties gprops = new GlobalProperties();
	gprops.setRangePartitioned(o);

	GlobalProperties result = gprops.filterBySemanticProperties(sprops, 0);

	assertEquals(PartitioningProperty.RANGE_PARTITIONED, result.getPartitioning());
	FieldList pFields = result.getPartitioningFields();
	assertEquals(3, pFields.size());
	assertEquals(1, pFields.get(0).intValue());
	assertEquals(5, pFields.get(1).intValue());
	assertEquals(2, pFields.get(2).intValue());
	Ordering pOrder = result.getPartitioningOrdering();
	assertEquals(3, pOrder.getNumberOfFields());
	assertEquals(1, pOrder.getFieldNumber(0).intValue());
	assertEquals(5, pOrder.getFieldNumber(1).intValue());
	assertEquals(2, pOrder.getFieldNumber(2).intValue());
	assertEquals(Order.ASCENDING, pOrder.getOrder(0));
	assertEquals(Order.DESCENDING, pOrder.getOrder(1));
	assertEquals(Order.ASCENDING, pOrder.getOrder(2));
	assertEquals(IntValue.class, pOrder.getType(0));
	assertEquals(LongValue.class, pOrder.getType(1));
	assertEquals(StringValue.class, pOrder.getType(2));
}
Example 18
Source File: RequestedGlobalPropertiesFilteringTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testRangePartitioningPreserved2() {

	SingleInputSemanticProperties sProp = new SingleInputSemanticProperties();
	SemanticPropUtil.getSemanticPropsSingleFromString(sProp, new String[]{"7->3;1->1;2->6"}, null, null, tupleInfo, tupleInfo);

	Ordering o = new Ordering();
	o.appendOrdering(3, LongValue.class, Order.DESCENDING);
	o.appendOrdering(1, IntValue.class, Order.ASCENDING);
	o.appendOrdering(6, ByteValue.class, Order.DESCENDING);

	RequestedGlobalProperties rgProps = new RequestedGlobalProperties();
	rgProps.setRangePartitioned(o);

	RequestedGlobalProperties filtered = rgProps.filterBySemanticProperties(sProp, 0);

	assertNotNull(filtered);
	assertEquals(PartitioningProperty.RANGE_PARTITIONED, filtered.getPartitioning());
	assertNotNull(filtered.getOrdering());
	assertEquals(3, filtered.getOrdering().getNumberOfFields());
	assertEquals(7, filtered.getOrdering().getFieldNumber(0).intValue());
	assertEquals(1, filtered.getOrdering().getFieldNumber(1).intValue());
	assertEquals(2, filtered.getOrdering().getFieldNumber(2).intValue());
	assertEquals(LongValue.class, filtered.getOrdering().getType(0));
	assertEquals(IntValue.class, filtered.getOrdering().getType(1));
	assertEquals(ByteValue.class, filtered.getOrdering().getType(2));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(0));
	assertEquals(Order.ASCENDING, filtered.getOrdering().getOrder(1));
	assertEquals(Order.DESCENDING, filtered.getOrdering().getOrder(2));
	assertNull(filtered.getPartitionedFields());
	assertNull(filtered.getDataDistribution());
	assertNull(filtered.getCustomPartitioner());
}
Example 19
Source File: SortPartitionOperator.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlow(Operator<T> input) {

		String name = "Sort at " + sortLocationName;

		if (useKeySelector) {
			return translateToDataFlowWithKeyExtractor(input, (Keys.SelectorFunctionKeys<T, ?>) keys.get(0), orders.get(0), name);
		}

		// flatten sort key positions
		List<Integer> allKeyPositions = new ArrayList<>();
		List<Order> allOrders = new ArrayList<>();
		for (int i = 0, length = keys.size(); i < length; i++) {
			int[] sortKeyPositions = keys.get(i).computeLogicalKeyPositions();
			Order order = orders.get(i);

			for (int sortKeyPosition : sortKeyPositions) {
				allKeyPositions.add(sortKeyPosition);
				allOrders.add(order);
			}
		}

		Ordering partitionOrdering = new Ordering();
		for (int i = 0, length = allKeyPositions.size(); i < length; i++) {
			partitionOrdering.appendOrdering(allKeyPositions.get(i), null, allOrders.get(i));
		}

		// distinguish between partition types
		UnaryOperatorInformation<T, T> operatorInfo = new UnaryOperatorInformation<>(getType(), getType());
		SortPartitionOperatorBase<T> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
		noop.setInput(input);
		if (this.getParallelism() < 0) {
			// use parallelism of input if not explicitly specified
			noop.setParallelism(input.getParallelism());
		} else {
			// use explicitly specified parallelism
			noop.setParallelism(this.getParallelism());
		}

		return noop;

	}
Example 20
Source File: SortPartitionOperator.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateToDataFlow(Operator<T> input) {

		String name = "Sort at " + sortLocationName;

		if (useKeySelector) {
			return translateToDataFlowWithKeyExtractor(input, (Keys.SelectorFunctionKeys<T, ?>) keys.get(0), orders.get(0), name);
		}

		// flatten sort key positions
		List<Integer> allKeyPositions = new ArrayList<>();
		List<Order> allOrders = new ArrayList<>();
		for (int i = 0, length = keys.size(); i < length; i++) {
			int[] sortKeyPositions = keys.get(i).computeLogicalKeyPositions();
			Order order = orders.get(i);

			for (int sortKeyPosition : sortKeyPositions) {
				allKeyPositions.add(sortKeyPosition);
				allOrders.add(order);
			}
		}

		Ordering partitionOrdering = new Ordering();
		for (int i = 0, length = allKeyPositions.size(); i < length; i++) {
			partitionOrdering.appendOrdering(allKeyPositions.get(i), null, allOrders.get(i));
		}

		// distinguish between partition types
		UnaryOperatorInformation<T, T> operatorInfo = new UnaryOperatorInformation<>(getType(), getType());
		SortPartitionOperatorBase<T> noop = new SortPartitionOperatorBase<>(operatorInfo, partitionOrdering, name);
		noop.setInput(input);
		if (this.getParallelism() < 0) {
			// use parallelism of input if not explicitly specified
			noop.setParallelism(input.getParallelism());
		} else {
			// use explicitly specified parallelism
			noop.setParallelism(this.getParallelism());
		}

		return noop;

	}