org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys Java Examples

The following examples show how to use org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: GroupCombineOperator.java From flink with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
private static <IN, OUT, K1, K2> PlanUnwrappingSortedGroupCombineOperator<IN, OUT, K1, K2> translateSelectorFunctionSortedReducer(
		SelectorFunctionKeys<IN, ?> rawGroupingKey,
		SelectorFunctionKeys<IN, ?> rawSortingKeys,
		Ordering groupOrder,
		GroupCombineFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input) {
	final SelectorFunctionKeys<IN, K1> groupingKey = (SelectorFunctionKeys<IN, K1>) rawGroupingKey;
	final SelectorFunctionKeys<IN, K2> sortingKey = (SelectorFunctionKeys<IN, K2>) rawSortingKeys;
	TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(groupingKey, sortingKey);

	Operator<Tuple3<K1, K2, IN>> inputWithKey = KeyFunctions.appendKeyExtractor(input, groupingKey, sortingKey);

	PlanUnwrappingSortedGroupCombineOperator<IN, OUT, K1, K2> reducer =
		new PlanUnwrappingSortedGroupCombineOperator<>(function, groupingKey, sortingKey, name, outputType, typeInfoWithKey);
	reducer.setInput(inputWithKey);
	reducer.setGroupOrder(groupOrder);

	return reducer;
}

Example #2

Source File: ReduceOperator.java From Flink-CEPplus with Apache License 2.0

6 votes

private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionReducer(
	SelectorFunctionKeys<T, ?> rawKeys,
	ReduceFunction<T> function,
	TypeInformation<T> inputType,
	String name,
	Operator<T> input,
	int parallelism,
	CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;

	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<T, K> reducer = new PlanUnwrappingReduceOperator<>(function, keys, name, inputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setParallelism(parallelism);
	reducer.setCombineHint(hint);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}

Example #3

Source File: GroupReduceOperator.java From flink with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionReducer(
		SelectorFunctionKeys<IN, ?> rawKeys,
		GroupReduceFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input,
		boolean combinable) {
	SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;
	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer =
		new PlanUnwrappingReduceGroupOperator(function, keys, name, outputType, typeInfoWithKey, combinable);
	reducer.setInput(keyedInput);

	return reducer;
}

Example #4

Source File: DistinctOperator.java From Flink-CEPplus with Apache License 2.0

6 votes

private static <IN, K> org.apache.flink.api.common.operators.SingleInputOperator<?, IN, ?> translateSelectorFunctionDistinct(
		SelectorFunctionKeys<IN, ?> rawKeys,
		ReduceFunction<IN> function,
		TypeInformation<IN> outputType,
		String name,
		Operator<IN> input,
		int parallelism,
		CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;

	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<IN, K> reducer =
			new PlanUnwrappingReduceOperator<>(function, keys, name, outputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setCombineHint(hint);
	reducer.setParallelism(parallelism);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}

Example #5

Source File: GroupReduceOperator.java From flink with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
private static <IN, OUT, K1, K2> PlanUnwrappingSortedReduceGroupOperator<IN, OUT, K1, K2> translateSelectorFunctionSortedReducer(
	SelectorFunctionKeys<IN, ?> rawGroupingKey,
	SelectorFunctionKeys<IN, ?> rawSortingKey,
	Ordering groupOrdering,
	GroupReduceFunction<IN, OUT> function,
	TypeInformation<OUT> outputType,
	String name,
	Operator<IN> input,
	boolean combinable) {
	final SelectorFunctionKeys<IN, K1> groupingKey = (SelectorFunctionKeys<IN, K1>) rawGroupingKey;
	final SelectorFunctionKeys<IN, K2> sortingKey = (SelectorFunctionKeys<IN, K2>) rawSortingKey;
	TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(groupingKey, sortingKey);

	Operator<Tuple3<K1, K2, IN>> inputWithKey = KeyFunctions.appendKeyExtractor(input, groupingKey, sortingKey);

	PlanUnwrappingSortedReduceGroupOperator<IN, OUT, K1, K2> reducer =
		new PlanUnwrappingSortedReduceGroupOperator<>(
			function, groupingKey, sortingKey, name, outputType, typeInfoWithKey, combinable);
	reducer.setInput(inputWithKey);
	reducer.setGroupOrder(groupOrdering);

	return reducer;
}

Example #6

Source File: JoinOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			(this.keys1 instanceof SelectorFunctionKeys ||
					this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}

Example #7

Source File: CoGroupOperator.java From flink with Apache License 2.0

6 votes

@Override
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
				(this.keys1 instanceof SelectorFunctionKeys ||
				this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}

Example #8

Source File: CoGroupOperator.java From flink with Apache License 2.0

6 votes

private static <I1, I2, K, OUT> PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> translateSelectorFunctionCoGroup(
		SelectorFunctionKeys<I1, ?> rawKeys1, SelectorFunctionKeys<I2, ?> rawKeys2,
		CoGroupFunction<I1, I2, OUT> function,
		TypeInformation<OUT> outputType, String name,
		Operator<I1> input1, Operator<I2> input2) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I1, K> keys1 = (SelectorFunctionKeys<I1, K>) rawKeys1;
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I2, K> keys2 = (SelectorFunctionKeys<I2, K>) rawKeys2;

	final TypeInformation<Tuple2<K, I1>> typeInfoWithKey1 = KeyFunctions.createTypeWithKey(keys1);
	final TypeInformation<Tuple2<K, I2>> typeInfoWithKey2 = KeyFunctions.createTypeWithKey(keys2);

	final Operator<Tuple2<K, I1>> keyedInput1 = KeyFunctions.appendKeyExtractor(input1, keys1);
	final Operator<Tuple2<K, I2>> keyedInput2 = KeyFunctions.appendKeyExtractor(input2, keys2);

	final PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> cogroup =
		new PlanBothUnwrappingCoGroupOperator<>(function, keys1, keys2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);

	cogroup.setFirstInput(keyedInput1);
	cogroup.setSecondInput(keyedInput2);

	return cogroup;
}

Example #9

Source File: ReduceOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}
		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}

Example #10

Source File: ReduceOperator.java From flink with Apache License 2.0

6 votes

private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionReducer(
	SelectorFunctionKeys<T, ?> rawKeys,
	ReduceFunction<T> function,
	TypeInformation<T> inputType,
	String name,
	Operator<T> input,
	int parallelism,
	CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;

	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<T, K> reducer = new PlanUnwrappingReduceOperator<>(function, keys, name, inputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setParallelism(parallelism);
	reducer.setCombineHint(hint);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}

Example #11

Source File: DistinctOperator.java From flink with Apache License 2.0

6 votes

private static <IN, K> org.apache.flink.api.common.operators.SingleInputOperator<?, IN, ?> translateSelectorFunctionDistinct(
		SelectorFunctionKeys<IN, ?> rawKeys,
		ReduceFunction<IN> function,
		TypeInformation<IN> outputType,
		String name,
		Operator<IN> input,
		int parallelism,
		CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;

	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<IN, K> reducer =
			new PlanUnwrappingReduceOperator<>(function, keys, name, outputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setCombineHint(hint);
	reducer.setParallelism(parallelism);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}

Example #12

Source File: PartitionOperator.java From flink with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner(
	SelectorFunctionKeys<T, ?> rawKeys,
	PartitionMethod pMethod,
	String name,
	Operator<T> input,
	int partitionDop,
	Partitioner<?> customPartitioner,
	Order[] orders) {
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput =
		new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[]{0}, name);
	keyedPartitionedInput.setInput(keyedInput);
	keyedPartitionedInput.setCustomPartitioner(customPartitioner);
	keyedPartitionedInput.setParallelism(partitionDop);
	keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING));

	return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys);
}

Example #13

Source File: KeyFunctions.java From flink with Apache License 2.0

6 votes

public static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> appendKeyRemover(
		org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> inputWithKey,
		SelectorFunctionKeys<T, K> key) {

	TypeInformation<T> inputType = key.getInputType();
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = createTypeWithKey(key);

	MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>> mapper =
			new MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>>(
					new KeyRemovingMapper<T, K>(),
					new UnaryOperatorInformation<>(typeInfoWithKey, inputType),
					"Key Remover"
			);
	mapper.setInput(inputWithKey);
	mapper.setParallelism(inputWithKey.getParallelism());

	return mapper;
}

Example #14

Source File: GroupCombineOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}

		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}

Example #15

Source File: GroupCombineOperator.java From flink with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
private static <IN, OUT, K> PlanUnwrappingGroupCombineOperator<IN, OUT, K> translateSelectorFunctionReducer(
		SelectorFunctionKeys<IN, ?> rawKeys,
		GroupCombineFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input) {
	final SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;

	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingGroupCombineOperator<IN, OUT, K> reducer =
		new PlanUnwrappingGroupCombineOperator<>(function, keys, name, outputType, typeInfoWithKey);
	reducer.setInput(keyedInput);

	return reducer;
}

Example #16

Source File: GroupReduceOperator.java From Flink-CEPplus with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionReducer(
		SelectorFunctionKeys<IN, ?> rawKeys,
		GroupReduceFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input,
		boolean combinable) {
	SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;
	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer =
		new PlanUnwrappingReduceGroupOperator(function, keys, name, outputType, typeInfoWithKey, combinable);
	reducer.setInput(keyedInput);

	return reducer;
}

Example #17

Source File: GroupReduceOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}
		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}

Example #18

Source File: GroupReduceOperator.java From flink with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
private static <IN, OUT, K> PlanUnwrappingReduceGroupOperator<IN, OUT, K> translateSelectorFunctionReducer(
		SelectorFunctionKeys<IN, ?> rawKeys,
		GroupReduceFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input,
		boolean combinable) {
	SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;
	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceGroupOperator<IN, OUT, K> reducer =
		new PlanUnwrappingReduceGroupOperator(function, keys, name, outputType, typeInfoWithKey, combinable);
	reducer.setInput(keyedInput);

	return reducer;
}

Example #19

Source File: GroupReduceOperator.java From flink with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
private static <IN, OUT, K1, K2> PlanUnwrappingSortedReduceGroupOperator<IN, OUT, K1, K2> translateSelectorFunctionSortedReducer(
	SelectorFunctionKeys<IN, ?> rawGroupingKey,
	SelectorFunctionKeys<IN, ?> rawSortingKey,
	Ordering groupOrdering,
	GroupReduceFunction<IN, OUT> function,
	TypeInformation<OUT> outputType,
	String name,
	Operator<IN> input,
	boolean combinable) {
	final SelectorFunctionKeys<IN, K1> groupingKey = (SelectorFunctionKeys<IN, K1>) rawGroupingKey;
	final SelectorFunctionKeys<IN, K2> sortingKey = (SelectorFunctionKeys<IN, K2>) rawSortingKey;
	TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(groupingKey, sortingKey);

	Operator<Tuple3<K1, K2, IN>> inputWithKey = KeyFunctions.appendKeyExtractor(input, groupingKey, sortingKey);

	PlanUnwrappingSortedReduceGroupOperator<IN, OUT, K1, K2> reducer =
		new PlanUnwrappingSortedReduceGroupOperator<>(
			function, groupingKey, sortingKey, name, outputType, typeInfoWithKey, combinable);
	reducer.setInput(inputWithKey);
	reducer.setGroupOrder(groupOrdering);

	return reducer;
}

Example #20

Source File: JoinOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			(this.keys1 instanceof SelectorFunctionKeys ||
					this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}

Example #21

Source File: CoGroupOperator.java From flink with Apache License 2.0

6 votes

@Override
public DualInputSemanticProperties getSemanticProperties() {

	DualInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
				(this.keys1 instanceof SelectorFunctionKeys ||
				this.keys2 instanceof SelectorFunctionKeys)) {

		int numFields1 = this.getInput1Type().getTotalFields();
		int numFields2 = this.getInput2Type().getTotalFields();
		int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
		int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ?
				((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;

		props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
	}

	return props;
}

Example #22

Source File: CoGroupOperator.java From flink with Apache License 2.0

6 votes

private static <I1, I2, K, OUT> PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> translateSelectorFunctionCoGroup(
		SelectorFunctionKeys<I1, ?> rawKeys1, SelectorFunctionKeys<I2, ?> rawKeys2,
		CoGroupFunction<I1, I2, OUT> function,
		TypeInformation<OUT> outputType, String name,
		Operator<I1> input1, Operator<I2> input2) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I1, K> keys1 = (SelectorFunctionKeys<I1, K>) rawKeys1;
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I2, K> keys2 = (SelectorFunctionKeys<I2, K>) rawKeys2;

	final TypeInformation<Tuple2<K, I1>> typeInfoWithKey1 = KeyFunctions.createTypeWithKey(keys1);
	final TypeInformation<Tuple2<K, I2>> typeInfoWithKey2 = KeyFunctions.createTypeWithKey(keys2);

	final Operator<Tuple2<K, I1>> keyedInput1 = KeyFunctions.appendKeyExtractor(input1, keys1);
	final Operator<Tuple2<K, I2>> keyedInput2 = KeyFunctions.appendKeyExtractor(input2, keys2);

	final PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> cogroup =
		new PlanBothUnwrappingCoGroupOperator<>(function, keys1, keys2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);

	cogroup.setFirstInput(keyedInput1);
	cogroup.setSecondInput(keyedInput2);

	return cogroup;
}

Example #23

Source File: ReduceOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}
		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}

Example #24

Source File: ReduceOperator.java From flink with Apache License 2.0

6 votes

private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionReducer(
	SelectorFunctionKeys<T, ?> rawKeys,
	ReduceFunction<T> function,
	TypeInformation<T> inputType,
	String name,
	Operator<T> input,
	int parallelism,
	CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;

	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<T, K> reducer = new PlanUnwrappingReduceOperator<>(function, keys, name, inputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setParallelism(parallelism);
	reducer.setCombineHint(hint);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}

Example #25

Source File: DistinctOperator.java From flink with Apache License 2.0

6 votes

private static <IN, K> org.apache.flink.api.common.operators.SingleInputOperator<?, IN, ?> translateSelectorFunctionDistinct(
		SelectorFunctionKeys<IN, ?> rawKeys,
		ReduceFunction<IN> function,
		TypeInformation<IN> outputType,
		String name,
		Operator<IN> input,
		int parallelism,
		CombineHint hint) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;

	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingReduceOperator<IN, K> reducer =
			new PlanUnwrappingReduceOperator<>(function, keys, name, outputType, typeInfoWithKey);
	reducer.setInput(keyedInput);
	reducer.setCombineHint(hint);
	reducer.setParallelism(parallelism);

	return KeyFunctions.appendKeyRemover(reducer, keys);
}

Example #26

Source File: PartitionOperator.java From flink with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
private static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> translateSelectorFunctionPartitioner(
	SelectorFunctionKeys<T, ?> rawKeys,
	PartitionMethod pMethod,
	String name,
	Operator<T> input,
	int partitionDop,
	Partitioner<?> customPartitioner,
	Order[] orders) {
	final SelectorFunctionKeys<T, K> keys = (SelectorFunctionKeys<T, K>) rawKeys;
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);

	Operator<Tuple2<K, T>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PartitionOperatorBase<Tuple2<K, T>> keyedPartitionedInput =
		new PartitionOperatorBase<>(new UnaryOperatorInformation<>(typeInfoWithKey, typeInfoWithKey), pMethod, new int[]{0}, name);
	keyedPartitionedInput.setInput(keyedInput);
	keyedPartitionedInput.setCustomPartitioner(customPartitioner);
	keyedPartitionedInput.setParallelism(partitionDop);
	keyedPartitionedInput.setOrdering(new Ordering(0, null, orders != null ? orders[0] : Order.ASCENDING));

	return KeyFunctions.appendKeyRemover(keyedPartitionedInput, keys);
}

Example #27

Source File: KeyFunctions.java From flink with Apache License 2.0

6 votes

public static <T, K> org.apache.flink.api.common.operators.SingleInputOperator<?, T, ?> appendKeyRemover(
		org.apache.flink.api.common.operators.Operator<Tuple2<K, T>> inputWithKey,
		SelectorFunctionKeys<T, K> key) {

	TypeInformation<T> inputType = key.getInputType();
	TypeInformation<Tuple2<K, T>> typeInfoWithKey = createTypeWithKey(key);

	MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>> mapper =
			new MapOperatorBase<Tuple2<K, T>, T, MapFunction<Tuple2<K, T>, T>>(
					new KeyRemovingMapper<T, K>(),
					new UnaryOperatorInformation<>(typeInfoWithKey, inputType),
					"Key Remover"
			);
	mapper.setInput(inputWithKey);
	mapper.setParallelism(inputWithKey.getParallelism());

	return mapper;
}

Example #28

Source File: GroupCombineOperator.java From flink with Apache License 2.0

6 votes

@Override
@Internal
public SingleInputSemanticProperties getSemanticProperties() {

	SingleInputSemanticProperties props = super.getSemanticProperties();

	// offset semantic information by extracted key fields
	if (props != null &&
			this.grouper != null &&
			this.grouper.keys instanceof SelectorFunctionKeys) {

		int offset = ((SelectorFunctionKeys<?, ?>) this.grouper.keys).getKeyType().getTotalFields();
		if (this.grouper instanceof SortedGrouping) {
			offset += ((SortedGrouping<?>) this.grouper).getSortSelectionFunctionKey().getKeyType().getTotalFields();
		}

		props = SemanticPropUtil.addSourceFieldOffset(props, this.getInputType().getTotalFields(), offset);
	}

	return props;
}

Example #29

Source File: GroupCombineOperator.java From flink with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
private static <IN, OUT, K> PlanUnwrappingGroupCombineOperator<IN, OUT, K> translateSelectorFunctionReducer(
		SelectorFunctionKeys<IN, ?> rawKeys,
		GroupCombineFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input) {
	final SelectorFunctionKeys<IN, K> keys = (SelectorFunctionKeys<IN, K>) rawKeys;

	TypeInformation<Tuple2<K, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(keys);
	Operator<Tuple2<K, IN>> keyedInput = KeyFunctions.appendKeyExtractor(input, keys);

	PlanUnwrappingGroupCombineOperator<IN, OUT, K> reducer =
		new PlanUnwrappingGroupCombineOperator<>(function, keys, name, outputType, typeInfoWithKey);
	reducer.setInput(keyedInput);

	return reducer;
}

Example #30

Source File: GroupCombineOperator.java From flink with Apache License 2.0

6 votes

@SuppressWarnings("unchecked")
private static <IN, OUT, K1, K2> PlanUnwrappingSortedGroupCombineOperator<IN, OUT, K1, K2> translateSelectorFunctionSortedReducer(
		SelectorFunctionKeys<IN, ?> rawGroupingKey,
		SelectorFunctionKeys<IN, ?> rawSortingKeys,
		Ordering groupOrder,
		GroupCombineFunction<IN, OUT> function,
		TypeInformation<OUT> outputType,
		String name,
		Operator<IN> input) {
	final SelectorFunctionKeys<IN, K1> groupingKey = (SelectorFunctionKeys<IN, K1>) rawGroupingKey;
	final SelectorFunctionKeys<IN, K2> sortingKey = (SelectorFunctionKeys<IN, K2>) rawSortingKeys;
	TypeInformation<Tuple3<K1, K2, IN>> typeInfoWithKey = KeyFunctions.createTypeWithKey(groupingKey, sortingKey);

	Operator<Tuple3<K1, K2, IN>> inputWithKey = KeyFunctions.appendKeyExtractor(input, groupingKey, sortingKey);

	PlanUnwrappingSortedGroupCombineOperator<IN, OUT, K1, K2> reducer =
		new PlanUnwrappingSortedGroupCombineOperator<>(function, groupingKey, sortingKey, name, outputType, typeInfoWithKey);
	reducer.setInput(inputWithKey);
	reducer.setGroupOrder(groupOrder);

	return reducer;
}