Java Code Examples for org.apache.flink.api.common.typeinfo.AtomicType

The following examples show how to use org.apache.flink.api.common.typeinfo.AtomicType. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: OuterJoinOperatorBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparator<T> buildComparatorFor(int input, ExecutionConfig executionConfig, TypeInformation<T> typeInformation) {
	TypeComparator<T> comparator;
	if (typeInformation instanceof AtomicType) {
		comparator = ((AtomicType<T>) typeInformation).createComparator(true, executionConfig);
	} else if (typeInformation instanceof CompositeType) {
		int[] keyPositions = getKeyColumns(input);
		boolean[] orders = new boolean[keyPositions.length];
		Arrays.fill(orders, true);

		comparator = ((CompositeType<T>) typeInformation).createComparator(keyPositions, orders, 0, executionConfig);
	} else {
		throw new RuntimeException("Type information for input of type " + typeInformation.getClass()
				.getCanonicalName() + " is not supported. Could not generate a comparator.");
	}
	return comparator;
}
 
Example 2
Source Project: Flink-CEPplus   Source File: SortPartitionOperatorBase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected List<IN> executeOnCollections(List<IN> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) {

	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	int[] sortColumns = this.partitionOrdering.getFieldPositions();
	boolean[] sortOrderings = this.partitionOrdering.getFieldSortDirections();

	final TypeComparator<IN> sortComparator;
	if (inputType instanceof CompositeType) {
		sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (inputType instanceof AtomicType) {
		sortComparator = ((AtomicType) inputType).createComparator(sortOrderings[0], executionConfig);
	} else {
		throw new UnsupportedOperationException("Partition sorting does not support type "+inputType+" yet.");
	}

	Collections.sort(inputData, new Comparator<IN>() {
		@Override
		public int compare(IN o1, IN o2) {
			return sortComparator.compare(o1, o2);
		}
	});

	return inputData;
}
 
Example 3
Source Project: Flink-CEPplus   Source File: JavaApiPostPass.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
	
	TypeComparator<T> comparator;
	if (typeInfo instanceof CompositeType) {
		comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
	}
	else if (typeInfo instanceof AtomicType) {
		// handle grouping of atomic types
		comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
	}
	else {
		throw new RuntimeException("Unrecognized type: " + typeInfo);
	}

	return new RuntimeComparatorFactory<T>(comparator);
}
 
Example 4
Source Project: Flink-CEPplus   Source File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) {

		TypeComparator<T> comparator;
		if (typeInfo instanceof CompositeType) {
			comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
		}
		else if (typeInfo instanceof AtomicType) {
			// handle grouping of atomic types
			comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
		}
		else {
			throw new RuntimeException("Unrecognized type: " + typeInfo);
		}

		return new RuntimeComparatorFactory<>(comparator);
	}
 
Example 5
Source Project: flink   Source File: OuterJoinOperatorBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparator<T> buildComparatorFor(int input, ExecutionConfig executionConfig, TypeInformation<T> typeInformation) {
	TypeComparator<T> comparator;
	if (typeInformation instanceof AtomicType) {
		comparator = ((AtomicType<T>) typeInformation).createComparator(true, executionConfig);
	} else if (typeInformation instanceof CompositeType) {
		int[] keyPositions = getKeyColumns(input);
		boolean[] orders = new boolean[keyPositions.length];
		Arrays.fill(orders, true);

		comparator = ((CompositeType<T>) typeInformation).createComparator(keyPositions, orders, 0, executionConfig);
	} else {
		throw new RuntimeException("Type information for input of type " + typeInformation.getClass()
				.getCanonicalName() + " is not supported. Could not generate a comparator.");
	}
	return comparator;
}
 
Example 6
Source Project: flink   Source File: SortPartitionOperatorBase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected List<IN> executeOnCollections(List<IN> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) {

	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	int[] sortColumns = this.partitionOrdering.getFieldPositions();
	boolean[] sortOrderings = this.partitionOrdering.getFieldSortDirections();

	final TypeComparator<IN> sortComparator;
	if (inputType instanceof CompositeType) {
		sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (inputType instanceof AtomicType) {
		sortComparator = ((AtomicType) inputType).createComparator(sortOrderings[0], executionConfig);
	} else {
		throw new UnsupportedOperationException("Partition sorting does not support type "+inputType+" yet.");
	}

	Collections.sort(inputData, new Comparator<IN>() {
		@Override
		public int compare(IN o1, IN o2) {
			return sortComparator.compare(o1, o2);
		}
	});

	return inputData;
}
 
Example 7
Source Project: flink   Source File: JavaApiPostPass.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
	
	TypeComparator<T> comparator;
	if (typeInfo instanceof CompositeType) {
		comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
	}
	else if (typeInfo instanceof AtomicType) {
		// handle grouping of atomic types
		comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
	}
	else {
		throw new RuntimeException("Unrecognized type: " + typeInfo);
	}

	return new RuntimeComparatorFactory<T>(comparator);
}
 
Example 8
Source Project: flink   Source File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) {

		TypeComparator<T> comparator;
		if (typeInfo instanceof CompositeType) {
			comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
		}
		else if (typeInfo instanceof AtomicType) {
			// handle grouping of atomic types
			comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
		}
		else {
			throw new RuntimeException("Unrecognized type: " + typeInfo);
		}

		return new RuntimeComparatorFactory<>(comparator);
	}
 
Example 9
Source Project: cascading-flink   Source File: FieldTypeInfo.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TypeComparator<Comparable> createComparator(boolean sortOrderAscending, ExecutionConfig config) {

	if(this.fieldTypeInfo != null) {
		TypeComparator<Comparable> fieldComparator = ((AtomicType)fieldTypeInfo).createComparator(sortOrderAscending, config);
		return new WrappingFieldComparator(fieldComparator, sortOrderAscending, Comparable.class);
	}
	else {
		TypeSerializer<Comparable> serializer = this.createSerializer(config);
		if (this.fieldComparator == null) {
			return new FieldComparator(sortOrderAscending, serializer, Comparable.class);
		} else {
			return new CustomFieldComparator(sortOrderAscending, this.fieldComparator, this.createSerializer(config));
		}
	}
}
 
Example 10
Source Project: flink   Source File: OuterJoinOperatorBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparator<T> buildComparatorFor(int input, ExecutionConfig executionConfig, TypeInformation<T> typeInformation) {
	TypeComparator<T> comparator;
	if (typeInformation instanceof AtomicType) {
		comparator = ((AtomicType<T>) typeInformation).createComparator(true, executionConfig);
	} else if (typeInformation instanceof CompositeType) {
		int[] keyPositions = getKeyColumns(input);
		boolean[] orders = new boolean[keyPositions.length];
		Arrays.fill(orders, true);

		comparator = ((CompositeType<T>) typeInformation).createComparator(keyPositions, orders, 0, executionConfig);
	} else {
		throw new RuntimeException("Type information for input of type " + typeInformation.getClass()
				.getCanonicalName() + " is not supported. Could not generate a comparator.");
	}
	return comparator;
}
 
Example 11
Source Project: flink   Source File: SortPartitionOperatorBase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected List<IN> executeOnCollections(List<IN> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) {

	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	int[] sortColumns = this.partitionOrdering.getFieldPositions();
	boolean[] sortOrderings = this.partitionOrdering.getFieldSortDirections();

	final TypeComparator<IN> sortComparator;
	if (inputType instanceof CompositeType) {
		sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (inputType instanceof AtomicType) {
		sortComparator = ((AtomicType) inputType).createComparator(sortOrderings[0], executionConfig);
	} else {
		throw new UnsupportedOperationException("Partition sorting does not support type "+inputType+" yet.");
	}

	Collections.sort(inputData, new Comparator<IN>() {
		@Override
		public int compare(IN o1, IN o2) {
			return sortComparator.compare(o1, o2);
		}
	});

	return inputData;
}
 
Example 12
Source Project: flink   Source File: JavaApiPostPass.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
	
	TypeComparator<T> comparator;
	if (typeInfo instanceof CompositeType) {
		comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
	}
	else if (typeInfo instanceof AtomicType) {
		// handle grouping of atomic types
		comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
	}
	else {
		throw new RuntimeException("Unrecognized type: " + typeInfo);
	}

	return new RuntimeComparatorFactory<T>(comparator);
}
 
Example 13
Source Project: flink   Source File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) {

		TypeComparator<T> comparator;
		if (typeInfo instanceof CompositeType) {
			comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
		}
		else if (typeInfo instanceof AtomicType) {
			// handle grouping of atomic types
			comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
		}
		else {
			throw new RuntimeException("Unrecognized type: " + typeInfo);
		}

		return new RuntimeComparatorFactory<>(comparator);
	}
 
Example 14
Source Project: Flink-CEPplus   Source File: CoGroupOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparator<T> getTypeComparator(ExecutionConfig executionConfig, TypeInformation<T> inputType, int[] inputKeys, boolean[] inputSortDirections) {
	if (inputType instanceof CompositeType) {
		return ((CompositeType<T>) inputType).createComparator(inputKeys, inputSortDirections, 0, executionConfig);
	} else if (inputType instanceof AtomicType) {
		return ((AtomicType<T>) inputType).createComparator(inputSortDirections[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of coGroup must be one of composite types or atomic types.");
}
 
Example 15
Source Project: Flink-CEPplus   Source File: GroupCombineOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
private TypeComparator<IN> getTypeComparator(TypeInformation<IN> typeInfo, int[] sortColumns, boolean[] sortOrderings, ExecutionConfig executionConfig) {
	if (typeInfo instanceof CompositeType) {
		return ((CompositeType<IN>) typeInfo).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (typeInfo instanceof AtomicType) {
		return ((AtomicType<IN>) typeInfo).createComparator(sortOrderings[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of GroupCombine must be one of composite types or atomic types.");
}
 
Example 16
Source Project: Flink-CEPplus   Source File: GroupReduceOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
private TypeComparator<IN> getTypeComparator(TypeInformation<IN> typeInfo, int[] sortColumns, boolean[] sortOrderings, ExecutionConfig executionConfig) {
	if (typeInfo instanceof CompositeType) {
		return ((CompositeType<IN>) typeInfo).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (typeInfo instanceof AtomicType) {
		return ((AtomicType<IN>) typeInfo).createComparator(sortOrderings[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of GroupReduce must be one of composite types or atomic types.");
}
 
Example 17
Source Project: flink   Source File: CoGroupOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparator<T> getTypeComparator(ExecutionConfig executionConfig, TypeInformation<T> inputType, int[] inputKeys, boolean[] inputSortDirections) {
	if (inputType instanceof CompositeType) {
		return ((CompositeType<T>) inputType).createComparator(inputKeys, inputSortDirections, 0, executionConfig);
	} else if (inputType instanceof AtomicType) {
		return ((AtomicType<T>) inputType).createComparator(inputSortDirections[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of coGroup must be one of composite types or atomic types.");
}
 
Example 18
Source Project: flink   Source File: GroupCombineOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
private TypeComparator<IN> getTypeComparator(TypeInformation<IN> typeInfo, int[] sortColumns, boolean[] sortOrderings, ExecutionConfig executionConfig) {
	if (typeInfo instanceof CompositeType) {
		return ((CompositeType<IN>) typeInfo).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (typeInfo instanceof AtomicType) {
		return ((AtomicType<IN>) typeInfo).createComparator(sortOrderings[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of GroupCombine must be one of composite types or atomic types.");
}
 
Example 19
Source Project: flink   Source File: GroupReduceOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
private TypeComparator<IN> getTypeComparator(TypeInformation<IN> typeInfo, int[] sortColumns, boolean[] sortOrderings, ExecutionConfig executionConfig) {
	if (typeInfo instanceof CompositeType) {
		return ((CompositeType<IN>) typeInfo).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (typeInfo instanceof AtomicType) {
		return ((AtomicType<IN>) typeInfo).createComparator(sortOrderings[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of GroupReduce must be one of composite types or atomic types.");
}
 
Example 20
Source Project: flink   Source File: CoGroupOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparator<T> getTypeComparator(ExecutionConfig executionConfig, TypeInformation<T> inputType, int[] inputKeys, boolean[] inputSortDirections) {
	if (inputType instanceof CompositeType) {
		return ((CompositeType<T>) inputType).createComparator(inputKeys, inputSortDirections, 0, executionConfig);
	} else if (inputType instanceof AtomicType) {
		return ((AtomicType<T>) inputType).createComparator(inputSortDirections[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of coGroup must be one of composite types or atomic types.");
}
 
Example 21
Source Project: flink   Source File: GroupCombineOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
private TypeComparator<IN> getTypeComparator(TypeInformation<IN> typeInfo, int[] sortColumns, boolean[] sortOrderings, ExecutionConfig executionConfig) {
	if (typeInfo instanceof CompositeType) {
		return ((CompositeType<IN>) typeInfo).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (typeInfo instanceof AtomicType) {
		return ((AtomicType<IN>) typeInfo).createComparator(sortOrderings[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of GroupCombine must be one of composite types or atomic types.");
}
 
Example 22
Source Project: flink   Source File: GroupReduceOperatorBase.java    License: Apache License 2.0 5 votes vote down vote up
private TypeComparator<IN> getTypeComparator(TypeInformation<IN> typeInfo, int[] sortColumns, boolean[] sortOrderings, ExecutionConfig executionConfig) {
	if (typeInfo instanceof CompositeType) {
		return ((CompositeType<IN>) typeInfo).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (typeInfo instanceof AtomicType) {
		return ((AtomicType<IN>) typeInfo).createComparator(sortOrderings[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of GroupReduce must be one of composite types or atomic types.");
}
 
Example 23
Source Project: Flink-CEPplus   Source File: CompositeType.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Generic implementation of the comparator creation. Composite types are supplying the infrastructure
 * to create the actual comparators
 * @return The comparator
 */
@PublicEvolving
public TypeComparator<T> createComparator(int[] logicalKeyFields, boolean[] orders, int logicalFieldOffset, ExecutionConfig config) {

	TypeComparatorBuilder<T> builder = createTypeComparatorBuilder();

	builder.initializeTypeComparatorBuilder(logicalKeyFields.length);

	for (int logicalKeyFieldIndex = 0; logicalKeyFieldIndex < logicalKeyFields.length; logicalKeyFieldIndex++) {
		int logicalKeyField = logicalKeyFields[logicalKeyFieldIndex];
		int logicalField = logicalFieldOffset; // this is the global/logical field number
		boolean comparatorAdded = false;

		for (int localFieldId = 0; localFieldId < this.getArity() && logicalField <= logicalKeyField && !comparatorAdded; localFieldId++) {
			TypeInformation<?> localFieldType = this.getTypeAt(localFieldId);
			
			if (localFieldType instanceof AtomicType && logicalField == logicalKeyField) {
				// we found an atomic key --> create comparator
				builder.addComparatorField(
					localFieldId,
					((AtomicType<?>) localFieldType).createComparator(
						orders[logicalKeyFieldIndex],
						config));

				comparatorAdded = true;
			}
			// must be composite type and check that the logicalKeyField is within the bounds
			// of the composite type's logical fields
			else if (localFieldType instanceof CompositeType &&
				logicalField <= logicalKeyField &&
				logicalKeyField <= logicalField + (localFieldType.getTotalFields() - 1)) {
				// we found a compositeType that is containing the logicalKeyField we are looking for --> create comparator
				builder.addComparatorField(
					localFieldId,
					((CompositeType<?>) localFieldType).createComparator(
						new int[]{logicalKeyField},
						new boolean[]{orders[logicalKeyFieldIndex]},
						logicalField,
						config)
				);

				comparatorAdded = true;
			}

			if (localFieldType instanceof CompositeType) {
				// we need to subtract 1 because we are not accounting for the local field (not accessible for the user)
				logicalField += localFieldType.getTotalFields() - 1;
			}
			
			logicalField++;
		}

		if (!comparatorAdded) {
			throw new IllegalArgumentException("Could not add a comparator for the logical" +
				"key field index " + logicalKeyFieldIndex + ".");
		}
	}

	return builder.createTypeComparator(config);
}
 
Example 24
Source Project: Flink-CEPplus   Source File: GenericDataSinkBase.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}
 
Example 25
Source Project: flink   Source File: CompositeType.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Generic implementation of the comparator creation. Composite types are supplying the infrastructure
 * to create the actual comparators
 * @return The comparator
 */
@PublicEvolving
public TypeComparator<T> createComparator(int[] logicalKeyFields, boolean[] orders, int logicalFieldOffset, ExecutionConfig config) {

	TypeComparatorBuilder<T> builder = createTypeComparatorBuilder();

	builder.initializeTypeComparatorBuilder(logicalKeyFields.length);

	for (int logicalKeyFieldIndex = 0; logicalKeyFieldIndex < logicalKeyFields.length; logicalKeyFieldIndex++) {
		int logicalKeyField = logicalKeyFields[logicalKeyFieldIndex];
		int logicalField = logicalFieldOffset; // this is the global/logical field number
		boolean comparatorAdded = false;

		for (int localFieldId = 0; localFieldId < this.getArity() && logicalField <= logicalKeyField && !comparatorAdded; localFieldId++) {
			TypeInformation<?> localFieldType = this.getTypeAt(localFieldId);
			
			if (localFieldType instanceof AtomicType && logicalField == logicalKeyField) {
				// we found an atomic key --> create comparator
				builder.addComparatorField(
					localFieldId,
					((AtomicType<?>) localFieldType).createComparator(
						orders[logicalKeyFieldIndex],
						config));

				comparatorAdded = true;
			}
			// must be composite type and check that the logicalKeyField is within the bounds
			// of the composite type's logical fields
			else if (localFieldType instanceof CompositeType &&
				logicalField <= logicalKeyField &&
				logicalKeyField <= logicalField + (localFieldType.getTotalFields() - 1)) {
				// we found a compositeType that is containing the logicalKeyField we are looking for --> create comparator
				builder.addComparatorField(
					localFieldId,
					((CompositeType<?>) localFieldType).createComparator(
						new int[]{logicalKeyField},
						new boolean[]{orders[logicalKeyFieldIndex]},
						logicalField,
						config)
				);

				comparatorAdded = true;
			}

			if (localFieldType instanceof CompositeType) {
				// we need to subtract 1 because we are not accounting for the local field (not accessible for the user)
				logicalField += localFieldType.getTotalFields() - 1;
			}
			
			logicalField++;
		}

		if (!comparatorAdded) {
			throw new IllegalArgumentException("Could not add a comparator for the logical" +
				"key field index " + logicalKeyFieldIndex + ".");
		}
	}

	return builder.createTypeComparator(config);
}
 
Example 26
Source Project: flink   Source File: GenericDataSinkBase.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}
 
Example 27
Source Project: flink-siddhi   Source File: StreamSchema.java    License: Apache License 2.0 4 votes vote down vote up
public boolean isAtomicType() {
    return typeInfo instanceof AtomicType;
}
 
Example 28
Source Project: bahir-flink   Source File: StreamSchema.java    License: Apache License 2.0 4 votes vote down vote up
public boolean isAtomicType() {
    return typeInfo instanceof AtomicType;
}
 
Example 29
Source Project: cascading-flink   Source File: TupleTypeInfo.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TypeComparator<Tuple> createComparator(int[] keyIdxs, boolean[] orders, int offset, ExecutionConfig config) {

	if(keyIdxs.length == 0) {
		throw new RuntimeException("Empty key indexes");
	}
	if(offset != 0) {
		throw new RuntimeException("Only 0 offset supported.");
	}

	// get key comparators
	TypeComparator<?>[] keyComps = new TypeComparator[keyIdxs.length];
	for(int i = 0; i < keyIdxs.length; i++) {
		keyComps[i] = ((AtomicType)this.getTypeAt(keyIdxs[i])).createComparator(orders[i], config);
	}

	if(length > 0) {
		// comparator for tuples with defined schema
		int maxKey = 0;
		for(int i = 0; i < keyIdxs.length; ++i) {
			int key = keyIdxs[i];
			maxKey = Math.max(maxKey, key);
		}

		// get field serializers up to max key
		TypeSerializer[] serializers = new TypeSerializer[maxKey + 1];
		for(int i = 0; i <= maxKey; ++i) {
			serializers[i] = this.fieldTypes.get(Integer.toString(i)).createSerializer(config);
		}

		return new DefinedTupleComparator(keyIdxs, keyComps, serializers, this.length);
	}
	else {
		// comparator for unknown tuples
		int[] cascadingKeyIdx = new int[keyIdxs.length];
		for(int i=0; i<cascadingKeyIdx.length; i++) {
			cascadingKeyIdx[i] = getCascadingPos(keyIdxs[i]);
		}

		return new UnknownTupleComparator(cascadingKeyIdx, keyComps, new FieldTypeInfo().createSerializer(config));
	}

}
 
Example 30
Source Project: flink   Source File: CompositeType.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Generic implementation of the comparator creation. Composite types are supplying the infrastructure
 * to create the actual comparators
 * @return The comparator
 */
@PublicEvolving
public TypeComparator<T> createComparator(int[] logicalKeyFields, boolean[] orders, int logicalFieldOffset, ExecutionConfig config) {

	TypeComparatorBuilder<T> builder = createTypeComparatorBuilder();

	builder.initializeTypeComparatorBuilder(logicalKeyFields.length);

	for (int logicalKeyFieldIndex = 0; logicalKeyFieldIndex < logicalKeyFields.length; logicalKeyFieldIndex++) {
		int logicalKeyField = logicalKeyFields[logicalKeyFieldIndex];
		int logicalField = logicalFieldOffset; // this is the global/logical field number
		boolean comparatorAdded = false;

		for (int localFieldId = 0; localFieldId < this.getArity() && logicalField <= logicalKeyField && !comparatorAdded; localFieldId++) {
			TypeInformation<?> localFieldType = this.getTypeAt(localFieldId);
			
			if (localFieldType instanceof AtomicType && logicalField == logicalKeyField) {
				// we found an atomic key --> create comparator
				builder.addComparatorField(
					localFieldId,
					((AtomicType<?>) localFieldType).createComparator(
						orders[logicalKeyFieldIndex],
						config));

				comparatorAdded = true;
			}
			// must be composite type and check that the logicalKeyField is within the bounds
			// of the composite type's logical fields
			else if (localFieldType instanceof CompositeType &&
				logicalField <= logicalKeyField &&
				logicalKeyField <= logicalField + (localFieldType.getTotalFields() - 1)) {
				// we found a compositeType that is containing the logicalKeyField we are looking for --> create comparator
				builder.addComparatorField(
					localFieldId,
					((CompositeType<?>) localFieldType).createComparator(
						new int[]{logicalKeyField},
						new boolean[]{orders[logicalKeyFieldIndex]},
						logicalField,
						config)
				);

				comparatorAdded = true;
			}

			if (localFieldType instanceof CompositeType) {
				// we need to subtract 1 because we are not accounting for the local field (not accessible for the user)
				logicalField += localFieldType.getTotalFields() - 1;
			}
			
			logicalField++;
		}

		if (!comparatorAdded) {
			throw new IllegalArgumentException("Could not add a comparator for the logical" +
				"key field index " + logicalKeyFieldIndex + ".");
		}
	}

	return builder.createTypeComparator(config);
}