org.apache.flink.api.common.typeinfo.AtomicType Java Examples

The following examples show how to use org.apache.flink.api.common.typeinfo.AtomicType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Utils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) {

		TypeComparator<T> comparator;
		if (typeInfo instanceof CompositeType) {
			comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
		}
		else if (typeInfo instanceof AtomicType) {
			// handle grouping of atomic types
			comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
		}
		else {
			throw new RuntimeException("Unrecognized type: " + typeInfo);
		}

		return new RuntimeComparatorFactory<>(comparator);
	}
 
Example #2
Source File: JavaApiPostPass.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
	
	TypeComparator<T> comparator;
	if (typeInfo instanceof CompositeType) {
		comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
	}
	else if (typeInfo instanceof AtomicType) {
		// handle grouping of atomic types
		comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
	}
	else {
		throw new RuntimeException("Unrecognized type: " + typeInfo);
	}

	return new RuntimeComparatorFactory<T>(comparator);
}
 
Example #3
Source File: Utils.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) {

		TypeComparator<T> comparator;
		if (typeInfo instanceof CompositeType) {
			comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
		}
		else if (typeInfo instanceof AtomicType) {
			// handle grouping of atomic types
			comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
		}
		else {
			throw new RuntimeException("Unrecognized type: " + typeInfo);
		}

		return new RuntimeComparatorFactory<>(comparator);
	}
 
Example #4
Source File: SortPartitionOperatorBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected List<IN> executeOnCollections(List<IN> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) {

	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	int[] sortColumns = this.partitionOrdering.getFieldPositions();
	boolean[] sortOrderings = this.partitionOrdering.getFieldSortDirections();

	final TypeComparator<IN> sortComparator;
	if (inputType instanceof CompositeType) {
		sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (inputType instanceof AtomicType) {
		sortComparator = ((AtomicType) inputType).createComparator(sortOrderings[0], executionConfig);
	} else {
		throw new UnsupportedOperationException("Partition sorting does not support type "+inputType+" yet.");
	}

	Collections.sort(inputData, new Comparator<IN>() {
		@Override
		public int compare(IN o1, IN o2) {
			return sortComparator.compare(o1, o2);
		}
	});

	return inputData;
}
 
Example #5
Source File: OuterJoinOperatorBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparator<T> buildComparatorFor(int input, ExecutionConfig executionConfig, TypeInformation<T> typeInformation) {
	TypeComparator<T> comparator;
	if (typeInformation instanceof AtomicType) {
		comparator = ((AtomicType<T>) typeInformation).createComparator(true, executionConfig);
	} else if (typeInformation instanceof CompositeType) {
		int[] keyPositions = getKeyColumns(input);
		boolean[] orders = new boolean[keyPositions.length];
		Arrays.fill(orders, true);

		comparator = ((CompositeType<T>) typeInformation).createComparator(keyPositions, orders, 0, executionConfig);
	} else {
		throw new RuntimeException("Type information for input of type " + typeInformation.getClass()
				.getCanonicalName() + " is not supported. Could not generate a comparator.");
	}
	return comparator;
}
 
Example #6
Source File: FieldTypeInfo.java    From cascading-flink with Apache License 2.0 6 votes vote down vote up
@Override
public TypeComparator<Comparable> createComparator(boolean sortOrderAscending, ExecutionConfig config) {

	if(this.fieldTypeInfo != null) {
		TypeComparator<Comparable> fieldComparator = ((AtomicType)fieldTypeInfo).createComparator(sortOrderAscending, config);
		return new WrappingFieldComparator(fieldComparator, sortOrderAscending, Comparable.class);
	}
	else {
		TypeSerializer<Comparable> serializer = this.createSerializer(config);
		if (this.fieldComparator == null) {
			return new FieldComparator(sortOrderAscending, serializer, Comparable.class);
		} else {
			return new CustomFieldComparator(sortOrderAscending, this.fieldComparator, this.createSerializer(config));
		}
	}
}
 
Example #7
Source File: JavaApiPostPass.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
	
	TypeComparator<T> comparator;
	if (typeInfo instanceof CompositeType) {
		comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
	}
	else if (typeInfo instanceof AtomicType) {
		// handle grouping of atomic types
		comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
	}
	else {
		throw new RuntimeException("Unrecognized type: " + typeInfo);
	}

	return new RuntimeComparatorFactory<T>(comparator);
}
 
Example #8
Source File: OuterJoinOperatorBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparator<T> buildComparatorFor(int input, ExecutionConfig executionConfig, TypeInformation<T> typeInformation) {
	TypeComparator<T> comparator;
	if (typeInformation instanceof AtomicType) {
		comparator = ((AtomicType<T>) typeInformation).createComparator(true, executionConfig);
	} else if (typeInformation instanceof CompositeType) {
		int[] keyPositions = getKeyColumns(input);
		boolean[] orders = new boolean[keyPositions.length];
		Arrays.fill(orders, true);

		comparator = ((CompositeType<T>) typeInformation).createComparator(keyPositions, orders, 0, executionConfig);
	} else {
		throw new RuntimeException("Type information for input of type " + typeInformation.getClass()
				.getCanonicalName() + " is not supported. Could not generate a comparator.");
	}
	return comparator;
}
 
Example #9
Source File: SortPartitionOperatorBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
protected List<IN> executeOnCollections(List<IN> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) {

	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	int[] sortColumns = this.partitionOrdering.getFieldPositions();
	boolean[] sortOrderings = this.partitionOrdering.getFieldSortDirections();

	final TypeComparator<IN> sortComparator;
	if (inputType instanceof CompositeType) {
		sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (inputType instanceof AtomicType) {
		sortComparator = ((AtomicType) inputType).createComparator(sortOrderings[0], executionConfig);
	} else {
		throw new UnsupportedOperationException("Partition sorting does not support type "+inputType+" yet.");
	}

	Collections.sort(inputData, new Comparator<IN>() {
		@Override
		public int compare(IN o1, IN o2) {
			return sortComparator.compare(o1, o2);
		}
	});

	return inputData;
}
 
Example #10
Source File: OuterJoinOperatorBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparator<T> buildComparatorFor(int input, ExecutionConfig executionConfig, TypeInformation<T> typeInformation) {
	TypeComparator<T> comparator;
	if (typeInformation instanceof AtomicType) {
		comparator = ((AtomicType<T>) typeInformation).createComparator(true, executionConfig);
	} else if (typeInformation instanceof CompositeType) {
		int[] keyPositions = getKeyColumns(input);
		boolean[] orders = new boolean[keyPositions.length];
		Arrays.fill(orders, true);

		comparator = ((CompositeType<T>) typeInformation).createComparator(keyPositions, orders, 0, executionConfig);
	} else {
		throw new RuntimeException("Type information for input of type " + typeInformation.getClass()
				.getCanonicalName() + " is not supported. Could not generate a comparator.");
	}
	return comparator;
}
 
Example #11
Source File: SortPartitionOperatorBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected List<IN> executeOnCollections(List<IN> inputData, RuntimeContext runtimeContext, ExecutionConfig executionConfig) {

	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	int[] sortColumns = this.partitionOrdering.getFieldPositions();
	boolean[] sortOrderings = this.partitionOrdering.getFieldSortDirections();

	final TypeComparator<IN> sortComparator;
	if (inputType instanceof CompositeType) {
		sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (inputType instanceof AtomicType) {
		sortComparator = ((AtomicType) inputType).createComparator(sortOrderings[0], executionConfig);
	} else {
		throw new UnsupportedOperationException("Partition sorting does not support type "+inputType+" yet.");
	}

	Collections.sort(inputData, new Comparator<IN>() {
		@Override
		public int compare(IN o1, IN o2) {
			return sortComparator.compare(o1, o2);
		}
	});

	return inputData;
}
 
Example #12
Source File: JavaApiPostPass.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder) {
	
	TypeComparator<T> comparator;
	if (typeInfo instanceof CompositeType) {
		comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
	}
	else if (typeInfo instanceof AtomicType) {
		// handle grouping of atomic types
		comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
	}
	else {
		throw new RuntimeException("Unrecognized type: " + typeInfo);
	}

	return new RuntimeComparatorFactory<T>(comparator);
}
 
Example #13
Source File: Utils.java    From flink with Apache License 2.0 6 votes vote down vote up
private static <T> TypeComparatorFactory<?> createComparator(TypeInformation<T> typeInfo, FieldList keys, boolean[] sortOrder, ExecutionConfig executionConfig) {

		TypeComparator<T> comparator;
		if (typeInfo instanceof CompositeType) {
			comparator = ((CompositeType<T>) typeInfo).createComparator(keys.toArray(), sortOrder, 0, executionConfig);
		}
		else if (typeInfo instanceof AtomicType) {
			// handle grouping of atomic types
			comparator = ((AtomicType<T>) typeInfo).createComparator(sortOrder[0], executionConfig);
		}
		else {
			throw new RuntimeException("Unrecognized type: " + typeInfo);
		}

		return new RuntimeComparatorFactory<>(comparator);
	}
 
Example #14
Source File: CoGroupOperatorBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparator<T> getTypeComparator(ExecutionConfig executionConfig, TypeInformation<T> inputType, int[] inputKeys, boolean[] inputSortDirections) {
	if (inputType instanceof CompositeType) {
		return ((CompositeType<T>) inputType).createComparator(inputKeys, inputSortDirections, 0, executionConfig);
	} else if (inputType instanceof AtomicType) {
		return ((AtomicType<T>) inputType).createComparator(inputSortDirections[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of coGroup must be one of composite types or atomic types.");
}
 
Example #15
Source File: GroupReduceOperatorBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private TypeComparator<IN> getTypeComparator(TypeInformation<IN> typeInfo, int[] sortColumns, boolean[] sortOrderings, ExecutionConfig executionConfig) {
	if (typeInfo instanceof CompositeType) {
		return ((CompositeType<IN>) typeInfo).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (typeInfo instanceof AtomicType) {
		return ((AtomicType<IN>) typeInfo).createComparator(sortOrderings[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of GroupReduce must be one of composite types or atomic types.");
}
 
Example #16
Source File: GroupCombineOperatorBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private TypeComparator<IN> getTypeComparator(TypeInformation<IN> typeInfo, int[] sortColumns, boolean[] sortOrderings, ExecutionConfig executionConfig) {
	if (typeInfo instanceof CompositeType) {
		return ((CompositeType<IN>) typeInfo).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (typeInfo instanceof AtomicType) {
		return ((AtomicType<IN>) typeInfo).createComparator(sortOrderings[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of GroupCombine must be one of composite types or atomic types.");
}
 
Example #17
Source File: CoGroupOperatorBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparator<T> getTypeComparator(ExecutionConfig executionConfig, TypeInformation<T> inputType, int[] inputKeys, boolean[] inputSortDirections) {
	if (inputType instanceof CompositeType) {
		return ((CompositeType<T>) inputType).createComparator(inputKeys, inputSortDirections, 0, executionConfig);
	} else if (inputType instanceof AtomicType) {
		return ((AtomicType<T>) inputType).createComparator(inputSortDirections[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of coGroup must be one of composite types or atomic types.");
}
 
Example #18
Source File: GroupReduceOperatorBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private TypeComparator<IN> getTypeComparator(TypeInformation<IN> typeInfo, int[] sortColumns, boolean[] sortOrderings, ExecutionConfig executionConfig) {
	if (typeInfo instanceof CompositeType) {
		return ((CompositeType<IN>) typeInfo).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (typeInfo instanceof AtomicType) {
		return ((AtomicType<IN>) typeInfo).createComparator(sortOrderings[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of GroupReduce must be one of composite types or atomic types.");
}
 
Example #19
Source File: GroupCombineOperatorBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private TypeComparator<IN> getTypeComparator(TypeInformation<IN> typeInfo, int[] sortColumns, boolean[] sortOrderings, ExecutionConfig executionConfig) {
	if (typeInfo instanceof CompositeType) {
		return ((CompositeType<IN>) typeInfo).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (typeInfo instanceof AtomicType) {
		return ((AtomicType<IN>) typeInfo).createComparator(sortOrderings[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of GroupCombine must be one of composite types or atomic types.");
}
 
Example #20
Source File: CoGroupOperatorBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> TypeComparator<T> getTypeComparator(ExecutionConfig executionConfig, TypeInformation<T> inputType, int[] inputKeys, boolean[] inputSortDirections) {
	if (inputType instanceof CompositeType) {
		return ((CompositeType<T>) inputType).createComparator(inputKeys, inputSortDirections, 0, executionConfig);
	} else if (inputType instanceof AtomicType) {
		return ((AtomicType<T>) inputType).createComparator(inputSortDirections[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of coGroup must be one of composite types or atomic types.");
}
 
Example #21
Source File: GroupReduceOperatorBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private TypeComparator<IN> getTypeComparator(TypeInformation<IN> typeInfo, int[] sortColumns, boolean[] sortOrderings, ExecutionConfig executionConfig) {
	if (typeInfo instanceof CompositeType) {
		return ((CompositeType<IN>) typeInfo).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (typeInfo instanceof AtomicType) {
		return ((AtomicType<IN>) typeInfo).createComparator(sortOrderings[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of GroupReduce must be one of composite types or atomic types.");
}
 
Example #22
Source File: GroupCombineOperatorBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private TypeComparator<IN> getTypeComparator(TypeInformation<IN> typeInfo, int[] sortColumns, boolean[] sortOrderings, ExecutionConfig executionConfig) {
	if (typeInfo instanceof CompositeType) {
		return ((CompositeType<IN>) typeInfo).createComparator(sortColumns, sortOrderings, 0, executionConfig);
	} else if (typeInfo instanceof AtomicType) {
		return ((AtomicType<IN>) typeInfo).createComparator(sortOrderings[0], executionConfig);
	}

	throw new InvalidProgramException("Input type of GroupCombine must be one of composite types or atomic types.");
}
 
Example #23
Source File: GenericDataSinkBase.java    From flink with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}
 
Example #24
Source File: StreamSchema.java    From flink-siddhi with Apache License 2.0 4 votes vote down vote up
public boolean isAtomicType() {
    return typeInfo instanceof AtomicType;
}
 
Example #25
Source File: StreamSchema.java    From bahir-flink with Apache License 2.0 4 votes vote down vote up
public boolean isAtomicType() {
    return typeInfo instanceof AtomicType;
}
 
Example #26
Source File: TupleTypeInfo.java    From cascading-flink with Apache License 2.0 4 votes vote down vote up
@Override
public TypeComparator<Tuple> createComparator(int[] keyIdxs, boolean[] orders, int offset, ExecutionConfig config) {

	if(keyIdxs.length == 0) {
		throw new RuntimeException("Empty key indexes");
	}
	if(offset != 0) {
		throw new RuntimeException("Only 0 offset supported.");
	}

	// get key comparators
	TypeComparator<?>[] keyComps = new TypeComparator[keyIdxs.length];
	for(int i = 0; i < keyIdxs.length; i++) {
		keyComps[i] = ((AtomicType)this.getTypeAt(keyIdxs[i])).createComparator(orders[i], config);
	}

	if(length > 0) {
		// comparator for tuples with defined schema
		int maxKey = 0;
		for(int i = 0; i < keyIdxs.length; ++i) {
			int key = keyIdxs[i];
			maxKey = Math.max(maxKey, key);
		}

		// get field serializers up to max key
		TypeSerializer[] serializers = new TypeSerializer[maxKey + 1];
		for(int i = 0; i <= maxKey; ++i) {
			serializers[i] = this.fieldTypes.get(Integer.toString(i)).createSerializer(config);
		}

		return new DefinedTupleComparator(keyIdxs, keyComps, serializers, this.length);
	}
	else {
		// comparator for unknown tuples
		int[] cascadingKeyIdx = new int[keyIdxs.length];
		for(int i=0; i<cascadingKeyIdx.length; i++) {
			cascadingKeyIdx[i] = getCascadingPos(keyIdxs[i]);
		}

		return new UnknownTupleComparator(cascadingKeyIdx, keyComps, new FieldTypeInfo().createSerializer(config));
	}

}
 
Example #27
Source File: CompositeType.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Generic implementation of the comparator creation. Composite types are supplying the infrastructure
 * to create the actual comparators
 * @return The comparator
 */
@PublicEvolving
public TypeComparator<T> createComparator(int[] logicalKeyFields, boolean[] orders, int logicalFieldOffset, ExecutionConfig config) {

	TypeComparatorBuilder<T> builder = createTypeComparatorBuilder();

	builder.initializeTypeComparatorBuilder(logicalKeyFields.length);

	for (int logicalKeyFieldIndex = 0; logicalKeyFieldIndex < logicalKeyFields.length; logicalKeyFieldIndex++) {
		int logicalKeyField = logicalKeyFields[logicalKeyFieldIndex];
		int logicalField = logicalFieldOffset; // this is the global/logical field number
		boolean comparatorAdded = false;

		for (int localFieldId = 0; localFieldId < this.getArity() && logicalField <= logicalKeyField && !comparatorAdded; localFieldId++) {
			TypeInformation<?> localFieldType = this.getTypeAt(localFieldId);
			
			if (localFieldType instanceof AtomicType && logicalField == logicalKeyField) {
				// we found an atomic key --> create comparator
				builder.addComparatorField(
					localFieldId,
					((AtomicType<?>) localFieldType).createComparator(
						orders[logicalKeyFieldIndex],
						config));

				comparatorAdded = true;
			}
			// must be composite type and check that the logicalKeyField is within the bounds
			// of the composite type's logical fields
			else if (localFieldType instanceof CompositeType &&
				logicalField <= logicalKeyField &&
				logicalKeyField <= logicalField + (localFieldType.getTotalFields() - 1)) {
				// we found a compositeType that is containing the logicalKeyField we are looking for --> create comparator
				builder.addComparatorField(
					localFieldId,
					((CompositeType<?>) localFieldType).createComparator(
						new int[]{logicalKeyField},
						new boolean[]{orders[logicalKeyFieldIndex]},
						logicalField,
						config)
				);

				comparatorAdded = true;
			}

			if (localFieldType instanceof CompositeType) {
				// we need to subtract 1 because we are not accounting for the local field (not accessible for the user)
				logicalField += localFieldType.getTotalFields() - 1;
			}
			
			logicalField++;
		}

		if (!comparatorAdded) {
			throw new IllegalArgumentException("Could not add a comparator for the logical" +
				"key field index " + logicalKeyFieldIndex + ".");
		}
	}

	return builder.createTypeComparator(config);
}
 
Example #28
Source File: CompositeType.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Generic implementation of the comparator creation. Composite types are supplying the infrastructure
 * to create the actual comparators
 * @return The comparator
 */
@PublicEvolving
public TypeComparator<T> createComparator(int[] logicalKeyFields, boolean[] orders, int logicalFieldOffset, ExecutionConfig config) {

	TypeComparatorBuilder<T> builder = createTypeComparatorBuilder();

	builder.initializeTypeComparatorBuilder(logicalKeyFields.length);

	for (int logicalKeyFieldIndex = 0; logicalKeyFieldIndex < logicalKeyFields.length; logicalKeyFieldIndex++) {
		int logicalKeyField = logicalKeyFields[logicalKeyFieldIndex];
		int logicalField = logicalFieldOffset; // this is the global/logical field number
		boolean comparatorAdded = false;

		for (int localFieldId = 0; localFieldId < this.getArity() && logicalField <= logicalKeyField && !comparatorAdded; localFieldId++) {
			TypeInformation<?> localFieldType = this.getTypeAt(localFieldId);
			
			if (localFieldType instanceof AtomicType && logicalField == logicalKeyField) {
				// we found an atomic key --> create comparator
				builder.addComparatorField(
					localFieldId,
					((AtomicType<?>) localFieldType).createComparator(
						orders[logicalKeyFieldIndex],
						config));

				comparatorAdded = true;
			}
			// must be composite type and check that the logicalKeyField is within the bounds
			// of the composite type's logical fields
			else if (localFieldType instanceof CompositeType &&
				logicalField <= logicalKeyField &&
				logicalKeyField <= logicalField + (localFieldType.getTotalFields() - 1)) {
				// we found a compositeType that is containing the logicalKeyField we are looking for --> create comparator
				builder.addComparatorField(
					localFieldId,
					((CompositeType<?>) localFieldType).createComparator(
						new int[]{logicalKeyField},
						new boolean[]{orders[logicalKeyFieldIndex]},
						logicalField,
						config)
				);

				comparatorAdded = true;
			}

			if (localFieldType instanceof CompositeType) {
				// we need to subtract 1 because we are not accounting for the local field (not accessible for the user)
				logicalField += localFieldType.getTotalFields() - 1;
			}
			
			logicalField++;
		}

		if (!comparatorAdded) {
			throw new IllegalArgumentException("Could not add a comparator for the logical" +
				"key field index " + logicalKeyFieldIndex + ".");
		}
	}

	return builder.createTypeComparator(config);
}
 
Example #29
Source File: GenericDataSinkBase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}
 
Example #30
Source File: GenericDataSinkBase.java    From flink with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
protected void executeOnCollections(List<IN> inputData, RuntimeContext ctx, ExecutionConfig executionConfig) throws Exception {
	OutputFormat<IN> format = this.formatWrapper.getUserCodeObject();
	TypeInformation<IN> inputType = getInput().getOperatorInfo().getOutputType();

	if (this.localOrdering != null) {
		int[] sortColumns = this.localOrdering.getFieldPositions();
		boolean[] sortOrderings = this.localOrdering.getFieldSortDirections();

		final TypeComparator<IN> sortComparator;
		if (inputType instanceof CompositeType) {
			sortComparator = ((CompositeType<IN>) inputType).createComparator(sortColumns, sortOrderings, 0, executionConfig);
		} else if (inputType instanceof AtomicType) {
			sortComparator = ((AtomicType<IN>) inputType).createComparator(sortOrderings[0], executionConfig);
		} else {
			throw new UnsupportedOperationException("Local output sorting does not support type "+inputType+" yet.");
		}

		Collections.sort(inputData, new Comparator<IN>() {
			@Override
			public int compare(IN o1, IN o2) {
				return sortComparator.compare(o1, o2);
			}
		});
	}

	if(format instanceof InitializeOnMaster) {
		((InitializeOnMaster)format).initializeGlobal(1);
	}
	format.configure(this.parameters);

	if(format instanceof RichOutputFormat){
		((RichOutputFormat<?>) format).setRuntimeContext(ctx);
	}
	format.open(0, 1);
	for (IN element : inputData) {
		format.writeRecord(element);
	}
	
	format.close();
	
	if(format instanceof FinalizeOnMaster) {
		((FinalizeOnMaster)format).finalizeGlobal(1);
	}
}