Java Code Examples for org.apache.flink.api.common.typeutils.TypeComparator#invertNormalizedKey()

The following examples show how to use org.apache.flink.api.common.typeutils.TypeComparator#invertNormalizedKey() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@SuppressWarnings("unchecked")
public TupleComparatorBase(int[] keyPositions, TypeComparator<?>[] comparators, TypeSerializer<?>[] serializers) {
	// set the default utils
	this.keyPositions = keyPositions;
	this.comparators = (TypeComparator<Object>[]) comparators;
	this.serializers = (TypeSerializer<Object>[]) serializers;

	// set up auxiliary fields for normalized key support
	this.normalizedKeyLengths = new int[keyPositions.length];
	int nKeys = 0;
	int nKeyLen = 0;
	boolean inverted = false;

	for (int i = 0; i < this.keyPositions.length; i++) {
		TypeComparator<?> k = this.comparators[i];

		// as long as the leading keys support normalized keys, we can build up the composite key
		if (k.supportsNormalizedKey()) {
			if (i == 0) {
				// the first comparator decides whether we need to invert the key direction
				inverted = k.invertNormalizedKey();
			}
			else if (k.invertNormalizedKey() != inverted) {
				// if a successor does not agree on the inversion direction, it cannot be part of the normalized key
				break;
			}

			nKeys++;
			final int len = k.getNormalizeKeyLen();
			if (len < 0) {
				throw new RuntimeException("Comparator " + k.getClass().getName() + " specifies an invalid length for the normalized key: " + len);
			}
			this.normalizedKeyLengths[i] = len;
			nKeyLen += len;

			if (nKeyLen < 0) {
				// overflow, which means we are out of budget for normalized key space anyways
				nKeyLen = Integer.MAX_VALUE;
				break;
			}
		} else {
			break;
		}
	}
	this.numLeadingNormalizableKeys = nKeys;
	this.normalizableKeyPrefixLen = nKeyLen;
	this.invertNormKey = inverted;
}
 
Example 2
Source Project: Flink-CEPplus   File: PojoComparator.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public PojoComparator(Field[] keyFields, TypeComparator<?>[] comparators, TypeSerializer<T> serializer, Class<T> type) {
	this.keyFields = keyFields;
	this.comparators = (TypeComparator<Object>[]) comparators;

	this.type = type;
	this.serializer = serializer;

	// set up auxiliary fields for normalized key support
	this.normalizedKeyLengths = new int[keyFields.length];
	int nKeys = 0;
	int nKeyLen = 0;
	boolean inverted = false;

	for (int i = 0; i < this.comparators.length; i++) {
		TypeComparator<?> k = this.comparators[i];
		if(k == null) {
			throw new IllegalArgumentException("One of the passed comparators is null");
		}
		if(keyFields[i] == null) {
			throw new IllegalArgumentException("One of the passed reflection fields is null");
		}

		// as long as the leading keys support normalized keys, we can build up the composite key
		if (k.supportsNormalizedKey()) {
			if (i == 0) {
				// the first comparator decides whether we need to invert the key direction
				inverted = k.invertNormalizedKey();
			}
			else if (k.invertNormalizedKey() != inverted) {
				// if a successor does not agree on the inversion direction, it cannot be part of the normalized key
				break;
			}

			nKeys++;
			final int len = k.getNormalizeKeyLen();
			if (len < 0) {
				throw new RuntimeException("Comparator " + k.getClass().getName() + " specifies an invalid length for the normalized key: " + len);
			}
			this.normalizedKeyLengths[i] = len;
			nKeyLen += this.normalizedKeyLengths[i];

			if (nKeyLen < 0) {
				// overflow, which means we are out of budget for normalized key space anyways
				nKeyLen = Integer.MAX_VALUE;
				break;
			}
		} else {
			break;
		}
	}
	this.numLeadingNormalizableKeys = nKeys;
	this.normalizableKeyPrefixLen = nKeyLen;
	this.invertNormKey = inverted;
}
 
Example 3
public NormalizedKeySorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory, int maxNormalizedKeyBytes)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	if (maxNormalizedKeyBytes < 0) {
		throw new IllegalArgumentException("Maximal number of normalized key bytes must not be negative.");
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortIndex = new ArrayList<MemorySegment>(16);
	this.recordBufferSegments = new ArrayList<MemorySegment>(16);
	
	// the views for the record collections
	this.recordCollector = new SimpleCollectingOutputView(this.recordBufferSegments,
		new ListMemorySegmentSource(this.freeMemory), this.segmentSize);
	this.recordBuffer = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	this.recordBufferForComparison = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	
	// set up normalized key characteristics
	if (this.comparator.supportsNormalizedKey()) {
		// compute the max normalized key length
		int numPartialKeys;
		try {
			numPartialKeys = this.comparator.getFlatComparators().length;
		} catch (Throwable t) {
			numPartialKeys = 1;
		}
		
		int maxLen = Math.min(maxNormalizedKeyBytes, MAX_NORMALIZED_KEY_LEN_PER_ELEMENT * numPartialKeys);
		
		this.numKeyBytes = Math.min(this.comparator.getNormalizeKeyLen(), maxLen);
		this.normalizedKeyFullyDetermines = !this.comparator.isNormalizedKeyPrefixOnly(this.numKeyBytes);
	}
	else {
		this.numKeyBytes = 0;
		this.normalizedKeyFullyDetermines = false;
	}
	
	// compute the index entry size and limits
	this.indexEntrySize = this.numKeyBytes + OFFSET_LEN;
	this.indexEntriesPerSegment = this.segmentSize / this.indexEntrySize;
	this.lastIndexEntryOffset = (this.indexEntriesPerSegment - 1) * this.indexEntrySize;
	this.swapBuffer = new byte[this.indexEntrySize];
	
	// set to initial state
	this.currentSortIndexSegment = nextMemorySegment();
	this.sortIndex.add(this.currentSortIndexSegment);
}
 
Example 4
public FixedLengthRecordSorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.recordSize = serializer.getLength();
	this.numKeyBytes = this.comparator.getNormalizeKeyLen();
	
	// check that the serializer and comparator allow our operations
	if (this.recordSize <= 0) {
		throw new IllegalArgumentException("This sorter works only for fixed-length data types.");
	} else if (this.recordSize > this.segmentSize) {
		throw new IllegalArgumentException("This sorter works only for record lengths below the memory segment size.");
	} else if (!comparator.supportsSerializationWithKeyNormalization()) {
		throw new IllegalArgumentException("This sorter requires a comparator that supports serialization with key normalization.");
	}
	
	// compute the entry size and limits
	this.recordsPerSegment = segmentSize / this.recordSize;
	this.lastEntryOffset = (this.recordsPerSegment - 1) * this.recordSize;
	this.swapBuffer = new byte[this.recordSize];
	
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortBuffer = new ArrayList<MemorySegment>(16);
	this.outView = new SingleSegmentOutputView(this.segmentSize);
	this.inView = new SingleSegmentInputView(this.lastEntryOffset + this.recordSize);
	this.currentSortBufferSegment = nextMemorySegment();
	this.sortBuffer.add(this.currentSortBufferSegment);
	this.outView.set(this.currentSortBufferSegment);
	
	this.recordInstance = this.serializer.createInstance();
}
 
Example 5
Source Project: flink   File: TupleComparatorBase.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public TupleComparatorBase(int[] keyPositions, TypeComparator<?>[] comparators, TypeSerializer<?>[] serializers) {
	// set the default utils
	this.keyPositions = keyPositions;
	this.comparators = (TypeComparator<Object>[]) comparators;
	this.serializers = (TypeSerializer<Object>[]) serializers;

	// set up auxiliary fields for normalized key support
	this.normalizedKeyLengths = new int[keyPositions.length];
	int nKeys = 0;
	int nKeyLen = 0;
	boolean inverted = false;

	for (int i = 0; i < this.keyPositions.length; i++) {
		TypeComparator<?> k = this.comparators[i];

		// as long as the leading keys support normalized keys, we can build up the composite key
		if (k.supportsNormalizedKey()) {
			if (i == 0) {
				// the first comparator decides whether we need to invert the key direction
				inverted = k.invertNormalizedKey();
			}
			else if (k.invertNormalizedKey() != inverted) {
				// if a successor does not agree on the inversion direction, it cannot be part of the normalized key
				break;
			}

			nKeys++;
			final int len = k.getNormalizeKeyLen();
			if (len < 0) {
				throw new RuntimeException("Comparator " + k.getClass().getName() + " specifies an invalid length for the normalized key: " + len);
			}
			this.normalizedKeyLengths[i] = len;
			nKeyLen += len;

			if (nKeyLen < 0) {
				// overflow, which means we are out of budget for normalized key space anyways
				nKeyLen = Integer.MAX_VALUE;
				break;
			}
		} else {
			break;
		}
	}
	this.numLeadingNormalizableKeys = nKeys;
	this.normalizableKeyPrefixLen = nKeyLen;
	this.invertNormKey = inverted;
}
 
Example 6
Source Project: flink   File: PojoComparator.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public PojoComparator(Field[] keyFields, TypeComparator<?>[] comparators, TypeSerializer<T> serializer, Class<T> type) {
	this.keyFields = keyFields;
	this.comparators = (TypeComparator<Object>[]) comparators;

	this.type = type;
	this.serializer = serializer;

	// set up auxiliary fields for normalized key support
	this.normalizedKeyLengths = new int[keyFields.length];
	int nKeys = 0;
	int nKeyLen = 0;
	boolean inverted = false;

	for (int i = 0; i < this.comparators.length; i++) {
		TypeComparator<?> k = this.comparators[i];
		if(k == null) {
			throw new IllegalArgumentException("One of the passed comparators is null");
		}
		if(keyFields[i] == null) {
			throw new IllegalArgumentException("One of the passed reflection fields is null");
		}

		// as long as the leading keys support normalized keys, we can build up the composite key
		if (k.supportsNormalizedKey()) {
			if (i == 0) {
				// the first comparator decides whether we need to invert the key direction
				inverted = k.invertNormalizedKey();
			}
			else if (k.invertNormalizedKey() != inverted) {
				// if a successor does not agree on the inversion direction, it cannot be part of the normalized key
				break;
			}

			nKeys++;
			final int len = k.getNormalizeKeyLen();
			if (len < 0) {
				throw new RuntimeException("Comparator " + k.getClass().getName() + " specifies an invalid length for the normalized key: " + len);
			}
			this.normalizedKeyLengths[i] = len;
			nKeyLen += this.normalizedKeyLengths[i];

			if (nKeyLen < 0) {
				// overflow, which means we are out of budget for normalized key space anyways
				nKeyLen = Integer.MAX_VALUE;
				break;
			}
		} else {
			break;
		}
	}
	this.numLeadingNormalizableKeys = nKeys;
	this.normalizableKeyPrefixLen = nKeyLen;
	this.invertNormKey = inverted;
}
 
Example 7
Source Project: flink   File: NormalizedKeySorter.java    License: Apache License 2.0 4 votes vote down vote up
public NormalizedKeySorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory, int maxNormalizedKeyBytes)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	if (maxNormalizedKeyBytes < 0) {
		throw new IllegalArgumentException("Maximal number of normalized key bytes must not be negative.");
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortIndex = new ArrayList<MemorySegment>(16);
	this.recordBufferSegments = new ArrayList<MemorySegment>(16);
	
	// the views for the record collections
	this.recordCollector = new SimpleCollectingOutputView(this.recordBufferSegments,
		new ListMemorySegmentSource(this.freeMemory), this.segmentSize);
	this.recordBuffer = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	this.recordBufferForComparison = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	
	// set up normalized key characteristics
	if (this.comparator.supportsNormalizedKey()) {
		// compute the max normalized key length
		int numPartialKeys;
		try {
			numPartialKeys = this.comparator.getFlatComparators().length;
		} catch (Throwable t) {
			numPartialKeys = 1;
		}
		
		int maxLen = Math.min(maxNormalizedKeyBytes, MAX_NORMALIZED_KEY_LEN_PER_ELEMENT * numPartialKeys);
		
		this.numKeyBytes = Math.min(this.comparator.getNormalizeKeyLen(), maxLen);
		this.normalizedKeyFullyDetermines = !this.comparator.isNormalizedKeyPrefixOnly(this.numKeyBytes);
	}
	else {
		this.numKeyBytes = 0;
		this.normalizedKeyFullyDetermines = false;
	}
	
	// compute the index entry size and limits
	this.indexEntrySize = this.numKeyBytes + OFFSET_LEN;
	this.indexEntriesPerSegment = this.segmentSize / this.indexEntrySize;
	this.lastIndexEntryOffset = (this.indexEntriesPerSegment - 1) * this.indexEntrySize;
	this.swapBuffer = new byte[this.indexEntrySize];
	
	// set to initial state
	this.currentSortIndexSegment = nextMemorySegment();
	this.sortIndex.add(this.currentSortIndexSegment);
}
 
Example 8
Source Project: flink   File: FixedLengthRecordSorter.java    License: Apache License 2.0 4 votes vote down vote up
public FixedLengthRecordSorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.recordSize = serializer.getLength();
	this.numKeyBytes = this.comparator.getNormalizeKeyLen();
	
	// check that the serializer and comparator allow our operations
	if (this.recordSize <= 0) {
		throw new IllegalArgumentException("This sorter works only for fixed-length data types.");
	} else if (this.recordSize > this.segmentSize) {
		throw new IllegalArgumentException("This sorter works only for record lengths below the memory segment size.");
	} else if (!comparator.supportsSerializationWithKeyNormalization()) {
		throw new IllegalArgumentException("This sorter requires a comparator that supports serialization with key normalization.");
	}
	
	// compute the entry size and limits
	this.recordsPerSegment = segmentSize / this.recordSize;
	this.lastEntryOffset = (this.recordsPerSegment - 1) * this.recordSize;
	this.swapBuffer = new byte[this.recordSize];
	
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortBuffer = new ArrayList<MemorySegment>(16);
	this.outView = new SingleSegmentOutputView(this.segmentSize);
	this.inView = new SingleSegmentInputView(this.lastEntryOffset + this.recordSize);
	this.currentSortBufferSegment = nextMemorySegment();
	this.sortBuffer.add(this.currentSortBufferSegment);
	this.outView.set(this.currentSortBufferSegment);
	
	this.recordInstance = this.serializer.createInstance();
}
 
Example 9
public UnknownTupleComparator(int[] keyPositions, TypeComparator<?>[] comparators, TypeSerializer<?> serializer) {

		this.keyPositions = keyPositions;
		this.comparators = comparators;
		this.serializer = serializer;

		// set up auxiliary fields for normalized key support
		this.normalizedKeyLengths = new int[keyPositions.length];
		int nKeys = 0;
		int nKeyLen = 0;
		boolean inverted = false;

		for (int i = 0; i < this.keyPositions.length; i++) {
			TypeComparator<?> k = this.comparators[i];

			// as long as the leading keys support normalized keys, we can build up the composite key
			if (k.supportsNormalizedKey()) {
				if (i == 0) {
					// the first comparator decides whether we need to invert the key direction
					inverted = k.invertNormalizedKey();
				}
				else if (k.invertNormalizedKey() != inverted) {
					// if a successor does not agree on the inversion direction, it cannot be part of the normalized key
					break;
				}

				nKeys++;
				final int len = k.getNormalizeKeyLen();
				if (len < 0) {
					throw new RuntimeException("Comparator " + k.getClass().getName() + " specifies an invalid length for the normalized key: " + len);
				}
				this.normalizedKeyLengths[i] = len;
				nKeyLen += len;

				if (nKeyLen < 0) {
					// overflow, which means we are out of budget for normalized key space anyways
					nKeyLen = Integer.MAX_VALUE;
					break;
				}
			} else {
				break;
			}
		}
		this.numLeadingNormalizableKeys = nKeys;
		this.normalizableKeyPrefixLen = nKeyLen;
		this.invertNormKey = inverted;
	}
 
Example 10
public DefinedTupleComparator(int[] keyPositions, TypeComparator<?>[] comparators, TypeSerializer<?>[] serializers, int tupleLength) {

		this.keyPositions = keyPositions;
		this.comparators = comparators;
		this.serializers = serializers;
		this.tupleLength = tupleLength;

		this.fields1 = new Object[serializers.length];
		this.fields2 = new Object[serializers.length];
		this.nullFields1 = new boolean[this.tupleLength];
		this.nullFields2 = new boolean[this.tupleLength];

		// set up auxiliary fields for normalized key support
		this.normalizedKeyLengths = new int[keyPositions.length];
		int nKeys = 0;
		int nKeyLen = 0;
		boolean inverted = false;

		for (int i = 0; i < this.keyPositions.length; i++) {
			TypeComparator<?> k = this.comparators[i];

			// as long as the leading keys support normalized keys, we can build up the composite key
			if (k.supportsNormalizedKey()) {
				if (i == 0) {
					// the first comparator decides whether we need to invert the key direction
					inverted = k.invertNormalizedKey();
				}
				else if (k.invertNormalizedKey() != inverted) {
					// if a successor does not agree on the inversion direction, it cannot be part of the normalized key
					break;
				}

				nKeys++;
				final int len = k.getNormalizeKeyLen();
				if (len < 0) {
					throw new RuntimeException("Comparator " + k.getClass().getName() + " specifies an invalid length for the normalized key: " + len);
				}
				this.normalizedKeyLengths[i] = len;
				nKeyLen += len;

				if (nKeyLen < 0) {
					// overflow, which means we are out of budget for normalized key space anyways
					nKeyLen = Integer.MAX_VALUE;
					break;
				}
			} else {
				break;
			}
		}
		this.numLeadingNormalizableKeys = nKeys;
		this.normalizableKeyPrefixLen = nKeyLen;
		this.invertNormKey = inverted;
	}
 
Example 11
Source Project: flink   File: TupleComparatorBase.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public TupleComparatorBase(int[] keyPositions, TypeComparator<?>[] comparators, TypeSerializer<?>[] serializers) {
	// set the default utils
	this.keyPositions = keyPositions;
	this.comparators = (TypeComparator<Object>[]) comparators;
	this.serializers = (TypeSerializer<Object>[]) serializers;

	// set up auxiliary fields for normalized key support
	this.normalizedKeyLengths = new int[keyPositions.length];
	int nKeys = 0;
	int nKeyLen = 0;
	boolean inverted = false;

	for (int i = 0; i < this.keyPositions.length; i++) {
		TypeComparator<?> k = this.comparators[i];

		// as long as the leading keys support normalized keys, we can build up the composite key
		if (k.supportsNormalizedKey()) {
			if (i == 0) {
				// the first comparator decides whether we need to invert the key direction
				inverted = k.invertNormalizedKey();
			}
			else if (k.invertNormalizedKey() != inverted) {
				// if a successor does not agree on the inversion direction, it cannot be part of the normalized key
				break;
			}

			nKeys++;
			final int len = k.getNormalizeKeyLen();
			if (len < 0) {
				throw new RuntimeException("Comparator " + k.getClass().getName() + " specifies an invalid length for the normalized key: " + len);
			}
			this.normalizedKeyLengths[i] = len;
			nKeyLen += len;

			if (nKeyLen < 0) {
				// overflow, which means we are out of budget for normalized key space anyways
				nKeyLen = Integer.MAX_VALUE;
				break;
			}
		} else {
			break;
		}
	}
	this.numLeadingNormalizableKeys = nKeys;
	this.normalizableKeyPrefixLen = nKeyLen;
	this.invertNormKey = inverted;
}
 
Example 12
Source Project: flink   File: PojoComparator.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public PojoComparator(Field[] keyFields, TypeComparator<?>[] comparators, TypeSerializer<T> serializer, Class<T> type) {
	this.keyFields = keyFields;
	this.comparators = (TypeComparator<Object>[]) comparators;

	this.type = type;
	this.serializer = serializer;

	// set up auxiliary fields for normalized key support
	this.normalizedKeyLengths = new int[keyFields.length];
	int nKeys = 0;
	int nKeyLen = 0;
	boolean inverted = false;

	for (int i = 0; i < this.comparators.length; i++) {
		TypeComparator<?> k = this.comparators[i];
		if(k == null) {
			throw new IllegalArgumentException("One of the passed comparators is null");
		}
		if(keyFields[i] == null) {
			throw new IllegalArgumentException("One of the passed reflection fields is null");
		}

		// as long as the leading keys support normalized keys, we can build up the composite key
		if (k.supportsNormalizedKey()) {
			if (i == 0) {
				// the first comparator decides whether we need to invert the key direction
				inverted = k.invertNormalizedKey();
			}
			else if (k.invertNormalizedKey() != inverted) {
				// if a successor does not agree on the inversion direction, it cannot be part of the normalized key
				break;
			}

			nKeys++;
			final int len = k.getNormalizeKeyLen();
			if (len < 0) {
				throw new RuntimeException("Comparator " + k.getClass().getName() + " specifies an invalid length for the normalized key: " + len);
			}
			this.normalizedKeyLengths[i] = len;
			nKeyLen += this.normalizedKeyLengths[i];

			if (nKeyLen < 0) {
				// overflow, which means we are out of budget for normalized key space anyways
				nKeyLen = Integer.MAX_VALUE;
				break;
			}
		} else {
			break;
		}
	}
	this.numLeadingNormalizableKeys = nKeys;
	this.normalizableKeyPrefixLen = nKeyLen;
	this.invertNormKey = inverted;
}
 
Example 13
Source Project: flink   File: NormalizedKeySorter.java    License: Apache License 2.0 4 votes vote down vote up
public NormalizedKeySorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory, int maxNormalizedKeyBytes)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	if (maxNormalizedKeyBytes < 0) {
		throw new IllegalArgumentException("Maximal number of normalized key bytes must not be negative.");
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortIndex = new ArrayList<MemorySegment>(16);
	this.recordBufferSegments = new ArrayList<MemorySegment>(16);
	
	// the views for the record collections
	this.recordCollector = new SimpleCollectingOutputView(this.recordBufferSegments,
		new ListMemorySegmentSource(this.freeMemory), this.segmentSize);
	this.recordBuffer = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	this.recordBufferForComparison = new RandomAccessInputView(this.recordBufferSegments, this.segmentSize);
	
	// set up normalized key characteristics
	if (this.comparator.supportsNormalizedKey()) {
		// compute the max normalized key length
		int numPartialKeys;
		try {
			numPartialKeys = this.comparator.getFlatComparators().length;
		} catch (Throwable t) {
			numPartialKeys = 1;
		}
		
		int maxLen = Math.min(maxNormalizedKeyBytes, MAX_NORMALIZED_KEY_LEN_PER_ELEMENT * numPartialKeys);
		
		this.numKeyBytes = Math.min(this.comparator.getNormalizeKeyLen(), maxLen);
		this.normalizedKeyFullyDetermines = !this.comparator.isNormalizedKeyPrefixOnly(this.numKeyBytes);
	}
	else {
		this.numKeyBytes = 0;
		this.normalizedKeyFullyDetermines = false;
	}
	
	// compute the index entry size and limits
	this.indexEntrySize = this.numKeyBytes + OFFSET_LEN;
	this.indexEntriesPerSegment = this.segmentSize / this.indexEntrySize;
	this.lastIndexEntryOffset = (this.indexEntriesPerSegment - 1) * this.indexEntrySize;
	this.swapBuffer = new byte[this.indexEntrySize];
	
	// set to initial state
	this.currentSortIndexSegment = nextMemorySegment();
	this.sortIndex.add(this.currentSortIndexSegment);
}
 
Example 14
Source Project: flink   File: FixedLengthRecordSorter.java    License: Apache License 2.0 4 votes vote down vote up
public FixedLengthRecordSorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.recordSize = serializer.getLength();
	this.numKeyBytes = this.comparator.getNormalizeKeyLen();
	
	// check that the serializer and comparator allow our operations
	if (this.recordSize <= 0) {
		throw new IllegalArgumentException("This sorter works only for fixed-length data types.");
	} else if (this.recordSize > this.segmentSize) {
		throw new IllegalArgumentException("This sorter works only for record lengths below the memory segment size.");
	} else if (!comparator.supportsSerializationWithKeyNormalization()) {
		throw new IllegalArgumentException("This sorter requires a comparator that supports serialization with key normalization.");
	}
	
	// compute the entry size and limits
	this.recordsPerSegment = segmentSize / this.recordSize;
	this.lastEntryOffset = (this.recordsPerSegment - 1) * this.recordSize;
	this.swapBuffer = new byte[this.recordSize];
	
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortBuffer = new ArrayList<MemorySegment>(16);
	this.outView = new SingleSegmentOutputView(this.segmentSize);
	this.inView = new SingleSegmentInputView(this.lastEntryOffset + this.recordSize);
	this.currentSortBufferSegment = nextMemorySegment();
	this.sortBuffer.add(this.currentSortBufferSegment);
	this.outView.set(this.currentSortBufferSegment);
	
	this.recordInstance = this.serializer.createInstance();
}