Java Code Examples for org.apache.flink.api.common.typeutils.TypeComparator#supportsSerializationWithKeyNormalization()

The following examples show how to use org.apache.flink.api.common.typeutils.TypeComparator#supportsSerializationWithKeyNormalization() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
DefaultInMemorySorterFactory(
		@Nonnull TypeSerializerFactory<T> typeSerializerFactory,
		@Nonnull TypeComparator<T> typeComparator,
		int thresholdForInPlaceSorting) {
	this.typeSerializerFactory = typeSerializerFactory;
	this.typeComparator = typeComparator;

	TypeSerializer<T> typeSerializer = typeSerializerFactory.getSerializer();

	this.useFixedLengthRecordSorter = typeComparator.supportsSerializationWithKeyNormalization() &&
		typeSerializer.getLength() > 0 && typeSerializer.getLength() <= thresholdForInPlaceSorting;
}
 
Example 2
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.combiner, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	
	this.serializer = serializerFactory.getSerializer();

	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();
	
	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 3
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.reducer, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	this.serializer = serializerFactory.getSerializer();
	
	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();

	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 4
@Override
public void prepare() throws Exception {
	final DriverStrategy driverStrategy = this.taskContext.getTaskConfig().getDriverStrategy();
	if (driverStrategy != DriverStrategy.SORTED_GROUP_COMBINE){
		throw new Exception("Invalid strategy " + driverStrategy + " for group reduce combiner.");
	}
	
	

	final TypeSerializerFactory<IN> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();

	final TypeComparator<IN> sortingComparator = this.taskContext.getDriverComparator(0);
	
	this.groupingComparator = this.taskContext.getDriverComparator(1);
	this.combiner = this.taskContext.getStub();
	this.output = this.taskContext.getOutputCollector();

	MemoryManager memManager = this.taskContext.getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.taskContext.getTaskConfig().getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.taskContext.getContainingTask(), numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
			this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceCombineDriver object reuse: {}.", (this.objectReuseEnabled ? "ENABLED" : "DISABLED"));
	}
}
 
Example 5
DefaultInMemorySorterFactory(
		@Nonnull TypeSerializerFactory<T> typeSerializerFactory,
		@Nonnull TypeComparator<T> typeComparator,
		int thresholdForInPlaceSorting) {
	this.typeSerializerFactory = typeSerializerFactory;
	this.typeComparator = typeComparator;

	TypeSerializer<T> typeSerializer = typeSerializerFactory.getSerializer();

	this.useFixedLengthRecordSorter = typeComparator.supportsSerializationWithKeyNormalization() &&
		typeSerializer.getLength() > 0 && typeSerializer.getLength() <= thresholdForInPlaceSorting;
}
 
Example 6
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.combiner, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	
	this.serializer = serializerFactory.getSerializer();

	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();
	
	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 7
Source Project: flink   File: GroupCombineChainedDriver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.reducer, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	this.serializer = serializerFactory.getSerializer();
	
	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();

	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 8
Source Project: flink   File: GroupReduceCombineDriver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prepare() throws Exception {
	final DriverStrategy driverStrategy = this.taskContext.getTaskConfig().getDriverStrategy();
	if (driverStrategy != DriverStrategy.SORTED_GROUP_COMBINE){
		throw new Exception("Invalid strategy " + driverStrategy + " for group reduce combiner.");
	}
	
	

	final TypeSerializerFactory<IN> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();

	final TypeComparator<IN> sortingComparator = this.taskContext.getDriverComparator(0);
	
	this.groupingComparator = this.taskContext.getDriverComparator(1);
	this.combiner = this.taskContext.getStub();
	this.output = this.taskContext.getOutputCollector();

	MemoryManager memManager = this.taskContext.getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.taskContext.getTaskConfig().getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.taskContext.getContainingTask(), numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
			this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceCombineDriver object reuse: {}.", (this.objectReuseEnabled ? "ENABLED" : "DISABLED"));
	}
}
 
Example 9
DefaultInMemorySorterFactory(
		@Nonnull TypeSerializerFactory<T> typeSerializerFactory,
		@Nonnull TypeComparator<T> typeComparator,
		int thresholdForInPlaceSorting) {
	this.typeSerializerFactory = typeSerializerFactory;
	this.typeComparator = typeComparator;

	TypeSerializer<T> typeSerializer = typeSerializerFactory.getSerializer();

	this.useFixedLengthRecordSorter = typeComparator.supportsSerializationWithKeyNormalization() &&
		typeSerializer.getLength() > 0 && typeSerializer.getLength() <= thresholdForInPlaceSorting;
}
 
Example 10
@Override
public void openTask() throws Exception {
	// open the stub first
	final Configuration stubConfig = this.config.getStubParameters();
	BatchTask.openUserCode(this.combiner, stubConfig);

	// ----------------- Set up the sorter -------------------------

	// instantiate the serializer / comparator
	final TypeSerializerFactory<IN> serializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> sortingComparatorFactory = this.config.getDriverComparator(0, this.userCodeClassLoader);
	final TypeComparatorFactory<IN> groupingComparatorFactory = this.config.getDriverComparator(1, this.userCodeClassLoader);
	
	this.serializer = serializerFactory.getSerializer();

	TypeComparator<IN> sortingComparator = sortingComparatorFactory.createComparator();
	this.groupingComparator = groupingComparatorFactory.createComparator();
	
	MemoryManager memManager = this.parent.getEnvironment().getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.config.getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.parent, numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
		this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), this.memory);
	}

	if (LOG.isDebugEnabled()) {
		LOG.debug("SynchronousChainedCombineDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
	}
}
 
Example 11
Source Project: flink   File: GroupReduceCombineDriver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prepare() throws Exception {
	final DriverStrategy driverStrategy = this.taskContext.getTaskConfig().getDriverStrategy();
	if (driverStrategy != DriverStrategy.SORTED_GROUP_COMBINE){
		throw new Exception("Invalid strategy " + driverStrategy + " for group reduce combiner.");
	}
	
	

	final TypeSerializerFactory<IN> serializerFactory = this.taskContext.getInputSerializer(0);
	this.serializer = serializerFactory.getSerializer();

	final TypeComparator<IN> sortingComparator = this.taskContext.getDriverComparator(0);
	
	this.groupingComparator = this.taskContext.getDriverComparator(1);
	this.combiner = this.taskContext.getStub();
	this.output = this.taskContext.getOutputCollector();

	MemoryManager memManager = this.taskContext.getMemoryManager();
	final int numMemoryPages = memManager.computeNumberOfPages(this.taskContext.getTaskConfig().getRelativeMemoryDriver());
	this.memory = memManager.allocatePages(this.taskContext.getContainingTask(), numMemoryPages);

	// instantiate a fix-length in-place sorter, if possible, otherwise the out-of-place sorter
	if (sortingComparator.supportsSerializationWithKeyNormalization() &&
			this.serializer.getLength() > 0 && this.serializer.getLength() <= THRESHOLD_FOR_IN_PLACE_SORTING)
	{
		this.sorter = new FixedLengthRecordSorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	} else {
		this.sorter = new NormalizedKeySorter<IN>(this.serializer, sortingComparator.duplicate(), memory);
	}

	ExecutionConfig executionConfig = taskContext.getExecutionConfig();
	this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();

	if (LOG.isDebugEnabled()) {
		LOG.debug("GroupReduceCombineDriver object reuse: {}.", (this.objectReuseEnabled ? "ENABLED" : "DISABLED"));
	}
}
 
Example 12
public FixedLengthRecordSorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.recordSize = serializer.getLength();
	this.numKeyBytes = this.comparator.getNormalizeKeyLen();
	
	// check that the serializer and comparator allow our operations
	if (this.recordSize <= 0) {
		throw new IllegalArgumentException("This sorter works only for fixed-length data types.");
	} else if (this.recordSize > this.segmentSize) {
		throw new IllegalArgumentException("This sorter works only for record lengths below the memory segment size.");
	} else if (!comparator.supportsSerializationWithKeyNormalization()) {
		throw new IllegalArgumentException("This sorter requires a comparator that supports serialization with key normalization.");
	}
	
	// compute the entry size and limits
	this.recordsPerSegment = segmentSize / this.recordSize;
	this.lastEntryOffset = (this.recordsPerSegment - 1) * this.recordSize;
	this.swapBuffer = new byte[this.recordSize];
	
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortBuffer = new ArrayList<MemorySegment>(16);
	this.outView = new SingleSegmentOutputView(this.segmentSize);
	this.inView = new SingleSegmentInputView(this.lastEntryOffset + this.recordSize);
	this.currentSortBufferSegment = nextMemorySegment();
	this.sortBuffer.add(this.currentSortBufferSegment);
	this.outView.set(this.currentSortBufferSegment);
	
	this.recordInstance = this.serializer.createInstance();
}
 
Example 13
Source Project: flink   File: FixedLengthRecordSorter.java    License: Apache License 2.0 4 votes vote down vote up
public FixedLengthRecordSorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.recordSize = serializer.getLength();
	this.numKeyBytes = this.comparator.getNormalizeKeyLen();
	
	// check that the serializer and comparator allow our operations
	if (this.recordSize <= 0) {
		throw new IllegalArgumentException("This sorter works only for fixed-length data types.");
	} else if (this.recordSize > this.segmentSize) {
		throw new IllegalArgumentException("This sorter works only for record lengths below the memory segment size.");
	} else if (!comparator.supportsSerializationWithKeyNormalization()) {
		throw new IllegalArgumentException("This sorter requires a comparator that supports serialization with key normalization.");
	}
	
	// compute the entry size and limits
	this.recordsPerSegment = segmentSize / this.recordSize;
	this.lastEntryOffset = (this.recordsPerSegment - 1) * this.recordSize;
	this.swapBuffer = new byte[this.recordSize];
	
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortBuffer = new ArrayList<MemorySegment>(16);
	this.outView = new SingleSegmentOutputView(this.segmentSize);
	this.inView = new SingleSegmentInputView(this.lastEntryOffset + this.recordSize);
	this.currentSortBufferSegment = nextMemorySegment();
	this.sortBuffer.add(this.currentSortBufferSegment);
	this.outView.set(this.currentSortBufferSegment);
	
	this.recordInstance = this.serializer.createInstance();
}
 
Example 14
Source Project: flink   File: FixedLengthRecordSorter.java    License: Apache License 2.0 4 votes vote down vote up
public FixedLengthRecordSorter(TypeSerializer<T> serializer, TypeComparator<T> comparator, 
		List<MemorySegment> memory)
{
	if (serializer == null || comparator == null || memory == null) {
		throw new NullPointerException();
	}
	
	this.serializer = serializer;
	this.comparator = comparator;
	this.useNormKeyUninverted = !comparator.invertNormalizedKey();
	
	// check the size of the first buffer and record it. all further buffers must have the same size.
	// the size must also be a power of 2
	this.totalNumBuffers = memory.size();
	if (this.totalNumBuffers < MIN_REQUIRED_BUFFERS) {
		throw new IllegalArgumentException("Normalized-Key sorter requires at least " + MIN_REQUIRED_BUFFERS + " memory buffers.");
	}
	this.segmentSize = memory.get(0).size();
	this.recordSize = serializer.getLength();
	this.numKeyBytes = this.comparator.getNormalizeKeyLen();
	
	// check that the serializer and comparator allow our operations
	if (this.recordSize <= 0) {
		throw new IllegalArgumentException("This sorter works only for fixed-length data types.");
	} else if (this.recordSize > this.segmentSize) {
		throw new IllegalArgumentException("This sorter works only for record lengths below the memory segment size.");
	} else if (!comparator.supportsSerializationWithKeyNormalization()) {
		throw new IllegalArgumentException("This sorter requires a comparator that supports serialization with key normalization.");
	}
	
	// compute the entry size and limits
	this.recordsPerSegment = segmentSize / this.recordSize;
	this.lastEntryOffset = (this.recordsPerSegment - 1) * this.recordSize;
	this.swapBuffer = new byte[this.recordSize];
	
	this.freeMemory = new ArrayList<MemorySegment>(memory);
	
	// create the buffer collections
	this.sortBuffer = new ArrayList<MemorySegment>(16);
	this.outView = new SingleSegmentOutputView(this.segmentSize);
	this.inView = new SingleSegmentInputView(this.lastEntryOffset + this.recordSize);
	this.currentSortBufferSegment = nextMemorySegment();
	this.sortBuffer.add(this.currentSortBufferSegment);
	this.outView.set(this.currentSortBufferSegment);
	
	this.recordInstance = this.serializer.createInstance();
}