org.apache.flink.runtime.io.disk.SeekableFileChannelInputView Java Examples

The following examples show how to use org.apache.flink.runtime.io.disk.SeekableFileChannelInputView. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LargeRecordHandler.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public FetchingIterator(TypeSerializer<T> serializer, MutableObjectIterator<Tuple> tupleInput,
		SeekableFileChannelInputView recordsInputs, TypeSerializer<Tuple> tupleSerializer, int pointerPos) {
	this.serializer = serializer;
	this.tupleInput = tupleInput;
	this.recordsInputs = recordsInputs;
	this.pointerPos = pointerPos;
	
	this.value = tupleSerializer.createInstance();
}
Example #2
Source File: LargeRecordHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
public FetchingIterator(TypeSerializer<T> serializer, MutableObjectIterator<Tuple> tupleInput,
		SeekableFileChannelInputView recordsInputs, TypeSerializer<Tuple> tupleSerializer, int pointerPos) {
	this.serializer = serializer;
	this.tupleInput = tupleInput;
	this.recordsInputs = recordsInputs;
	this.pointerPos = pointerPos;
	
	this.value = tupleSerializer.createInstance();
}
Example #3
Source File: LargeRecordHandler.java    From flink with Apache License 2.0 5 votes vote down vote up 
public FetchingIterator(TypeSerializer<T> serializer, MutableObjectIterator<Tuple> tupleInput,
		SeekableFileChannelInputView recordsInputs, TypeSerializer<Tuple> tupleSerializer, int pointerPos) {
	this.serializer = serializer;
	this.tupleInput = tupleInput;
	this.recordsInputs = recordsInputs;
	this.pointerPos = pointerPos;
	
	this.value = tupleSerializer.createInstance();
}
Example #4
Source File: LargeRecordHandler.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
public MutableObjectIterator<T> finishWriteAndSortKeys(List<MemorySegment> memory) throws IOException {
	if (recordsOutFile == null || keysOutFile == null) {
		throw new IllegalStateException("The LargeRecordHandler has not spilled any records");
	}
	
	// close the writers and 
	final int lastBlockBytesKeys;
	final int lastBlockBytesRecords;
	
	recordsOutFile.close();
	keysOutFile.close();
	lastBlockBytesKeys = keysOutFile.getBytesInLatestSegment();
	lastBlockBytesRecords = recordsOutFile.getBytesInLatestSegment();
	recordsOutFile = null;
	keysOutFile = null;
	
	final int pagesForReaders = Math.max(3*MIN_SEGMENTS_FOR_KEY_SPILLING, Math.min(2*MAX_SEGMENTS_FOR_KEY_SPILLING, memory.size() / 50));
	final int pagesForKeyReader = Math.min(pagesForReaders - MIN_SEGMENTS_FOR_KEY_SPILLING, MAX_SEGMENTS_FOR_KEY_SPILLING);
	final int pagesForRecordReader = pagesForReaders - pagesForKeyReader;
	
	// grab memory for the record reader
	ArrayList<MemorySegment> memForRecordReader = new ArrayList<MemorySegment>();
	ArrayList<MemorySegment> memForKeysReader = new ArrayList<MemorySegment>();
	
	for (int i = 0; i < pagesForRecordReader; i++) {
		memForRecordReader.add(memory.remove(memory.size() - 1));
	}
	for (int i = 0; i < pagesForKeyReader; i++) {
		memForKeysReader.add(memory.remove(memory.size() - 1));
	}
	
	keysReader = new FileChannelInputView(ioManager.createBlockChannelReader(keysChannel),
			memManager, memForKeysReader, lastBlockBytesKeys);
	InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer);
	
	keySorter = new UnilateralSortMerger<Tuple>(memManager, memory, ioManager, 
			keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false,
			this.executionConfig.isObjectReuseEnabled());

	// wait for the sorter to sort the keys
	MutableObjectIterator<Tuple> result;
	try {
		result = keySorter.getIterator();
	} catch (InterruptedException e) {
		throw new IOException(e);
	}
	
	recordsReader = new SeekableFileChannelInputView(ioManager, recordsChannel, memManager, memForRecordReader, lastBlockBytesRecords);
	
	return new FetchingIterator<T>(serializer, result, recordsReader, keySerializer, numKeyFields);
}
Example #5
Source File: LargeRecordHandler.java    From flink with Apache License 2.0 4 votes vote down vote up 
public MutableObjectIterator<T> finishWriteAndSortKeys(List<MemorySegment> memory) throws IOException {
	if (recordsOutFile == null || keysOutFile == null) {
		throw new IllegalStateException("The LargeRecordHandler has not spilled any records");
	}
	
	// close the writers and 
	final int lastBlockBytesKeys;
	final int lastBlockBytesRecords;
	
	recordsOutFile.close();
	keysOutFile.close();
	lastBlockBytesKeys = keysOutFile.getBytesInLatestSegment();
	lastBlockBytesRecords = recordsOutFile.getBytesInLatestSegment();
	recordsOutFile = null;
	keysOutFile = null;
	
	final int pagesForReaders = Math.max(3*MIN_SEGMENTS_FOR_KEY_SPILLING, Math.min(2*MAX_SEGMENTS_FOR_KEY_SPILLING, memory.size() / 50));
	final int pagesForKeyReader = Math.min(pagesForReaders - MIN_SEGMENTS_FOR_KEY_SPILLING, MAX_SEGMENTS_FOR_KEY_SPILLING);
	final int pagesForRecordReader = pagesForReaders - pagesForKeyReader;
	
	// grab memory for the record reader
	ArrayList<MemorySegment> memForRecordReader = new ArrayList<MemorySegment>();
	ArrayList<MemorySegment> memForKeysReader = new ArrayList<MemorySegment>();
	
	for (int i = 0; i < pagesForRecordReader; i++) {
		memForRecordReader.add(memory.remove(memory.size() - 1));
	}
	for (int i = 0; i < pagesForKeyReader; i++) {
		memForKeysReader.add(memory.remove(memory.size() - 1));
	}
	
	keysReader = new FileChannelInputView(ioManager.createBlockChannelReader(keysChannel),
			memManager, memForKeysReader, lastBlockBytesKeys);
	InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer);
	
	keySorter = new UnilateralSortMerger<Tuple>(memManager, memory, ioManager, 
			keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false,
			this.executionConfig.isObjectReuseEnabled());

	// wait for the sorter to sort the keys
	MutableObjectIterator<Tuple> result;
	try {
		result = keySorter.getIterator();
	} catch (InterruptedException e) {
		throw new IOException(e);
	}
	
	recordsReader = new SeekableFileChannelInputView(ioManager, recordsChannel, memManager, memForRecordReader, lastBlockBytesRecords);
	
	return new FetchingIterator<T>(serializer, result, recordsReader, keySerializer, numKeyFields);
}
Example #6
Source File: LargeRecordHandler.java    From flink with Apache License 2.0 4 votes vote down vote up 
public MutableObjectIterator<T> finishWriteAndSortKeys(List<MemorySegment> memory) throws IOException {
	if (recordsOutFile == null || keysOutFile == null) {
		throw new IllegalStateException("The LargeRecordHandler has not spilled any records");
	}
	
	// close the writers and 
	final int lastBlockBytesKeys;
	final int lastBlockBytesRecords;
	
	recordsOutFile.close();
	keysOutFile.close();
	lastBlockBytesKeys = keysOutFile.getBytesInLatestSegment();
	lastBlockBytesRecords = recordsOutFile.getBytesInLatestSegment();
	recordsOutFile = null;
	keysOutFile = null;
	
	final int pagesForReaders = Math.max(3*MIN_SEGMENTS_FOR_KEY_SPILLING, Math.min(2*MAX_SEGMENTS_FOR_KEY_SPILLING, memory.size() / 50));
	final int pagesForKeyReader = Math.min(pagesForReaders - MIN_SEGMENTS_FOR_KEY_SPILLING, MAX_SEGMENTS_FOR_KEY_SPILLING);
	final int pagesForRecordReader = pagesForReaders - pagesForKeyReader;
	
	// grab memory for the record reader
	ArrayList<MemorySegment> memForRecordReader = new ArrayList<MemorySegment>();
	ArrayList<MemorySegment> memForKeysReader = new ArrayList<MemorySegment>();
	
	for (int i = 0; i < pagesForRecordReader; i++) {
		memForRecordReader.add(memory.remove(memory.size() - 1));
	}
	for (int i = 0; i < pagesForKeyReader; i++) {
		memForKeysReader.add(memory.remove(memory.size() - 1));
	}
	
	keysReader = new FileChannelInputView(ioManager.createBlockChannelReader(keysChannel),
			memManager, memForKeysReader, lastBlockBytesKeys);
	InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer);
	
	keySorter = new UnilateralSortMerger<Tuple>(memManager, memory, ioManager, 
			keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false,
			this.executionConfig.isObjectReuseEnabled());

	// wait for the sorter to sort the keys
	MutableObjectIterator<Tuple> result;
	try {
		result = keySorter.getIterator();
	} catch (InterruptedException e) {
		throw new IOException(e);
	}
	
	recordsReader = new SeekableFileChannelInputView(ioManager, recordsChannel, memManager, memForRecordReader, lastBlockBytesRecords);
	
	return new FetchingIterator<T>(serializer, result, recordsReader, keySerializer, numKeyFields);
}