Java Code Examples for org.apache.flink.runtime.io.disk.FileChannelInputView

The following examples show how to use org.apache.flink.runtime.io.disk.FileChannelInputView. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: LargeRecordHandler.java    License: Apache License 2.0 4 votes vote down vote up
public MutableObjectIterator<T> finishWriteAndSortKeys(List<MemorySegment> memory) throws IOException {
	if (recordsOutFile == null || keysOutFile == null) {
		throw new IllegalStateException("The LargeRecordHandler has not spilled any records");
	}
	
	// close the writers and 
	final int lastBlockBytesKeys;
	final int lastBlockBytesRecords;
	
	recordsOutFile.close();
	keysOutFile.close();
	lastBlockBytesKeys = keysOutFile.getBytesInLatestSegment();
	lastBlockBytesRecords = recordsOutFile.getBytesInLatestSegment();
	recordsOutFile = null;
	keysOutFile = null;
	
	final int pagesForReaders = Math.max(3*MIN_SEGMENTS_FOR_KEY_SPILLING, Math.min(2*MAX_SEGMENTS_FOR_KEY_SPILLING, memory.size() / 50));
	final int pagesForKeyReader = Math.min(pagesForReaders - MIN_SEGMENTS_FOR_KEY_SPILLING, MAX_SEGMENTS_FOR_KEY_SPILLING);
	final int pagesForRecordReader = pagesForReaders - pagesForKeyReader;
	
	// grab memory for the record reader
	ArrayList<MemorySegment> memForRecordReader = new ArrayList<MemorySegment>();
	ArrayList<MemorySegment> memForKeysReader = new ArrayList<MemorySegment>();
	
	for (int i = 0; i < pagesForRecordReader; i++) {
		memForRecordReader.add(memory.remove(memory.size() - 1));
	}
	for (int i = 0; i < pagesForKeyReader; i++) {
		memForKeysReader.add(memory.remove(memory.size() - 1));
	}
	
	keysReader = new FileChannelInputView(ioManager.createBlockChannelReader(keysChannel),
			memManager, memForKeysReader, lastBlockBytesKeys);
	InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer);
	
	keySorter = new UnilateralSortMerger<Tuple>(memManager, memory, ioManager, 
			keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false,
			this.executionConfig.isObjectReuseEnabled());

	// wait for the sorter to sort the keys
	MutableObjectIterator<Tuple> result;
	try {
		result = keySorter.getIterator();
	} catch (InterruptedException e) {
		throw new IOException(e);
	}
	
	recordsReader = new SeekableFileChannelInputView(ioManager, recordsChannel, memManager, memForRecordReader, lastBlockBytesRecords);
	
	return new FetchingIterator<T>(serializer, result, recordsReader, keySerializer, numKeyFields);
}
 
Example 2
Source Project: flink   Source File: LargeRecordHandler.java    License: Apache License 2.0 4 votes vote down vote up
public MutableObjectIterator<T> finishWriteAndSortKeys(List<MemorySegment> memory) throws IOException {
	if (recordsOutFile == null || keysOutFile == null) {
		throw new IllegalStateException("The LargeRecordHandler has not spilled any records");
	}
	
	// close the writers and 
	final int lastBlockBytesKeys;
	final int lastBlockBytesRecords;
	
	recordsOutFile.close();
	keysOutFile.close();
	lastBlockBytesKeys = keysOutFile.getBytesInLatestSegment();
	lastBlockBytesRecords = recordsOutFile.getBytesInLatestSegment();
	recordsOutFile = null;
	keysOutFile = null;
	
	final int pagesForReaders = Math.max(3*MIN_SEGMENTS_FOR_KEY_SPILLING, Math.min(2*MAX_SEGMENTS_FOR_KEY_SPILLING, memory.size() / 50));
	final int pagesForKeyReader = Math.min(pagesForReaders - MIN_SEGMENTS_FOR_KEY_SPILLING, MAX_SEGMENTS_FOR_KEY_SPILLING);
	final int pagesForRecordReader = pagesForReaders - pagesForKeyReader;
	
	// grab memory for the record reader
	ArrayList<MemorySegment> memForRecordReader = new ArrayList<MemorySegment>();
	ArrayList<MemorySegment> memForKeysReader = new ArrayList<MemorySegment>();
	
	for (int i = 0; i < pagesForRecordReader; i++) {
		memForRecordReader.add(memory.remove(memory.size() - 1));
	}
	for (int i = 0; i < pagesForKeyReader; i++) {
		memForKeysReader.add(memory.remove(memory.size() - 1));
	}
	
	keysReader = new FileChannelInputView(ioManager.createBlockChannelReader(keysChannel),
			memManager, memForKeysReader, lastBlockBytesKeys);
	InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer);
	
	keySorter = new UnilateralSortMerger<Tuple>(memManager, memory, ioManager, 
			keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false,
			this.executionConfig.isObjectReuseEnabled());

	// wait for the sorter to sort the keys
	MutableObjectIterator<Tuple> result;
	try {
		result = keySorter.getIterator();
	} catch (InterruptedException e) {
		throw new IOException(e);
	}
	
	recordsReader = new SeekableFileChannelInputView(ioManager, recordsChannel, memManager, memForRecordReader, lastBlockBytesRecords);
	
	return new FetchingIterator<T>(serializer, result, recordsReader, keySerializer, numKeyFields);
}
 
Example 3
Source Project: flink   Source File: LargeRecordHandler.java    License: Apache License 2.0 4 votes vote down vote up
public MutableObjectIterator<T> finishWriteAndSortKeys(List<MemorySegment> memory) throws IOException {
	if (recordsOutFile == null || keysOutFile == null) {
		throw new IllegalStateException("The LargeRecordHandler has not spilled any records");
	}
	
	// close the writers and 
	final int lastBlockBytesKeys;
	final int lastBlockBytesRecords;
	
	recordsOutFile.close();
	keysOutFile.close();
	lastBlockBytesKeys = keysOutFile.getBytesInLatestSegment();
	lastBlockBytesRecords = recordsOutFile.getBytesInLatestSegment();
	recordsOutFile = null;
	keysOutFile = null;
	
	final int pagesForReaders = Math.max(3*MIN_SEGMENTS_FOR_KEY_SPILLING, Math.min(2*MAX_SEGMENTS_FOR_KEY_SPILLING, memory.size() / 50));
	final int pagesForKeyReader = Math.min(pagesForReaders - MIN_SEGMENTS_FOR_KEY_SPILLING, MAX_SEGMENTS_FOR_KEY_SPILLING);
	final int pagesForRecordReader = pagesForReaders - pagesForKeyReader;
	
	// grab memory for the record reader
	ArrayList<MemorySegment> memForRecordReader = new ArrayList<MemorySegment>();
	ArrayList<MemorySegment> memForKeysReader = new ArrayList<MemorySegment>();
	
	for (int i = 0; i < pagesForRecordReader; i++) {
		memForRecordReader.add(memory.remove(memory.size() - 1));
	}
	for (int i = 0; i < pagesForKeyReader; i++) {
		memForKeysReader.add(memory.remove(memory.size() - 1));
	}
	
	keysReader = new FileChannelInputView(ioManager.createBlockChannelReader(keysChannel),
			memManager, memForKeysReader, lastBlockBytesKeys);
	InputViewIterator<Tuple> keyIterator = new InputViewIterator<Tuple>(keysReader, keySerializer);
	
	keySorter = new UnilateralSortMerger<Tuple>(memManager, memory, ioManager, 
			keyIterator, memoryOwner, keySerializerFactory, keyComparator, 1, maxFilehandles, 1.0f, false,
			this.executionConfig.isObjectReuseEnabled());

	// wait for the sorter to sort the keys
	MutableObjectIterator<Tuple> result;
	try {
		result = keySorter.getIterator();
	} catch (InterruptedException e) {
		throw new IOException(e);
	}
	
	recordsReader = new SeekableFileChannelInputView(ioManager, recordsChannel, memManager, memForRecordReader, lastBlockBytesRecords);
	
	return new FetchingIterator<T>(serializer, result, recordsReader, keySerializer, numKeyFields);
}