Java Code Examples for org.apache.flink.runtime.io.disk.iomanager.FileIOChannel

The following examples show how to use org.apache.flink.runtime.io.disk.iomanager.FileIOChannel. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: FileChannelUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static BlockChannelWriter<MemorySegment> createBlockChannelWriter(
		IOManager ioManager,
		FileIOChannel.ID channel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		return new CompressedBlockChannelWriter(
				ioManager,
				channel,
				bufferReturnQueue,
				compressionCodecFactory,
				compressionBlockSize,
				segmentSize
		);
	} else {
		return ioManager.createBlockChannelWriter(channel, bufferReturnQueue);
	}
}
 
Example 2
Source Project: Flink-CEPplus   Source File: HashPartition.java    License: Apache License 2.0 6 votes vote down vote up
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();
	
	if (!isInMemory()) {
		// close the channel. note that in the spilled case, the build-side-buffer will have sent off
		// the last segment and it will be returned to the write-behind-buffer queue.
		this.buildSideChannel.close();
		
		// create the channel for the probe side and claim one buffer for it
		this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
		// creating the ChannelWriterOutputView without memory will cause it to draw one segment from the
		// write behind queue, which is the spare segment we had above.
		this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
	}
}
 
Example 3
Source Project: flink   Source File: BufferedKVExternalSorter.java    License: Apache License 2.0 6 votes vote down vote up
public MutableObjectIterator<Tuple2<BinaryRowData, BinaryRowData>> getKVIterator() throws IOException {
	// 1. merge if more than maxNumFile
	// merge channels until sufficient file handles are available
	List<ChannelWithMeta> channelIDs = this.channelIDs;
	while (!closed && channelIDs.size() > this.maxNumFileHandles) {
		channelIDs = merger.mergeChannelList(channelIDs);
	}

	// 2. final merge
	List<FileIOChannel> openChannels = new ArrayList<>();
	BinaryMergeIterator<Tuple2<BinaryRowData, BinaryRowData>> iterator =
			merger.getMergingIterator(channelIDs, openChannels);
	channelManager.addOpenChannels(openChannels);

	return iterator;
}
 
Example 4
Source Project: flink   Source File: ReOpenableHashPartition.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Spills this partition to disk. This method is invoked once after the initial open() method
 * 
 * @return Number of memorySegments in the writeBehindBuffers!
 */
int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException {
	this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap
	this.initialBuildSideChannel = targetChannel;
	
	initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers);
	
	final int numSegments = this.partitionBuffers.length;
	for (int i = 0; i < numSegments; i++) {
		initialBuildSideWriter.writeBlock(partitionBuffers[i]);
	}
	this.partitionBuffers = null;
	initialBuildSideWriter.close();
	// num partitions are now in the writeBehindBuffers. We propagate this information back
	return numSegments;
	
}
 
Example 5
Source Project: flink   Source File: LongHashPartition.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * After build phase.
 *
 * @return build spill return buffer, if have spilled, it returns the current write buffer,
 * because it was used all the time in build phase, so it can only be returned at this time.
 */
int finalizeBuildPhase(
		IOManager ioAccess,
		FileIOChannel.Enumerator probeChannelEnumerator) throws IOException {
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();

	if (!isInMemory()) {
		// close the channel.
		this.buildSideChannel.close();

		this.probeSideBuffer = FileChannelUtil.createOutputView(
				ioAccess,
				probeChannelEnumerator.next(),
				longTable.compressionEnable(),
				longTable.compressionCodecFactory(),
				longTable.compressionBlockSize(),
				segmentSize);
		return 1;
	} else {
		return 0;
	}
}
 
Example 6
Source Project: flink   Source File: BufferedKVExternalSorter.java    License: Apache License 2.0 6 votes vote down vote up
public MutableObjectIterator<Tuple2<BinaryRow, BinaryRow>> getKVIterator() throws IOException {
	// 1. merge if more than maxNumFile
	// merge channels until sufficient file handles are available
	List<ChannelWithMeta> channelIDs = this.channelIDs;
	while (!closed && channelIDs.size() > this.maxNumFileHandles) {
		channelIDs = merger.mergeChannelList(channelIDs);
	}

	// 2. final merge
	List<FileIOChannel> openChannels = new ArrayList<>();
	BinaryMergeIterator<Tuple2<BinaryRow, BinaryRow>> iterator =
			merger.getMergingIterator(channelIDs, openChannels);
	channelManager.addOpenChannels(openChannels);

	return iterator;
}
 
Example 7
Source Project: flink   Source File: AbstractBinaryExternalMerger.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns an iterator that iterates over the merged result from all given channels.
 *
 * @param channelIDs    The channels that are to be merged and returned.
 * @return An iterator over the merged records of the input channels.
 * @throws IOException Thrown, if the readers encounter an I/O problem.
 */
public BinaryMergeIterator<Entry> getMergingIterator(
		List<ChannelWithMeta> channelIDs,
		List<FileIOChannel> openChannels)
		throws IOException {
	// create one iterator per channel id
	if (LOG.isDebugEnabled()) {
		LOG.debug("Performing merge of " + channelIDs.size() + " sorted streams.");
	}

	final List<MutableObjectIterator<Entry>> iterators = new ArrayList<>(channelIDs.size() + 1);

	for (ChannelWithMeta channel : channelIDs) {
		AbstractChannelReaderInputView view = FileChannelUtil.createInputView(
				ioManager, channel, openChannels, compressionEnable, compressionCodecFactory,
				compressionBlockSize, pageSize);
		iterators.add(channelReaderInputViewIterator(view));
	}

	return new BinaryMergeIterator<>(
			iterators, mergeReusedEntries(channelIDs.size()), mergeComparator());
}
 
Example 8
Source Project: flink   Source File: LongHashPartition.java    License: Apache License 2.0 6 votes vote down vote up
int spillPartition(IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException {
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition with less than two buffers.");
	}

	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = FileChannelUtil.createBlockChannelWriter(
			ioAccess,
			targetChannel,
			bufferReturnQueue,
			longTable.compressionEnable(),
			longTable.compressionCodecFactory(),
			longTable.compressionBlockSize(),
			segmentSize);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 
Example 9
Source Project: flink   Source File: LongHashPartition.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * After build phase.
 *
 * @return build spill return buffer, if have spilled, it returns the current write buffer,
 * because it was used all the time in build phase, so it can only be returned at this time.
 */
int finalizeBuildPhase(
		IOManager ioAccess,
		FileIOChannel.Enumerator probeChannelEnumerator) throws IOException {
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();

	if (!isInMemory()) {
		// close the channel.
		this.buildSideChannel.close();

		this.probeSideBuffer = FileChannelUtil.createOutputView(
				ioAccess,
				probeChannelEnumerator.next(),
				longTable.compressionEnable(),
				longTable.compressionCodecFactory(),
				longTable.compressionBlockSize(),
				segmentSize);
		return 1;
	} else {
		return 0;
	}
}
 
Example 10
Source Project: flink   Source File: BinaryHashPartition.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * After build phase.
 * @return build spill return buffer, if have spilled, it returns the current write buffer,
 * because it was used all the time in build phase, so it can only be returned at this time.
 */
int finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator) throws IOException {
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();

	if (!isInMemory()) {
		// close the channel.
		this.buildSideChannel.close();

		this.probeSideBuffer = FileChannelUtil.createOutputView(
				ioAccess,
				probeChannelEnumerator.next(),
				compressionEnable,
				compressionCodecFactory,
				compressionBlockSize,
				memorySegmentSize);
		return 1;
	} else {
		return 0;
	}
}
 
Example 11
Source Project: flink   Source File: FileChannelUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static AbstractChannelWriterOutputView createOutputView(
		IOManager ioManager,
		FileIOChannel.ID channel,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		BufferFileWriter bufferWriter = ioManager.createBufferFileWriter(channel);
		return new CompressedHeaderlessChannelWriterOutputView(
				bufferWriter,
				compressionCodecFactory,
				compressionBlockSize);
	} else {
		BlockChannelWriter<MemorySegment> blockWriter =
				ioManager.createBlockChannelWriter(channel);
		return new HeaderlessChannelWriterOutputView(
				blockWriter,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				segmentSize
		);
	}
}
 
Example 12
Source Project: flink   Source File: FileChannelUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static BlockChannelReader<MemorySegment> createBlockChannelReader(
		IOManager ioManager,
		FileIOChannel.ID channel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		return new CompressedBlockChannelReader(
				ioManager,
				channel,
				bufferReturnQueue,
				compressionCodecFactory,
				compressionBlockSize,
				segmentSize
		);
	} else {
		return ioManager.createBlockChannelReader(channel, bufferReturnQueue);
	}
}
 
Example 13
public CompressedHeaderlessChannelReaderInputView(
		FileIOChannel.ID id,
		IOManager ioManager,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int numBlocks) throws IOException {
	super(0);
	this.numBlocksRemaining = numBlocks;
	this.reader = ioManager.createBufferFileReader(id, this);
	uncompressedBuffer = MemorySegmentFactory.wrap(new byte[compressionBlockSize]);
	decompressor = compressionCodecFactory.getDecompressor();
	cause = new AtomicReference<>();

	BlockCompressor compressor = compressionCodecFactory.getCompressor();
	for (int i = 0; i < 2; i++) {
		MemorySegment segment = MemorySegmentFactory.wrap(new byte[compressor.getMaxCompressedSize(
				compressionBlockSize)]);
		reader.readInto(new NetworkBuffer(segment, this));
	}
}
 
Example 14
Source Project: flink   Source File: ReOpenableHashPartition.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Spills this partition to disk. This method is invoked once after the initial open() method
 * 
 * @return Number of memorySegments in the writeBehindBuffers!
 */
int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException {
	this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap
	this.initialBuildSideChannel = targetChannel;
	
	initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers);
	
	final int numSegments = this.partitionBuffers.length;
	for (int i = 0; i < numSegments; i++) {
		initialBuildSideWriter.writeBlock(partitionBuffers[i]);
	}
	this.partitionBuffers = null;
	initialBuildSideWriter.close();
	// num partitions are now in the writeBehindBuffers. We propagate this information back
	return numSegments;
	
}
 
Example 15
Source Project: flink   Source File: HashPartition.java    License: Apache License 2.0 6 votes vote down vote up
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();
	
	if (!isInMemory()) {
		// close the channel. note that in the spilled case, the build-side-buffer will have sent off
		// the last segment and it will be returned to the write-behind-buffer queue.
		this.buildSideChannel.close();
		
		// create the channel for the probe side and claim one buffer for it
		this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
		// creating the ChannelWriterOutputView without memory will cause it to draw one segment from the
		// write behind queue, which is the spare segment we had above.
		this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
	}
}
 
Example 16
Source Project: flink   Source File: UnilateralSortMerger.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns an iterator that iterates over the merged result from all given channels.
 * 
 * @param channelIDs The channels that are to be merged and returned.
 * @param inputSegments The buffers to be used for reading. The list contains for each channel one
 *                      list of input segments. The size of the <code>inputSegments</code> list must be equal to
 *                      that of the <code>channelIDs</code> list.
 * @return An iterator over the merged records of the input channels.
 * @throws IOException Thrown, if the readers encounter an I/O problem.
 */
protected final MergeIterator<E> getMergingIterator(final List<ChannelWithBlockCount> channelIDs,
		final List<List<MemorySegment>> inputSegments, List<FileIOChannel> readerList, MutableObjectIterator<E> largeRecords)
	throws IOException
{
	// create one iterator per channel id
	if (LOG.isDebugEnabled()) {
		LOG.debug("Performing merge of " + channelIDs.size() + " sorted streams.");
	}
	
	final List<MutableObjectIterator<E>> iterators = new ArrayList<MutableObjectIterator<E>>(channelIDs.size() + 1);
	
	for (int i = 0; i < channelIDs.size(); i++) {
		final ChannelWithBlockCount channel = channelIDs.get(i);
		final List<MemorySegment> segsForChannel = inputSegments.get(i);
		
		// create a reader. if there are multiple segments for the reader, issue multiple together per I/O request
		final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel.getChannel());
			
		readerList.add(reader);
		registerOpenChannelToBeRemovedAtShudown(reader);
		unregisterChannelToBeRemovedAtShudown(channel.getChannel());
		
		// wrap channel reader as a view, to get block spanning record deserialization
		final ChannelReaderInputView inView = new ChannelReaderInputView(reader, segsForChannel, 
																	channel.getBlockCount(), false);
		iterators.add(new ChannelReaderInputViewIterator<E>(inView, null, this.serializer));
	}
	
	if (largeRecords != null) {
		iterators.add(largeRecords);
	}

	return new MergeIterator<E>(iterators, this.comparator);
}
 
Example 17
Source Project: flink   Source File: BufferedKVExternalSorter.java    License: Apache License 2.0 5 votes vote down vote up
public void sortAndSpill(
		ArrayList<MemorySegment> recordBufferSegments,
		long numElements,
		MemorySegmentPool pool) throws IOException {

	// 1. sort buffer
	BinaryKVInMemorySortBuffer buffer =
			BinaryKVInMemorySortBuffer.createBuffer(
					nKeyComputer, keySerializer, valueSerializer, comparator,
					recordBufferSegments, numElements, pool);
	this.sorter.sort(buffer);

	// 2. spill
	FileIOChannel.ID channel = enumerator.next();
	channelManager.addChannel(channel);

	AbstractChannelWriterOutputView output = null;
	int bytesInLastBuffer;
	int blockCount;
	try {
		numSpillFiles++;
		output = FileChannelUtil.createOutputView(ioManager, channel, compressionEnable,
				compressionCodecFactory, compressionBlockSize, pageSize);
		buffer.writeToOutput(output);
		spillInBytes += output.getNumBytes();
		spillInCompressedBytes += output.getNumCompressedBytes();
		bytesInLastBuffer = output.close();
		blockCount = output.getBlockCount();
		LOG.info("here spill the {}th kv external buffer data with {} bytes and {} compressed bytes",
				numSpillFiles, spillInBytes, spillInCompressedBytes);
	} catch (IOException e) {
		if (output != null) {
			output.close();
			output.getChannel().deleteChannel();
		}
		throw e;
	}
	channelIDs.add(new ChannelWithMeta(channel, blockCount, bytesInLastBuffer));
}
 
Example 18
Source Project: flink   Source File: BaseHybridHashTable.java    License: Apache License 2.0 5 votes vote down vote up
protected HeaderlessChannelReaderInputView createInputView(FileIOChannel.ID id, int blockCount, int lastSegmentLimit) throws IOException {
	BlockChannelReader<MemorySegment> inReader = FileChannelUtil.createBlockChannelReader(
			ioManager, id, new LinkedBlockingQueue<>(),
			compressionEnable, compressionCodecFactory, compressionBlockSize, segmentSize);
	return new HeaderlessChannelReaderInputView(inReader,
												Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)),
												blockCount, lastSegmentLimit, false);

}
 
Example 19
Source Project: flink   Source File: SpillChannelManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Open File channels.
 */
public synchronized void addOpenChannels(List<FileIOChannel> toOpen) {
	checkArgument(!closed);
	for (FileIOChannel channel : toOpen) {
		openChannels.add(channel);
		channels.remove(channel.getChannelID());
	}
}
 
Example 20
Source Project: Flink-CEPplus   Source File: SpillableSubpartitionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public BufferFileWriter createBufferFileWriter(FileIOChannel.ID channelID)
		throws IOException {
	BufferFileWriter bufferFileWriter = super.createBufferFileWriter(channelID);
	bufferFileWriter.close();
	return bufferFileWriter;
}
 
Example 21
Source Project: Flink-CEPplus   Source File: SpillableSubpartitionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public BufferFileWriter createBufferFileWriter(FileIOChannel.ID channelID) throws IOException {
	blockLatch.countDown();
	try {
		doneLatch.await();
	} catch (InterruptedException e) {
		throw new IOException("Blocking operation was interrupted.", e);
	}

	return super.createBufferFileWriter(channelID);
}
 
Example 22
Source Project: Flink-CEPplus   Source File: FileChannelStreamsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCloseAndDeleteOutputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		
		FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
		new StringValue("Some test text").write(out);
		
		// close for the first time, make sure all memory returns
		out.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		out.close();
		
		// delete, make sure file is removed
		out.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
 
Example 23
Source Project: flink   Source File: ResettableExternalBuffer.java    License: Apache License 2.0 5 votes vote down vote up
private void spill() throws IOException {
	FileIOChannel.ID channel = ioManager.createChannel();

	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	int numRecordBuffers = inMemoryBuffer.getNumRecordBuffers();
	ArrayList<MemorySegment> segments = inMemoryBuffer.getRecordBufferSegments();
	try {
		// spill in memory buffer in zero-copy.
		for (int i = 0; i < numRecordBuffers; i++) {
			writer.writeBlock(segments.get(i));
		}
		LOG.info("here spill the reset buffer data with {} bytes", writer.getSize());
		writer.close();
	} catch (IOException e) {
		writer.closeAndDelete();
		throw e;
	}

	spillSize += numRecordBuffers * segmentSize;
	spilledChannelIDs.add(new ChannelWithMeta(
		channel,
		inMemoryBuffer.getNumRecordBuffers(),
		inMemoryBuffer.getNumBytesInLastBuffer()));
	this.spilledChannelRowOffsets.add(numRows);

	inMemoryBuffer.reset();
}
 
Example 24
Source Project: flink   Source File: SpillChannelManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Open File channels.
 */
public synchronized void addOpenChannels(List<FileIOChannel> toOpen) {
	checkArgument(!closed);
	for (FileIOChannel channel : toOpen) {
		openChannels.add(channel);
		channels.remove(channel.getChannelID());
	}
}
 
Example 25
Source Project: flink   Source File: BinaryHashPartition.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Spills this partition to disk and sets it up such that it continues spilling records that are
 * added to
 * it. The spilling process must free at least one buffer, either in the partition's record
 * buffers, or in
 * the memory segments for overflow buckets.
 * The partition immediately takes back one buffer to use it for further spilling.
 *
 * @param ioAccess      The I/O manager to be used to create a writer to disk.
 * @param targetChannel The id of the target channel for this partition.
 * @return The number of buffers that were freed by spilling this partition.
 * @throws IOException Thrown, if the writing failed.
 */
int spillPartition(
		IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException {
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition with less than two buffers.");
	}

	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = FileChannelUtil.createBlockChannelWriter(
			ioAccess,
			targetChannel,
			bufferReturnQueue,
			compressionEnable,
			compressionCodecFactory,
			compressionBlockSize,
			memorySegmentSize);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 
Example 26
Source Project: flink   Source File: SeekableFileChannelInputView.java    License: Apache License 2.0 5 votes vote down vote up
public SeekableFileChannelInputView(IOManager ioManager, FileIOChannel.ID channelId, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(ioManager);
	checkNotNull(channelId);
	checkNotNull(memManager);
	checkNotNull(memory);
	
	this.ioManager = ioManager;
	this.channelId = channelId;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	this.segmentSize = memManager.getPageSize();
	
	this.reader = ioManager.createBlockChannelReader(channelId);
	
	try {
		final long channelLength = reader.getSize();
		
		final int blockCount =  MathUtils.checkedDownCast(channelLength / segmentSize);
		this.numBlocksTotal = (channelLength % segmentSize == 0) ? blockCount : blockCount + 1;

		this.numBlocksRemaining = this.numBlocksTotal;
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}
 
Example 27
Source Project: flink   Source File: BaseHybridHashTable.java    License: Apache License 2.0 5 votes vote down vote up
protected HeaderlessChannelReaderInputView createInputView(FileIOChannel.ID id, int blockCount, int lastSegmentLimit) throws IOException {
	BlockChannelReader<MemorySegment> inReader = FileChannelUtil.createBlockChannelReader(
			ioManager, id, new LinkedBlockingQueue<>(),
			compressionEnable, compressionCodecFactory, compressionBlockSize, segmentSize);
	return new HeaderlessChannelReaderInputView(inReader,
												Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)),
												blockCount, lastSegmentLimit, false);

}
 
Example 28
Source Project: flink   Source File: FileChannelUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static AbstractChannelReaderInputView createInputView(
		IOManager ioManager,
		ChannelWithMeta channel,
		List<FileIOChannel> channels,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		CompressedHeaderlessChannelReaderInputView in =
				new CompressedHeaderlessChannelReaderInputView(
						channel.getChannel(),
						ioManager,
						compressionCodecFactory,
						compressionBlockSize,
						channel.getBlockCount()
				);
		channels.add(in.getReader());
		return in;
	} else {
		BlockChannelReader<MemorySegment> reader =
				ioManager.createBlockChannelReader(channel.getChannel());
		channels.add(reader);
		return new HeaderlessChannelReaderInputView(
				reader,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				channel.getBlockCount(),
				channel.getNumBytesInLastBlock(), false
		);
	}
}
 
Example 29
Source Project: flink   Source File: HashPartition.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Spills this partition to disk and sets it up such that it continues spilling records that are added to
 * it. The spilling process must free at least one buffer, either in the partition's record buffers, or in
 * the memory segments for overflow buckets.
 * The partition immediately takes back one buffer to use it for further spilling.
 * 
 * @param target The list to which memory segments from overflow buckets are added.
 * @param ioAccess The I/O manager to be used to create a writer to disk.
 * @param targetChannel The id of the target channel for this partition.
 * @return The number of buffers that were freed by spilling this partition.
 * @throws IOException Thrown, if the writing failed.
 */
public int spillPartition(List<MemorySegment> target, IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
			"Request to spill a partition with less than two buffers.");
	}
	
	// return the memory from the overflow segments
	for (int i = 0; i < this.numOverflowSegments; i++) {
		target.add(this.overflowSegments[i]);
	}
	this.overflowSegments = null;
	this.numOverflowSegments = 0;
	this.nextOverflowBucket = 0;
	
	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = ioAccess.createBlockChannelWriter(targetChannel, bufferReturnQueue);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 
Example 30
Source Project: flink   Source File: ChannelReaderInputViewIterator.java    License: Apache License 2.0 5 votes vote down vote up
public ChannelReaderInputViewIterator(IOManager ioAccess, FileIOChannel.ID channel,  LinkedBlockingQueue<MemorySegment> returnQueue,
		List<MemorySegment> segments, List<MemorySegment> freeMemTarget, TypeSerializer<E> accessors, int numBlocks)
throws IOException
{
	this(ioAccess.createBlockChannelReader(channel, returnQueue), returnQueue,
		segments, freeMemTarget, accessors, numBlocks);
}