org.apache.flink.runtime.io.disk.iomanager.FileIOChannel Java Examples

The following examples show how to use org.apache.flink.runtime.io.disk.iomanager.FileIOChannel. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CompressedHeaderlessChannelReaderInputView.java    From flink with Apache License 2.0 6 votes vote down vote up
public CompressedHeaderlessChannelReaderInputView(
		FileIOChannel.ID id,
		IOManager ioManager,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int numBlocks) throws IOException {
	super(0);
	this.numBlocksRemaining = numBlocks;
	this.reader = ioManager.createBufferFileReader(id, this);
	uncompressedBuffer = MemorySegmentFactory.wrap(new byte[compressionBlockSize]);
	decompressor = compressionCodecFactory.getDecompressor();
	cause = new AtomicReference<>();

	BlockCompressor compressor = compressionCodecFactory.getCompressor();
	for (int i = 0; i < 2; i++) {
		MemorySegment segment = MemorySegmentFactory.wrap(new byte[compressor.getMaxCompressedSize(
				compressionBlockSize)]);
		reader.readInto(new NetworkBuffer(segment, this));
	}
}
 
Example #2
Source File: HashPartition.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();
	
	if (!isInMemory()) {
		// close the channel. note that in the spilled case, the build-side-buffer will have sent off
		// the last segment and it will be returned to the write-behind-buffer queue.
		this.buildSideChannel.close();
		
		// create the channel for the probe side and claim one buffer for it
		this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
		// creating the ChannelWriterOutputView without memory will cause it to draw one segment from the
		// write behind queue, which is the spare segment we had above.
		this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
	}
}
 
Example #3
Source File: FileChannelUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
public static BlockChannelWriter<MemorySegment> createBlockChannelWriter(
		IOManager ioManager,
		FileIOChannel.ID channel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		return new CompressedBlockChannelWriter(
				ioManager,
				channel,
				bufferReturnQueue,
				compressionCodecFactory,
				compressionBlockSize,
				segmentSize
		);
	} else {
		return ioManager.createBlockChannelWriter(channel, bufferReturnQueue);
	}
}
 
Example #4
Source File: BufferedKVExternalSorter.java    From flink with Apache License 2.0 6 votes vote down vote up
public MutableObjectIterator<Tuple2<BinaryRowData, BinaryRowData>> getKVIterator() throws IOException {
	// 1. merge if more than maxNumFile
	// merge channels until sufficient file handles are available
	List<ChannelWithMeta> channelIDs = this.channelIDs;
	while (!closed && channelIDs.size() > this.maxNumFileHandles) {
		channelIDs = merger.mergeChannelList(channelIDs);
	}

	// 2. final merge
	List<FileIOChannel> openChannels = new ArrayList<>();
	BinaryMergeIterator<Tuple2<BinaryRowData, BinaryRowData>> iterator =
			merger.getMergingIterator(channelIDs, openChannels);
	channelManager.addOpenChannels(openChannels);

	return iterator;
}
 
Example #5
Source File: ReOpenableHashPartition.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Spills this partition to disk. This method is invoked once after the initial open() method
 * 
 * @return Number of memorySegments in the writeBehindBuffers!
 */
int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException {
	this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap
	this.initialBuildSideChannel = targetChannel;
	
	initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers);
	
	final int numSegments = this.partitionBuffers.length;
	for (int i = 0; i < numSegments; i++) {
		initialBuildSideWriter.writeBlock(partitionBuffers[i]);
	}
	this.partitionBuffers = null;
	initialBuildSideWriter.close();
	// num partitions are now in the writeBehindBuffers. We propagate this information back
	return numSegments;
	
}
 
Example #6
Source File: LongHashPartition.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * After build phase.
 *
 * @return build spill return buffer, if have spilled, it returns the current write buffer,
 * because it was used all the time in build phase, so it can only be returned at this time.
 */
int finalizeBuildPhase(
		IOManager ioAccess,
		FileIOChannel.Enumerator probeChannelEnumerator) throws IOException {
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();

	if (!isInMemory()) {
		// close the channel.
		this.buildSideChannel.close();

		this.probeSideBuffer = FileChannelUtil.createOutputView(
				ioAccess,
				probeChannelEnumerator.next(),
				longTable.compressionEnable(),
				longTable.compressionCodecFactory(),
				longTable.compressionBlockSize(),
				segmentSize);
		return 1;
	} else {
		return 0;
	}
}
 
Example #7
Source File: HashPartition.java    From flink with Apache License 2.0 6 votes vote down vote up
public void finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();
	
	if (!isInMemory()) {
		// close the channel. note that in the spilled case, the build-side-buffer will have sent off
		// the last segment and it will be returned to the write-behind-buffer queue.
		this.buildSideChannel.close();
		
		// create the channel for the probe side and claim one buffer for it
		this.probeSideChannel = ioAccess.createBlockChannelWriter(probeChannelEnumerator.next(), bufferReturnQueue);
		// creating the ChannelWriterOutputView without memory will cause it to draw one segment from the
		// write behind queue, which is the spare segment we had above.
		this.probeSideBuffer = new ChannelWriterOutputView(this.probeSideChannel, this.memorySegmentSize);
	}
}
 
Example #8
Source File: ReOpenableHashPartition.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Spills this partition to disk. This method is invoked once after the initial open() method
 * 
 * @return Number of memorySegments in the writeBehindBuffers!
 */
int spillInMemoryPartition(FileIOChannel.ID targetChannel, IOManager ioManager, LinkedBlockingQueue<MemorySegment> writeBehindBuffers) throws IOException {
	this.initialPartitionBuffersCount = partitionBuffers.length; // for ReOpenableHashMap
	this.initialBuildSideChannel = targetChannel;
	
	initialBuildSideWriter = ioManager.createBlockChannelWriter(targetChannel, writeBehindBuffers);
	
	final int numSegments = this.partitionBuffers.length;
	for (int i = 0; i < numSegments; i++) {
		initialBuildSideWriter.writeBlock(partitionBuffers[i]);
	}
	this.partitionBuffers = null;
	initialBuildSideWriter.close();
	// num partitions are now in the writeBehindBuffers. We propagate this information back
	return numSegments;
	
}
 
Example #9
Source File: BufferedKVExternalSorter.java    From flink with Apache License 2.0 6 votes vote down vote up
public MutableObjectIterator<Tuple2<BinaryRow, BinaryRow>> getKVIterator() throws IOException {
	// 1. merge if more than maxNumFile
	// merge channels until sufficient file handles are available
	List<ChannelWithMeta> channelIDs = this.channelIDs;
	while (!closed && channelIDs.size() > this.maxNumFileHandles) {
		channelIDs = merger.mergeChannelList(channelIDs);
	}

	// 2. final merge
	List<FileIOChannel> openChannels = new ArrayList<>();
	BinaryMergeIterator<Tuple2<BinaryRow, BinaryRow>> iterator =
			merger.getMergingIterator(channelIDs, openChannels);
	channelManager.addOpenChannels(openChannels);

	return iterator;
}
 
Example #10
Source File: AbstractBinaryExternalMerger.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Returns an iterator that iterates over the merged result from all given channels.
 *
 * @param channelIDs    The channels that are to be merged and returned.
 * @return An iterator over the merged records of the input channels.
 * @throws IOException Thrown, if the readers encounter an I/O problem.
 */
public BinaryMergeIterator<Entry> getMergingIterator(
		List<ChannelWithMeta> channelIDs,
		List<FileIOChannel> openChannels)
		throws IOException {
	// create one iterator per channel id
	if (LOG.isDebugEnabled()) {
		LOG.debug("Performing merge of " + channelIDs.size() + " sorted streams.");
	}

	final List<MutableObjectIterator<Entry>> iterators = new ArrayList<>(channelIDs.size() + 1);

	for (ChannelWithMeta channel : channelIDs) {
		AbstractChannelReaderInputView view = FileChannelUtil.createInputView(
				ioManager, channel, openChannels, compressionEnable, compressionCodecFactory,
				compressionBlockSize, pageSize);
		iterators.add(channelReaderInputViewIterator(view));
	}

	return new BinaryMergeIterator<>(
			iterators, mergeReusedEntries(channelIDs.size()), mergeComparator());
}
 
Example #11
Source File: LongHashPartition.java    From flink with Apache License 2.0 6 votes vote down vote up
int spillPartition(IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException {
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition with less than two buffers.");
	}

	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = FileChannelUtil.createBlockChannelWriter(
			ioAccess,
			targetChannel,
			bufferReturnQueue,
			longTable.compressionEnable(),
			longTable.compressionCodecFactory(),
			longTable.compressionBlockSize(),
			segmentSize);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 
Example #12
Source File: LongHashPartition.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * After build phase.
 *
 * @return build spill return buffer, if have spilled, it returns the current write buffer,
 * because it was used all the time in build phase, so it can only be returned at this time.
 */
int finalizeBuildPhase(
		IOManager ioAccess,
		FileIOChannel.Enumerator probeChannelEnumerator) throws IOException {
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();

	if (!isInMemory()) {
		// close the channel.
		this.buildSideChannel.close();

		this.probeSideBuffer = FileChannelUtil.createOutputView(
				ioAccess,
				probeChannelEnumerator.next(),
				longTable.compressionEnable(),
				longTable.compressionCodecFactory(),
				longTable.compressionBlockSize(),
				segmentSize);
		return 1;
	} else {
		return 0;
	}
}
 
Example #13
Source File: BinaryHashPartition.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * After build phase.
 * @return build spill return buffer, if have spilled, it returns the current write buffer,
 * because it was used all the time in build phase, so it can only be returned at this time.
 */
int finalizeBuildPhase(IOManager ioAccess, FileIOChannel.Enumerator probeChannelEnumerator) throws IOException {
	this.finalBufferLimit = this.buildSideWriteBuffer.getCurrentPositionInSegment();
	this.partitionBuffers = this.buildSideWriteBuffer.close();

	if (!isInMemory()) {
		// close the channel.
		this.buildSideChannel.close();

		this.probeSideBuffer = FileChannelUtil.createOutputView(
				ioAccess,
				probeChannelEnumerator.next(),
				compressionEnable,
				compressionCodecFactory,
				compressionBlockSize,
				memorySegmentSize);
		return 1;
	} else {
		return 0;
	}
}
 
Example #14
Source File: FileChannelUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
public static AbstractChannelWriterOutputView createOutputView(
		IOManager ioManager,
		FileIOChannel.ID channel,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		BufferFileWriter bufferWriter = ioManager.createBufferFileWriter(channel);
		return new CompressedHeaderlessChannelWriterOutputView(
				bufferWriter,
				compressionCodecFactory,
				compressionBlockSize);
	} else {
		BlockChannelWriter<MemorySegment> blockWriter =
				ioManager.createBlockChannelWriter(channel);
		return new HeaderlessChannelWriterOutputView(
				blockWriter,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				segmentSize
		);
	}
}
 
Example #15
Source File: FileChannelUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
public static BlockChannelReader<MemorySegment> createBlockChannelReader(
		IOManager ioManager,
		FileIOChannel.ID channel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		return new CompressedBlockChannelReader(
				ioManager,
				channel,
				bufferReturnQueue,
				compressionCodecFactory,
				compressionBlockSize,
				segmentSize
		);
	} else {
		return ioManager.createBlockChannelReader(channel, bufferReturnQueue);
	}
}
 
Example #16
Source File: SeekableFileChannelInputView.java    From flink with Apache License 2.0 5 votes vote down vote up
public SeekableFileChannelInputView(IOManager ioManager, FileIOChannel.ID channelId, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(ioManager);
	checkNotNull(channelId);
	checkNotNull(memManager);
	checkNotNull(memory);
	
	this.ioManager = ioManager;
	this.channelId = channelId;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	this.segmentSize = memManager.getPageSize();
	
	this.reader = ioManager.createBlockChannelReader(channelId);
	
	try {
		final long channelLength = reader.getSize();
		
		final int blockCount =  MathUtils.checkedDownCast(channelLength / segmentSize);
		this.numBlocksTotal = (channelLength % segmentSize == 0) ? blockCount : blockCount + 1;

		this.numBlocksRemaining = this.numBlocksTotal;
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}
 
Example #17
Source File: FileChannelStreamsTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testCloseAndDeleteOutputView() {
	final IOManager ioManager = new IOManagerAsync();
	try {
		MemoryManager memMan = new MemoryManager(4 * 16*1024, 1, 16*1024, MemoryType.HEAP, true);
		List<MemorySegment> memory = new ArrayList<MemorySegment>();
		memMan.allocatePages(new DummyInvokable(), memory, 4);
		
		FileIOChannel.ID channel = ioManager.createChannel();
		BlockChannelWriter<MemorySegment> writer = ioManager.createBlockChannelWriter(channel);
		
		FileChannelOutputView out = new FileChannelOutputView(writer, memMan, memory, memMan.getPageSize());
		new StringValue("Some test text").write(out);
		
		// close for the first time, make sure all memory returns
		out.close();
		assertTrue(memMan.verifyEmpty());
		
		// close again, should not cause an exception
		out.close();
		
		// delete, make sure file is removed
		out.closeAndDelete();
		assertFalse(new File(channel.getPath()).exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
	finally {
		ioManager.shutdown();
	}
}
 
Example #18
Source File: CompressedHeaderlessChannelTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testCompressedView() throws IOException {
	for (int testTime = 0; testTime < 10; testTime++) {
		int testRounds = new Random().nextInt(20000);
		FileIOChannel.ID channel = ioManager.createChannel();
		BufferFileWriter writer = this.ioManager.createBufferFileWriter(channel);
		CompressedHeaderlessChannelWriterOutputView outputView =
				new CompressedHeaderlessChannelWriterOutputView(
						writer,
						compressionFactory,
						BUFFER_SIZE
				);

		for (int i = 0; i < testRounds; i++) {
			outputView.writeInt(i);
		}
		outputView.close();
		int blockCount = outputView.getBlockCount();

		CompressedHeaderlessChannelReaderInputView inputView =
				new CompressedHeaderlessChannelReaderInputView(
						channel,
						ioManager,
						compressionFactory,
						BUFFER_SIZE,
						blockCount
				);

		for (int i = 0; i < testRounds; i++) {
			assertEquals(i, inputView.readInt());
		}
		inputView.close();
	}
}
 
Example #19
Source File: SpillableSubpartitionTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public BufferFileWriter createBufferFileWriter(FileIOChannel.ID channelID) throws IOException {
	blockLatch.countDown();
	try {
		doneLatch.await();
	} catch (InterruptedException e) {
		throw new IOException("Blocking operation was interrupted.", e);
	}

	return super.createBufferFileWriter(channelID);
}
 
Example #20
Source File: SpillableSubpartitionTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public BufferFileWriter createBufferFileWriter(FileIOChannel.ID channelID)
		throws IOException {
	BufferFileWriter bufferFileWriter = super.createBufferFileWriter(channelID);
	bufferFileWriter.close();
	return bufferFileWriter;
}
 
Example #21
Source File: ResettableExternalBuffer.java    From flink with Apache License 2.0 5 votes vote down vote up
private void spill() throws IOException {
	FileIOChannel.ID channel = ioManager.createChannel();

	final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
	int numRecordBuffers = inMemoryBuffer.getNumRecordBuffers();
	ArrayList<MemorySegment> segments = inMemoryBuffer.getRecordBufferSegments();
	try {
		// spill in memory buffer in zero-copy.
		for (int i = 0; i < numRecordBuffers; i++) {
			writer.writeBlock(segments.get(i));
		}
		LOG.info("here spill the reset buffer data with {} bytes", writer.getSize());
		writer.close();
	} catch (IOException e) {
		writer.closeAndDelete();
		throw e;
	}

	spillSize += numRecordBuffers * segmentSize;
	spilledChannelIDs.add(new ChannelWithMeta(
		channel,
		inMemoryBuffer.getNumRecordBuffers(),
		inMemoryBuffer.getNumBytesInLastBuffer()));
	this.spilledChannelRowOffsets.add(numRows);

	inMemoryBuffer.reset();
}
 
Example #22
Source File: BaseHybridHashTable.java    From flink with Apache License 2.0 5 votes vote down vote up
protected HeaderlessChannelReaderInputView createInputView(FileIOChannel.ID id, int blockCount, int lastSegmentLimit) throws IOException {
	BlockChannelReader<MemorySegment> inReader = FileChannelUtil.createBlockChannelReader(
			ioManager, id, new LinkedBlockingQueue<>(),
			compressionEnable, compressionCodecFactory, compressionBlockSize, segmentSize);
	return new HeaderlessChannelReaderInputView(inReader,
												Arrays.asList(allocateUnpooledSegment(segmentSize), allocateUnpooledSegment(segmentSize)),
												blockCount, lastSegmentLimit, false);

}
 
Example #23
Source File: SeekableFileChannelInputView.java    From flink with Apache License 2.0 5 votes vote down vote up
public SeekableFileChannelInputView(IOManager ioManager, FileIOChannel.ID channelId, MemoryManager memManager, List<MemorySegment> memory, int sizeOfLastBlock) throws IOException {
	super(0);
	
	checkNotNull(ioManager);
	checkNotNull(channelId);
	checkNotNull(memManager);
	checkNotNull(memory);
	
	this.ioManager = ioManager;
	this.channelId = channelId;
	this.memManager = memManager;
	this.memory = memory;
	this.sizeOfLastBlock = sizeOfLastBlock;
	this.segmentSize = memManager.getPageSize();
	
	this.reader = ioManager.createBlockChannelReader(channelId);
	
	try {
		final long channelLength = reader.getSize();
		
		final int blockCount =  MathUtils.checkedDownCast(channelLength / segmentSize);
		this.numBlocksTotal = (channelLength % segmentSize == 0) ? blockCount : blockCount + 1;

		this.numBlocksRemaining = this.numBlocksTotal;
		this.numRequestsRemaining = numBlocksRemaining;
		
		for (int i = 0; i < memory.size(); i++) {
			sendReadRequest(memory.get(i));
		}
		
		advance();
	}
	catch (IOException e) {
		memManager.release(memory);
		throw e;
	}
}
 
Example #24
Source File: HashPartition.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Spills this partition to disk and sets it up such that it continues spilling records that are added to
 * it. The spilling process must free at least one buffer, either in the partition's record buffers, or in
 * the memory segments for overflow buckets.
 * The partition immediately takes back one buffer to use it for further spilling.
 * 
 * @param target The list to which memory segments from overflow buckets are added.
 * @param ioAccess The I/O manager to be used to create a writer to disk.
 * @param targetChannel The id of the target channel for this partition.
 * @return The number of buffers that were freed by spilling this partition.
 * @throws IOException Thrown, if the writing failed.
 */
public int spillPartition(List<MemorySegment> target, IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue)
throws IOException
{
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
			"Request to spill a partition with less than two buffers.");
	}
	
	// return the memory from the overflow segments
	for (int i = 0; i < this.numOverflowSegments; i++) {
		target.add(this.overflowSegments[i]);
	}
	this.overflowSegments = null;
	this.numOverflowSegments = 0;
	this.nextOverflowBucket = 0;
	
	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = ioAccess.createBlockChannelWriter(targetChannel, bufferReturnQueue);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 
Example #25
Source File: SpillChannelManager.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Open File channels.
 */
public synchronized void addOpenChannels(List<FileIOChannel> toOpen) {
	checkArgument(!closed);
	for (FileIOChannel channel : toOpen) {
		openChannels.add(channel);
		channels.remove(channel.getChannelID());
	}
}
 
Example #26
Source File: AbstractBinaryExternalMerger.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Merges the sorted runs described by the given Channel IDs into a single sorted run.
 *
 * @param channelIDs   The IDs of the runs' channels.
 * @return The ID and number of blocks of the channel that describes the merged run.
 */
private ChannelWithMeta mergeChannels(List<ChannelWithMeta> channelIDs) throws IOException {
	// the list with the target iterators
	List<FileIOChannel> openChannels = new ArrayList<>(channelIDs.size());
	final BinaryMergeIterator<Entry> mergeIterator =
			getMergingIterator(channelIDs, openChannels);

	// create a new channel writer
	final FileIOChannel.ID mergedChannelID = ioManager.createChannel();
	channelManager.addChannel(mergedChannelID);
	AbstractChannelWriterOutputView output = null;

	int numBytesInLastBlock;
	int numBlocksWritten;
	try {
		output = FileChannelUtil.createOutputView(
				ioManager, mergedChannelID, compressionEnable,
				compressionCodecFactory, compressionBlockSize, pageSize);
		writeMergingOutput(mergeIterator, output);
		numBytesInLastBlock = output.close();
		numBlocksWritten = output.getBlockCount();
	} catch (IOException e) {
		if (output != null) {
			output.close();
			output.getChannel().deleteChannel();
		}
		throw e;
	}

	// remove, close and delete channels
	for (FileIOChannel channel : openChannels) {
		channelManager.removeChannel(channel.getChannelID());
		try {
			channel.closeAndDelete();
		} catch (Throwable ignored) {
		}
	}

	return new ChannelWithMeta(mergedChannelID, numBlocksWritten, numBytesInLastBlock);
}
 
Example #27
Source File: BinaryHashPartition.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Spills this partition to disk and sets it up such that it continues spilling records that are
 * added to
 * it. The spilling process must free at least one buffer, either in the partition's record
 * buffers, or in
 * the memory segments for overflow buckets.
 * The partition immediately takes back one buffer to use it for further spilling.
 *
 * @param ioAccess      The I/O manager to be used to create a writer to disk.
 * @param targetChannel The id of the target channel for this partition.
 * @return The number of buffers that were freed by spilling this partition.
 * @throws IOException Thrown, if the writing failed.
 */
int spillPartition(
		IOManager ioAccess, FileIOChannel.ID targetChannel,
		LinkedBlockingQueue<MemorySegment> bufferReturnQueue) throws IOException {
	// sanity checks
	if (!isInMemory()) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition that has already been spilled.");
	}
	if (getNumOccupiedMemorySegments() < 2) {
		throw new RuntimeException("Bug in Hybrid Hash Join: " +
				"Request to spill a partition with less than two buffers.");
	}

	// create the channel block writer and spill the current buffers
	// that keep the build side buffers current block, as it is most likely not full, yet
	// we return the number of blocks that become available
	this.buildSideChannel = FileChannelUtil.createBlockChannelWriter(
			ioAccess,
			targetChannel,
			bufferReturnQueue,
			compressionEnable,
			compressionCodecFactory,
			compressionBlockSize,
			memorySegmentSize);
	return this.buildSideWriteBuffer.spill(this.buildSideChannel);
}
 
Example #28
Source File: UnilateralSortMerger.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Returns an iterator that iterates over the merged result from all given channels.
 * 
 * @param channelIDs The channels that are to be merged and returned.
 * @param inputSegments The buffers to be used for reading. The list contains for each channel one
 *                      list of input segments. The size of the <code>inputSegments</code> list must be equal to
 *                      that of the <code>channelIDs</code> list.
 * @return An iterator over the merged records of the input channels.
 * @throws IOException Thrown, if the readers encounter an I/O problem.
 */
protected final MergeIterator<E> getMergingIterator(final List<ChannelWithBlockCount> channelIDs,
		final List<List<MemorySegment>> inputSegments, List<FileIOChannel> readerList, MutableObjectIterator<E> largeRecords)
	throws IOException
{
	// create one iterator per channel id
	if (LOG.isDebugEnabled()) {
		LOG.debug("Performing merge of " + channelIDs.size() + " sorted streams.");
	}
	
	final List<MutableObjectIterator<E>> iterators = new ArrayList<MutableObjectIterator<E>>(channelIDs.size() + 1);
	
	for (int i = 0; i < channelIDs.size(); i++) {
		final ChannelWithBlockCount channel = channelIDs.get(i);
		final List<MemorySegment> segsForChannel = inputSegments.get(i);
		
		// create a reader. if there are multiple segments for the reader, issue multiple together per I/O request
		final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channel.getChannel());
			
		readerList.add(reader);
		registerOpenChannelToBeRemovedAtShudown(reader);
		unregisterChannelToBeRemovedAtShudown(channel.getChannel());
		
		// wrap channel reader as a view, to get block spanning record deserialization
		final ChannelReaderInputView inView = new ChannelReaderInputView(reader, segsForChannel, 
																	channel.getBlockCount(), false);
		iterators.add(new ChannelReaderInputViewIterator<E>(inView, null, this.serializer));
	}
	
	if (largeRecords != null) {
		iterators.add(largeRecords);
	}

	return new MergeIterator<E>(iterators, this.comparator);
}
 
Example #29
Source File: BufferedKVExternalSorter.java    From flink with Apache License 2.0 5 votes vote down vote up
public void sortAndSpill(
		ArrayList<MemorySegment> recordBufferSegments,
		long numElements,
		MemorySegmentPool pool) throws IOException {

	// 1. sort buffer
	BinaryKVInMemorySortBuffer buffer =
			BinaryKVInMemorySortBuffer.createBuffer(
					nKeyComputer, keySerializer, valueSerializer, comparator,
					recordBufferSegments, numElements, pool);
	this.sorter.sort(buffer);

	// 2. spill
	FileIOChannel.ID channel = enumerator.next();
	channelManager.addChannel(channel);

	AbstractChannelWriterOutputView output = null;
	int bytesInLastBuffer;
	int blockCount;
	try {
		numSpillFiles++;
		output = FileChannelUtil.createOutputView(ioManager, channel, compressionEnable,
				compressionCodecFactory, compressionBlockSize, pageSize);
		buffer.writeToOutput(output);
		spillInBytes += output.getNumBytes();
		spillInCompressedBytes += output.getNumCompressedBytes();
		bytesInLastBuffer = output.close();
		blockCount = output.getBlockCount();
		LOG.info("here spill the {}th kv external buffer data with {} bytes and {} compressed bytes",
				numSpillFiles, spillInBytes, spillInCompressedBytes);
	} catch (IOException e) {
		if (output != null) {
			output.close();
			output.getChannel().deleteChannel();
		}
		throw e;
	}
	channelIDs.add(new ChannelWithMeta(channel, blockCount, bytesInLastBuffer));
}
 
Example #30
Source File: FileChannelUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
public static AbstractChannelReaderInputView createInputView(
		IOManager ioManager,
		ChannelWithMeta channel,
		List<FileIOChannel> channels,
		boolean compressionEnable,
		BlockCompressionFactory compressionCodecFactory,
		int compressionBlockSize,
		int segmentSize) throws IOException {
	if (compressionEnable) {
		CompressedHeaderlessChannelReaderInputView in =
				new CompressedHeaderlessChannelReaderInputView(
						channel.getChannel(),
						ioManager,
						compressionCodecFactory,
						compressionBlockSize,
						channel.getBlockCount()
				);
		channels.add(in.getReader());
		return in;
	} else {
		BlockChannelReader<MemorySegment> reader =
				ioManager.createBlockChannelReader(channel.getChannel());
		channels.add(reader);
		return new HeaderlessChannelReaderInputView(
				reader,
				Arrays.asList(
						allocateUnpooledSegment(segmentSize),
						allocateUnpooledSegment(segmentSize)
				),
				channel.getBlockCount(),
				channel.getNumBytesInLastBlock(), false
		);
	}
}