Java Code Examples for org.apache.flink.util.FileUtils#writeCompletely()

private static BufferOrEvent generateAndWriteEvent(FileChannel fileChannel, Random rnd, int numberOfChannels) throws IOException {
	long magicNumber = rnd.nextLong();
	byte[] data = new byte[rnd.nextInt(1000)];
	rnd.nextBytes(data);
	TestEvent evt = new TestEvent(magicNumber, data);

	int channelIndex = rnd.nextInt(numberOfChannels);

	ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
	ByteBuffer header = ByteBuffer.allocate(9);
	header.order(ByteOrder.LITTLE_ENDIAN);

	header.putInt(channelIndex);
	header.putInt(serializedEvent.remaining());
	header.put((byte) 1);
	header.flip();

	FileUtils.writeCompletely(fileChannel, header);
	FileUtils.writeCompletely(fileChannel, serializedEvent);
	return new BufferOrEvent(evt, channelIndex);
}
 

@Test
public void testCleanup() {
	try {
		ByteBuffer data = ByteBuffer.allocate(157);
		data.order(ByteOrder.LITTLE_ENDIAN);

		FileUtils.writeCompletely(fileChannel, data);
		fileChannel.position(54);

		SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
		seq.open();
		seq.cleanup();

		assertFalse(fileChannel.isOpen());
		assertFalse(tempFile.exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testCleanup() {
	try {
		ByteBuffer data = ByteBuffer.allocate(157);
		data.order(ByteOrder.LITTLE_ENDIAN);

		FileUtils.writeCompletely(fileChannel, data);
		fileChannel.position(54);

		SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
		seq.open();
		seq.cleanup();

		assertFalse(fileChannel.isOpen());
		assertFalse(tempFile.exists());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

private static BufferOrEvent generateAndWriteEvent(FileChannel fileChannel, Random rnd, int numberOfChannels) throws IOException {
	long magicNumber = rnd.nextLong();
	byte[] data = new byte[rnd.nextInt(1000)];
	rnd.nextBytes(data);
	TestEvent evt = new TestEvent(magicNumber, data);

	int channelIndex = rnd.nextInt(numberOfChannels);

	ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
	ByteBuffer header = ByteBuffer.allocate(9);
	header.order(ByteOrder.LITTLE_ENDIAN);

	header.putInt(channelIndex);
	header.putInt(serializedEvent.remaining());
	header.put((byte) 1);
	header.flip();

	FileUtils.writeCompletely(fileChannel, header);
	FileUtils.writeCompletely(fileChannel, serializedEvent);
	return new BufferOrEvent(evt, channelIndex);
}
 

private void initializeWithPartialRecord(NonSpanningWrapper partial, int nextRecordLength) throws IOException {
	// set the length and copy what is available to the buffer
	this.recordLength = nextRecordLength;

	final int numBytesChunk = partial.remaining();

	if (nextRecordLength > THRESHOLD_FOR_SPILLING) {
		// create a spilling channel and put the data there
		this.spillingChannel = createSpillingChannel();

		ByteBuffer toWrite = partial.segment.wrap(partial.position, numBytesChunk);
		FileUtils.writeCompletely(this.spillingChannel, toWrite);
	}
	else {
		// collect in memory
		ensureBufferCapacity(nextRecordLength);
		partial.segment.get(partial.position, buffer, 0, numBytesChunk);
	}

	this.accumulatedRecordBytes = numBytesChunk;
}
 

private void initializeWithPartialRecord(NonSpanningWrapper partial, int nextRecordLength) throws IOException {
	// set the length and copy what is available to the buffer
	this.recordLength = nextRecordLength;

	final int numBytesChunk = partial.remaining();

	if (nextRecordLength > THRESHOLD_FOR_SPILLING) {
		// create a spilling channel and put the data there
		this.spillingChannel = createSpillingChannel();

		ByteBuffer toWrite = partial.segment.wrap(partial.position, numBytesChunk);
		FileUtils.writeCompletely(this.spillingChannel, toWrite);
	}
	else {
		// collect in memory
		ensureBufferCapacity(nextRecordLength);
		partial.segment.get(partial.position, buffer, 0, numBytesChunk);
	}

	this.accumulatedRecordBytes = numBytesChunk;
}
 

@Override
public void write() throws IOException {
	try {
		FileUtils.writeCompletely(this.channel.fileChannel, this.segment.wrap(0, this.segment.size()));
	}
	catch (NullPointerException npex) {
		throw new IOException("Memory segment has been released.");
	}
}
 

private static void writeBuffer(FileChannel fileChannel, int size, int channelIndex) throws IOException {
	ByteBuffer data = ByteBuffer.allocate(size + 9);
	data.order(ByteOrder.LITTLE_ENDIAN);

	data.putInt(channelIndex);
	data.putInt(size);
	data.put((byte) 0);
	for (int i = 0; i < size; i++) {
		data.put((byte) i);
	}
	data.flip();
	FileUtils.writeCompletely(fileChannel, data);
}
 

@Override
public void add(BufferOrEvent boe) throws IOException {
	try {
		ByteBuffer contents;
		if (boe.isBuffer()) {
			Buffer buf = boe.getBuffer();
			contents = buf.getNioBufferReadable();
		}
		else {
			contents = EventSerializer.toSerializedEvent(boe.getEvent());
		}

		headBuffer.clear();
		headBuffer.putInt(boe.getChannelIndex());
		headBuffer.putInt(contents.remaining());
		headBuffer.put((byte) (boe.isBuffer() ? 0 : 1));
		headBuffer.flip();

		bytesWritten += (headBuffer.remaining() + contents.remaining());

		FileUtils.writeCompletely(currentChannel, headBuffer);
		FileUtils.writeCompletely(currentChannel, contents);
	}
	finally {
		if (boe.isBuffer()) {
			boe.getBuffer().recycleBuffer();
		}
	}
}
 

@Override
public void write() throws IOException {
	ByteBuffer nioBufferReadable = buffer.getNioBufferReadable();

	final ByteBuffer header = ByteBuffer.allocateDirect(8);

	header.putInt(buffer.isBuffer() ? 1 : 0);
	header.putInt(nioBufferReadable.remaining());
	header.flip();

	FileUtils.writeCompletely(channel.fileChannel, header);
	FileUtils.writeCompletely(channel.fileChannel, nioBufferReadable);
}
 

@Override
public void write() throws IOException {
	try {
		FileUtils.writeCompletely(this.channel.fileChannel, this.segment.wrap(0, this.segment.size()));
	}
	catch (NullPointerException npex) {
		throw new IOException("Memory segment has been released.");
	}
}
 

private static void writeBuffer(FileChannel fileChannel, int size, int channelIndex) throws IOException {
	ByteBuffer data = ByteBuffer.allocate(size + 9);
	data.order(ByteOrder.LITTLE_ENDIAN);

	data.putInt(channelIndex);
	data.putInt(size);
	data.put((byte) 0);
	for (int i = 0; i < size; i++) {
		data.put((byte) i);
	}
	data.flip();
	FileUtils.writeCompletely(fileChannel, data);
}
 

/**
 * Adds a buffer or event to the sequence of spilled buffers and events.
 *
 * @param boe The buffer or event to add and spill.
 * @throws IOException Thrown, if the buffer of event could not be spilled.
 */
@Override
public void add(BufferOrEvent boe) throws IOException {
	try {
		ByteBuffer contents;
		if (boe.isBuffer()) {
			Buffer buf = boe.getBuffer();
			contents = buf.getNioBufferReadable();
		}
		else {
			contents = EventSerializer.toSerializedEvent(boe.getEvent());
		}

		headBuffer.clear();
		headBuffer.putInt(boe.getChannelIndex());
		headBuffer.putInt(contents.remaining());
		headBuffer.put((byte) (boe.isBuffer() ? 0 : 1));
		headBuffer.flip();

		bytesWritten += (headBuffer.remaining() + contents.remaining());

		FileUtils.writeCompletely(currentChannel, headBuffer);
		FileUtils.writeCompletely(currentChannel, contents);
	}
	finally {
		if (boe.isBuffer()) {
			boe.getBuffer().recycleBuffer();
		}
	}
}
 

@Override
public void write() throws IOException {
	ByteBuffer nioBufferReadable = buffer.getNioBufferReadable();

	final ByteBuffer header = ByteBuffer.allocateDirect(8);

	header.putInt(buffer.isBuffer() ? 1 : 0);
	header.putInt(nioBufferReadable.remaining());
	header.flip();

	FileUtils.writeCompletely(channel.fileChannel, header);
	FileUtils.writeCompletely(channel.fileChannel, nioBufferReadable);
}
 

@Override
public void write() throws IOException {
	try {
		FileUtils.writeCompletely(this.channel.fileChannel, this.segment.wrap(0, this.segment.size()));
	}
	catch (NullPointerException npex) {
		throw new IOException("Memory segment has been released.");
	}
}
 

@Override
public void write() throws IOException {
	ByteBuffer nioBufferReadable = buffer.getNioBufferReadable();

	final ByteBuffer header = ByteBuffer.allocateDirect(8);

	header.putInt(buffer.isBuffer() ? 1 : 0);
	header.putInt(nioBufferReadable.remaining());
	header.flip();

	FileUtils.writeCompletely(channel.fileChannel, header);
	FileUtils.writeCompletely(channel.fileChannel, nioBufferReadable);
}
 

private void addNextChunkFromMemorySegment(MemorySegment segment, int offset, int numBytes) throws IOException {
	int segmentPosition = offset;
	int segmentRemaining = numBytes;
	// check where to go. if we have a partial length, we need to complete it first
	if (this.lengthBuffer.position() > 0) {
		int toPut = Math.min(this.lengthBuffer.remaining(), segmentRemaining);
		segment.get(segmentPosition, this.lengthBuffer, toPut);
		// did we complete the length?
		if (this.lengthBuffer.hasRemaining()) {
			return;
		} else {
			this.recordLength = this.lengthBuffer.getInt(0);

			this.lengthBuffer.clear();
			segmentPosition += toPut;
			segmentRemaining -= toPut;
			if (this.recordLength > THRESHOLD_FOR_SPILLING) {
				this.spillingChannel = createSpillingChannel();
			} else {
				ensureBufferCapacity(this.recordLength);
			}
		}
	}

	// copy as much as we need or can for this next spanning record
	int needed = this.recordLength - this.accumulatedRecordBytes;
	int toCopy = Math.min(needed, segmentRemaining);

	if (spillingChannel != null) {
		// spill to file
		ByteBuffer toWrite = segment.wrap(segmentPosition, toCopy);
		FileUtils.writeCompletely(this.spillingChannel, toWrite);
	} else {
		segment.get(segmentPosition, buffer, this.accumulatedRecordBytes, toCopy);
	}

	this.accumulatedRecordBytes += toCopy;

	if (toCopy < segmentRemaining) {
		// there is more data in the segment
		this.leftOverData = segment;
		this.leftOverStart = segmentPosition + toCopy;
		this.leftOverLimit = numBytes + offset;
	}

	if (accumulatedRecordBytes == recordLength) {
		// we have the full record
		if (spillingChannel == null) {
			this.serializationReadBuffer.setBuffer(buffer, 0, recordLength);
		}
		else {
			spillingChannel.close();

			BufferedInputStream inStream = new BufferedInputStream(new FileInputStream(spillFile), 2 * 1024 * 1024);
			this.spillFileReader = new DataInputViewStreamWrapper(inStream);
		}
	}
}
 

private void addNextChunkFromMemorySegment(MemorySegment segment, int offset, int numBytes) throws IOException {
	int segmentPosition = offset;
	int segmentRemaining = numBytes;
	// check where to go. if we have a partial length, we need to complete it first
	if (this.lengthBuffer.position() > 0) {
		int toPut = Math.min(this.lengthBuffer.remaining(), segmentRemaining);
		segment.get(segmentPosition, this.lengthBuffer, toPut);
		// did we complete the length?
		if (this.lengthBuffer.hasRemaining()) {
			return;
		} else {
			this.recordLength = this.lengthBuffer.getInt(0);

			this.lengthBuffer.clear();
			segmentPosition += toPut;
			segmentRemaining -= toPut;
			if (this.recordLength > THRESHOLD_FOR_SPILLING) {
				this.spillingChannel = createSpillingChannel();
			} else {
				ensureBufferCapacity(this.recordLength);
			}
		}
	}

	// copy as much as we need or can for this next spanning record
	int needed = this.recordLength - this.accumulatedRecordBytes;
	int toCopy = Math.min(needed, segmentRemaining);

	if (spillingChannel != null) {
		// spill to file
		ByteBuffer toWrite = segment.wrap(segmentPosition, toCopy);
		FileUtils.writeCompletely(this.spillingChannel, toWrite);
	} else {
		segment.get(segmentPosition, buffer, this.accumulatedRecordBytes, toCopy);
	}

	this.accumulatedRecordBytes += toCopy;

	if (toCopy < segmentRemaining) {
		// there is more data in the segment
		this.leftOverData = segment;
		this.leftOverStart = segmentPosition + toCopy;
		this.leftOverLimit = numBytes + offset;
	}

	if (accumulatedRecordBytes == recordLength) {
		// we have the full record
		if (spillingChannel == null) {
			this.serializationReadBuffer.setBuffer(buffer, 0, recordLength);
		}
		else {
			spillingChannel.close();

			BufferedInputStream inStream = new BufferedInputStream(new FileInputStream(spillFile), 2 * 1024 * 1024);
			this.spillFileReader = new DataInputViewStreamWrapper(inStream);
		}
	}
}