Java Code Examples for org.apache.flink.runtime.io.network.buffer.Buffer#isBuffer()

@Override
protected void deserializeBuffer(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		deserializer.setNextBuffer(buffer);

		while (deserializer.hasUnfinishedData()) {
			RecordDeserializer.DeserializationResult result =
				deserializer.getNextRecord(delegate);

			if (result.isFullRecord()) {
				output.add(delegate.getInstance());
			}

			if (result == RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER
				|| result == RecordDeserializer.DeserializationResult.PARTIAL_RECORD) {
				break;
			}
		}
	} else {
		// is event
		AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		output.add(event);
	}
}
 

private void writeAndCloseBufferConsumer(BufferConsumer bufferConsumer) throws IOException {
	try {
		final Buffer buffer = bufferConsumer.build();
		try {
			data.writeBuffer(buffer);

			numBuffersAndEventsWritten++;
			if (buffer.isBuffer()) {
				numDataBuffersWritten++;
			}
		}
		finally {
			buffer.recycleBuffer();
		}
	}
	finally {
		bufferConsumer.close();
	}
}
 

@Override
protected void deserializeBuffer(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		deserializer.setNextBuffer(buffer);

		while (deserializer.hasUnfinishedData()) {
			RecordDeserializer.DeserializationResult result =
				deserializer.getNextRecord(delegate);

			if (result.isFullRecord()) {
				output.add(delegate.getInstance());
			}

			if (result == RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER
				|| result == RecordDeserializer.DeserializationResult.PARTIAL_RECORD) {
				break;
			}
		}
	} else {
		// is event
		AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		output.add(event);
	}
}
 

private static BufferOrEvent parseBuffer(BufferConsumer bufferConsumer, int targetChannel) throws IOException {
	Buffer buffer = buildSingleBuffer(bufferConsumer);
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, targetChannel);
	} else {
		// is event:
		AbstractEvent event = EventSerializer.fromBuffer(buffer, RecordWriterTest.class.getClassLoader());
		buffer.recycleBuffer(); // the buffer is not needed anymore
		return new BufferOrEvent(event, targetChannel);
	}
}
 

private void writeAndCloseBufferConsumer(BufferConsumer bufferConsumer) throws IOException {
	try {
		final Buffer buffer = bufferConsumer.build();
		try {
			if (canBeCompressed(buffer)) {
				final Buffer compressedBuffer = parent.bufferCompressor.compressToIntermediateBuffer(buffer);
				data.writeBuffer(compressedBuffer);
				if (compressedBuffer != buffer) {
					compressedBuffer.recycleBuffer();
				}
			} else {
				data.writeBuffer(buffer);
			}

			numBuffersAndEventsWritten++;
			if (buffer.isBuffer()) {
				numDataBuffersWritten++;
			}
		}
		finally {
			buffer.recycleBuffer();
		}
	}
	finally {
		bufferConsumer.close();
	}
}
 

static BufferOrEvent parseBuffer(BufferConsumer bufferConsumer, int targetChannel) throws IOException {
	Buffer buffer = buildSingleBuffer(bufferConsumer);
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, new InputChannelInfo(0, targetChannel));
	} else {
		// is event:
		AbstractEvent event = EventSerializer.fromBuffer(buffer, RecordWriterTest.class.getClassLoader());
		buffer.recycleBuffer(); // the buffer is not needed anymore
		return new BufferOrEvent(event, new InputChannelInfo(0, targetChannel));
	}
}
 

/**
 * Parses the buffer as an event and returns the {@link CheckpointBarrier} if the event is indeed a barrier or
 * returns null in all other cases.
 */
@Nullable
protected CheckpointBarrier parseCheckpointBarrierOrNull(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		return null;
	}

	AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	// reset the buffer because it would be deserialized again in SingleInputGate while getting next buffer.
	// we can further improve to avoid double deserialization in the future.
	buffer.setReaderIndex(0);
	return event.getClass() == CheckpointBarrier.class ? (CheckpointBarrier) event : null;
}
 

private boolean isEndOfChannelStateEvent(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		return false;
	}

	AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	buffer.setReaderIndex(0);
	return event.getClass() == EndOfChannelStateEvent.class;
}
 

private void assertReturnedBuffersAreRecycled(boolean assertBuffers, boolean assertEvents) {
	for (Buffer b : allReturnedBuffers) {
		if (b.isBuffer() && assertBuffers && !b.isRecycled()) {
			fail("Data Buffer " + b + " not recycled");
		}
		if (!b.isBuffer() && assertEvents && !b.isRecycled()) {
			fail("Event Buffer " + b + " not recycled");
		}
	}
}
 

private void assertReturnedBuffersAreRecycled(boolean assertBuffers, boolean assertEvents) {
	for (Buffer b : allReturnedBuffers) {
		if (b.isBuffer() && assertBuffers && !b.isRecycled()) {
			fail("Data Buffer " + b + " not recycled");
		}
		if (!b.isBuffer() && assertEvents && !b.isRecycled()) {
			fail("Event Buffer " + b + " not recycled");
		}
	}
}
 

public static BufferAndBacklog fromBufferAndLookahead(Buffer current, Buffer lookahead, int backlog) {
	return new BufferAndBacklog(
			current,
			lookahead != null,
			backlog,
			lookahead != null && !lookahead.isBuffer());
}
 

private BufferOrEvent transformToBufferOrEvent(
		Buffer buffer,
		boolean moreAvailable,
		InputChannel currentChannel) throws IOException, InterruptedException {
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, currentChannel.getChannelIndex(), moreAvailable);
	}
	else {
		final AbstractEvent event;
		try {
			event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		}
		finally {
			buffer.recycleBuffer();
		}

		if (event.getClass() == EndOfPartitionEvent.class) {
			channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());

			if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
				// Because of race condition between:
				// 1. releasing inputChannelsWithData lock in this method and reaching this place
				// 2. empty data notification that re-enqueues a channel
				// we can end up with moreAvailable flag set to true, while we expect no more data.
				checkState(!moreAvailable || !pollNext().isPresent());
				moreAvailable = false;
				hasReceivedAllEndOfPartitionEvents = true;
				markAvailable();
			}

			currentChannel.notifySubpartitionConsumed();
			currentChannel.releaseAllResources();
		}

		return new BufferOrEvent(event, currentChannel.getChannelIndex(), moreAvailable, buffer.getSize());
	}
}
 

BufferResponse(
		Buffer buffer,
		int sequenceNumber,
		InputChannelID receiverId,
		int backlog) {
	this.buffer = checkNotNull(buffer).asByteBuf();
	this.isBuffer = buffer.isBuffer();
	this.sequenceNumber = sequenceNumber;
	this.receiverId = checkNotNull(receiverId);
	this.backlog = backlog;
}
 

private static BufferOrEvent parseBuffer(BufferConsumer bufferConsumer, int targetChannel) throws IOException {
	Buffer buffer = buildSingleBuffer(bufferConsumer);
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, targetChannel);
	} else {
		// is event:
		AbstractEvent event = EventSerializer.fromBuffer(buffer, RecordWriterTest.class.getClassLoader());
		buffer.recycleBuffer(); // the buffer is not needed anymore
		return new BufferOrEvent(event, targetChannel);
	}
}
 

public static BufferAndBacklog fromBufferAndLookahead(Buffer current, Buffer lookahead, int backlog) {
	return new BufferAndBacklog(
			current,
			lookahead != null,
			backlog,
			lookahead != null && !lookahead.isBuffer());
}
 

private Optional<BufferOrEvent> getNextBufferOrEvent(boolean blocking) throws IOException, InterruptedException {
	if (hasReceivedAllEndOfPartitionEvents) {
		return Optional.empty();
	}

	if (isReleased) {
		throw new IllegalStateException("Released");
	}

	requestPartitions();

	InputChannel currentChannel;
	boolean moreAvailable;
	Optional<BufferAndAvailability> result = Optional.empty();

	do {
		synchronized (inputChannelsWithData) {
			while (inputChannelsWithData.size() == 0) {
				if (isReleased) {
					throw new IllegalStateException("Released");
				}

				if (blocking) {
					inputChannelsWithData.wait();
				}
				else {
					return Optional.empty();
				}
			}

			currentChannel = inputChannelsWithData.remove();
			enqueuedInputChannelsWithData.clear(currentChannel.getChannelIndex());
			moreAvailable = !inputChannelsWithData.isEmpty();
		}

		result = currentChannel.getNextBuffer();
	} while (!result.isPresent());

	// this channel was now removed from the non-empty channels queue
	// we re-add it in case it has more data, because in that case no "non-empty" notification
	// will come for that channel
	if (result.get().moreAvailable()) {
		queueChannel(currentChannel);
		moreAvailable = true;
	}

	final Buffer buffer = result.get().buffer();
	if (buffer.isBuffer()) {
		return Optional.of(new BufferOrEvent(buffer, currentChannel.getChannelIndex(), moreAvailable));
	}
	else {
		final AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());

		if (event.getClass() == EndOfPartitionEvent.class) {
			channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());

			if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
				// Because of race condition between:
				// 1. releasing inputChannelsWithData lock in this method and reaching this place
				// 2. empty data notification that re-enqueues a channel
				// we can end up with moreAvailable flag set to true, while we expect no more data.
				checkState(!moreAvailable || !pollNextBufferOrEvent().isPresent());
				moreAvailable = false;
				hasReceivedAllEndOfPartitionEvents = true;
			}

			currentChannel.notifySubpartitionConsumed();

			currentChannel.releaseAllResources();
		}

		return Optional.of(new BufferOrEvent(event, currentChannel.getChannelIndex(), moreAvailable));
	}
}
 

public void onBuffer(Buffer buffer, int sequenceNumber, int backlog) throws IOException {
	boolean recycleBuffer = true;

	try {
		if (expectedSequenceNumber != sequenceNumber) {
			onError(new BufferReorderingException(expectedSequenceNumber, sequenceNumber));
			return;
		}

		final boolean wasEmpty;
		final CheckpointBarrier notifyReceivedBarrier;
		final Buffer notifyReceivedBuffer;
		final BufferReceivedListener listener = inputGate.getBufferReceivedListener();
		synchronized (receivedBuffers) {
			// Similar to notifyBufferAvailable(), make sure that we never add a buffer
			// after releaseAllResources() released all buffers from receivedBuffers
			// (see above for details).
			if (isReleased.get()) {
				return;
			}

			wasEmpty = receivedBuffers.isEmpty();
			receivedBuffers.add(buffer);

			if (listener != null && buffer.isBuffer() && receivedCheckpointId < lastRequestedCheckpointId) {
				notifyReceivedBuffer = buffer.retainBuffer();
			} else {
				notifyReceivedBuffer = null;
			}
			notifyReceivedBarrier = listener != null ? parseCheckpointBarrierOrNull(buffer) : null;
		}
		recycleBuffer = false;

		++expectedSequenceNumber;

		if (wasEmpty) {
			notifyChannelNonEmpty();
		}

		if (backlog >= 0) {
			onSenderBacklog(backlog);
		}

		if (notifyReceivedBarrier != null) {
			receivedCheckpointId = notifyReceivedBarrier.getId();
			if (notifyReceivedBarrier.isCheckpoint()) {
				listener.notifyBarrierReceived(notifyReceivedBarrier, channelInfo);
			}
		} else if (notifyReceivedBuffer != null) {
			listener.notifyBufferReceived(notifyReceivedBuffer, channelInfo);
		}
	} finally {
		if (recycleBuffer) {
			buffer.recycleBuffer();
		}
	}
}
 

/**
 * Identifies whether the given buffer encodes the given event. Custom events are not supported.
 *
 * @param buffer the buffer to peak into
 * @param eventClass the expected class of the event type
 * @return whether the event class of the <tt>buffer</tt> matches the given <tt>eventClass</tt>
 */
public static boolean isEvent(Buffer buffer, Class<?> eventClass) throws IOException {
	return !buffer.isBuffer() && isEvent(buffer.getNioBufferReadable(), eventClass);
}
 

/**
 * Whether the buffer can be compressed or not. Note that event is not compressed because it
 * is usually small and the size can become even larger after compression.
 */
protected boolean canBeCompressed(Buffer buffer) {
	return parent.bufferCompressor != null && buffer.isBuffer() && buffer.readableBytes() > 0;
}
 

/**
 * Identifies whether the given buffer encodes the given event. Custom events are not supported.
 *
 * @param buffer the buffer to peak into
 * @param eventClass the expected class of the event type
 * @return whether the event class of the <tt>buffer</tt> matches the given <tt>eventClass</tt>
 */
public static boolean isEvent(Buffer buffer, Class<?> eventClass) throws IOException {
	return !buffer.isBuffer() && isEvent(buffer.getNioBufferReadable(), eventClass);
}