Java Code Examples for org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent#getEvent()

The following examples show how to use org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent#getEvent() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MockInputGate.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Override
public Optional<BufferOrEvent> getNextBufferOrEvent() {
	BufferOrEvent next = bufferOrEvents.poll();
	if (next == null) {
		return Optional.empty();
	}

	int channelIdx = next.getChannelIndex();
	if (closed[channelIdx]) {
		throw new RuntimeException("Inconsistent: Channel " + channelIdx
			+ " has data even though it is already closed.");
	}
	if (next.isEvent() && next.getEvent() instanceof EndOfPartitionEvent) {
		closed[channelIdx] = true;
		closedChannels++;
	}
	return Optional.of(next);
}
Example 2
Source File: StreamTwoInputProcessor.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void processBufferOrEvent(BufferOrEvent bufferOrEvent) throws Exception {
	if (bufferOrEvent.isBuffer()) {
		currentChannel = bufferOrEvent.getChannelIndex();
		currentRecordDeserializer = recordDeserializers[currentChannel];
		currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
	}
	else {
		// Event received
		final AbstractEvent event = bufferOrEvent.getEvent();
		// TODO: with barrierHandler.isFinished() we might not need to support any events on this level.
		if (event.getClass() != EndOfPartitionEvent.class) {
			throw new IOException("Unexpected event: " + event);
		}

		handleEndOfPartitionEvent(bufferOrEvent.getChannelIndex());
	}
}
Example 3
Source File: MockInputGate.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public Optional<BufferOrEvent> getNext() {
	BufferOrEvent next = bufferOrEvents.poll();
	if (!finishAfterLastBuffer && bufferOrEvents.isEmpty()) {
		resetIsAvailable();
	}
	if (next == null) {
		return Optional.empty();
	}

	int channelIdx = next.getChannelIndex();
	if (closed[channelIdx]) {
		throw new RuntimeException("Inconsistent: Channel " + channelIdx
			+ " has data even though it is already closed.");
	}
	if (next.isEvent() && next.getEvent() instanceof EndOfPartitionEvent) {
		closed[channelIdx] = true;
	}
	return Optional.of(next);
}
Example 4
Source File: StreamTaskNetworkInput.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void processBufferOrEvent(BufferOrEvent bufferOrEvent) throws IOException {
	if (bufferOrEvent.isBuffer()) {
		lastChannel = channelIndexes.get(bufferOrEvent.getChannelInfo());
		checkState(lastChannel != StreamTaskInput.UNSPECIFIED);
		currentRecordDeserializer = recordDeserializers[lastChannel];
		checkState(currentRecordDeserializer != null,
			"currentRecordDeserializer has already been released");

		currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
	}
	else {
		// Event received
		final AbstractEvent event = bufferOrEvent.getEvent();
		// TODO: with checkpointedInputGate.isFinished() we might not need to support any events on this level.
		if (event.getClass() != EndOfPartitionEvent.class) {
			throw new IOException("Unexpected event: " + event);
		}

		// release the record deserializer immediately,
		// which is very valuable in case of bounded stream
		releaseDeserializer(channelIndexes.get(bufferOrEvent.getChannelInfo()));
	}
}
Example 5
Source File: MockInputGate.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
public Optional<BufferOrEvent> getNext() {
	BufferOrEvent next = bufferOrEvents.poll();
	if (!finishAfterLastBuffer && bufferOrEvents.isEmpty()) {
		availabilityHelper.resetUnavailable();
	}
	if (next == null) {
		return Optional.empty();
	}

	int channelIdx = next.getChannelInfo().getInputChannelIdx();
	if (closed[channelIdx]) {
		throw new RuntimeException("Inconsistent: Channel " + channelIdx
			+ " has data even though it is already closed.");
	}
	if (next.isEvent() && next.getEvent() instanceof EndOfPartitionEvent) {
		closed[channelIdx] = true;
	}
	return Optional.of(next);
}
Example 6
Source File: StreamTaskNetworkInput.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void processBufferOrEvent(BufferOrEvent bufferOrEvent) throws IOException {
	if (bufferOrEvent.isBuffer()) {
		lastChannel = bufferOrEvent.getChannelIndex();
		currentRecordDeserializer = recordDeserializers[lastChannel];
		currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
	}
	else {
		// Event received
		final AbstractEvent event = bufferOrEvent.getEvent();
		// TODO: with checkpointedInputGate.isFinished() we might not need to support any events on this level.
		if (event.getClass() != EndOfPartitionEvent.class) {
			throw new IOException("Unexpected event: " + event);
		}
	}
}
Example 7
Source File: CheckpointedInputGate.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public Optional<BufferOrEvent> pollNext() throws Exception {
	while (true) {
		Optional<BufferOrEvent> next = inputGate.pollNext();

		if (!next.isPresent()) {
			return handleEmptyBuffer();
		}

		BufferOrEvent bufferOrEvent = next.get();
		checkState(!barrierHandler.isBlocked(bufferOrEvent.getChannelInfo()));

		if (bufferOrEvent.isBuffer()) {
			return next;
		}
		else if (bufferOrEvent.getEvent().getClass() == CheckpointBarrier.class) {
			CheckpointBarrier checkpointBarrier = (CheckpointBarrier) bufferOrEvent.getEvent();
			barrierHandler.processBarrier(checkpointBarrier, bufferOrEvent.getChannelInfo());
			return next;
		}
		else if (bufferOrEvent.getEvent().getClass() == CancelCheckpointMarker.class) {
			barrierHandler.processCancellationBarrier((CancelCheckpointMarker) bufferOrEvent.getEvent());
		}
		else {
			if (bufferOrEvent.getEvent().getClass() == EndOfPartitionEvent.class) {
				barrierHandler.processEndOfPartition();
			}
			return next;
		}
	}
}
Example 8
Source File: CheckpointedInputGate.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Override
public Optional<BufferOrEvent> pollNext() throws Exception {
	while (true) {
		// process buffered BufferOrEvents before grabbing new ones
		Optional<BufferOrEvent> next;
		if (bufferStorage.isEmpty()) {
			next = inputGate.pollNext();
		}
		else {
			// TODO: FLINK-12536 for non credit-based flow control, getNext method is blocking
			next = bufferStorage.pollNext();
			if (!next.isPresent()) {
				return pollNext();
			}
		}

		if (!next.isPresent()) {
			return handleEmptyBuffer();
		}

		BufferOrEvent bufferOrEvent = next.get();
		if (barrierHandler.isBlocked(offsetChannelIndex(bufferOrEvent.getChannelIndex()))) {
			// if the channel is blocked, we just store the BufferOrEvent
			bufferStorage.add(bufferOrEvent);
			if (bufferStorage.isFull()) {
				barrierHandler.checkpointSizeLimitExceeded(bufferStorage.getMaxBufferedBytes());
				bufferStorage.rollOver();
			}
		}
		else if (bufferOrEvent.isBuffer()) {
			return next;
		}
		else if (bufferOrEvent.getEvent().getClass() == CheckpointBarrier.class) {
			CheckpointBarrier checkpointBarrier = (CheckpointBarrier) bufferOrEvent.getEvent();
			if (!endOfInputGate) {
				// process barriers only if there is a chance of the checkpoint completing
				if (barrierHandler.processBarrier(checkpointBarrier, offsetChannelIndex(bufferOrEvent.getChannelIndex()), bufferStorage.getPendingBytes())) {
					bufferStorage.rollOver();
				}
			}
		}
		else if (bufferOrEvent.getEvent().getClass() == CancelCheckpointMarker.class) {
			if (barrierHandler.processCancellationBarrier((CancelCheckpointMarker) bufferOrEvent.getEvent())) {
				bufferStorage.rollOver();
			}
		}
		else {
			if (bufferOrEvent.getEvent().getClass() == EndOfPartitionEvent.class) {
				if (barrierHandler.processEndOfPartition()) {
					bufferStorage.rollOver();
				}
			}
			return next;
		}
	}
}